import itertools
import functools
import sys
import time

def progress_animation():
    symbols = itertools.cycle('-\|/')
    while True:
        sys.stdout.write(f'\rworking... {next(symbols)}')
        sys.stdout.flush()
        time.sleep(0.1)

def progress_bar(current_index, max_index):
    """Prints progress in units of '-'
    """

    white_space = ' ' * 100
    sys.stdout.write(f'\r [{white_space}]')
    sys.stdout.flush()

    progress_done = int((current_index + 1) / max_index * 100)
    progress_remaining = 100 - progress_done

    sys.stdout.write('\r [' + '-' * progress_done + ' ' * progress_remaining + ']')
    sys.stdout.flush()

    #print()

def time_function(function):
    """Measures the time required for
    function completion
    """

    @functools.wraps(function)
    def wrapper(*args, **kwargs):
        # time before function call
        start_time = time.time()

        result = function(*args, **kwargs)

        # time after functtion executed
        end_time = time.time()

        # time required by function
        time_required = end_time - start_time
        print(f'{function.__name__} required {time_required}s for completion')

        return result
    return wrapper

# an example of a decorator with optional arguments
#
# the functools.partial cycles through the argument
# to bring the function to be wrapped to the keyword
# argument 'function'. It's important to have
# 'repeat=repeat' inside partial instead of just
# 'repeat' as an argument so that the cycling of
# the keyword arguments by partial brings the function
# to 'function' instead of 'repeat'
def repeat_time_function(function=None, repeat=1):
    """Measures the time required for 'repeat'
    number of function completions
    """
    if not function:
        return functools.partial(repeat_time_function, repeat=repeat)

    @functools.wraps(function)
    def wrapper(*args, **kwargs):

        print(f'Timer starting for function {function.__name__} with {repeat} repetitions')

        # time before function call
        start_time = time.time()

        for _ in range(repeat - 1):
            function(*args, **kwargs)
        result = function(*args, **kwargs)

        # time after functtion executed
        end_time = time.time()

        # time required by function
        time_required = end_time - start_time
        print(f'{function.__name__} required {time_required}s for completion')

        return result
    return wrapper

if __name__=="__main__":

    # should write a more optimized function
    @repeat_time_function(repeat=10)
    def perm_and_comb(array):

        size_double_array = len(array) * (len(array) - 1) / 2 * 2
        progress_index = 0

        counter_combinations = 0
        combinations = itertools.combinations(array, 2)
        for combination in combinations:

            # # add to progress index
            # progress_index += 1
            # progress_bar(progress_index, size_double_array)

            if sum(combination) == 10:
                counter_combinations += 1

        counter_permutations = 0
        permutations = itertools.permutations(array, 2)
        for permutation in permutations:

            # # add to progress index
            # progress_index += 1
            # progress_bar(progress_index, size_double_array)

            if sum(permutation) == 10:
                counter_permutations += 1

        return counter_combinations, counter_permutations

    test_array = list(range(2000))
    print(perm_and_comb(test_array))


import unittest

from etl import extract_xls_html as etl
from fixtures import TEST_CONTRIBUTIONS_LIST, TEST_BUSINESS_LICENSE_LIST

class TestExtractHtmlData(unittest.TestCase):

    def test_extracts_headers(self):

        data = etl.extract_data_from_html(TEST_CONTRIBUTIONS_LIST)

        self.assertEqual(
            data.headers,
            ['Contribution Date',
             'Contributor Name',
             'Contributor Address']
        )

    def test_extracts_data_to_py_objs(self):

        data = etl.extract_data_from_html(TEST_CONTRIBUTIONS_LIST)

        self.assertEqual(
            list(data),
            [
                {
                    'Contribution Date': '2/2/2004',
                    'Contributor Name': 'Inc, Caldera   Management',
                    'Contributor Address': '4260 Hwy 1, Rehoboth, DE 19971'
                },
                {
                    'Contribution Date': '9/22/2004',
                    'Contributor Name': 'Conset-vancy/LLC, Rehoboth   Bay',
                    'Contributor Address': '1207 Delaware AveWilinin too/DE 19806, DE',

                },
                {
                    'Contribution Date': '9/17/2004',
                    'Contributor Name': 'lingua, James.A',
                    'Contributor Address': '28 The Circle.Georgetown/DE 19947, DE',

                }
            ]
        )

    def test_extracts_business_list_headers(self):
        headers = [
            'Business name',
            'Trade name',
            'Category',
            'From',
            'To',
         ]

        data = etl.extract_data_from_html(
            TEST_BUSINESS_LICENSE_LIST, headers=headers)

        self.assertEqual(data.headers, headers)

    def test_extracts_business_list_data_to_py_objs(self):

        headers = [
            'Business name',
            'Trade name',
            'Category',
            'Current License Valid From',
            'Current License Valid To',
            'Address 1',
            'Address 2',
            'City',
            'State',
            'Zip',
            'Country',
            'License number'
        ]

        expected = [
            {
                'Business name': u'000 CHOICE MANAGEMENT INC',
                'Trade name': u' ',
                'Category': u'PROFESSIONAL AND/OR PRSL SRVCS-UNCLASSIFIED',
                'Current License Valid From': u'1/1/05',
                'Current License Valid To': u'12/31/07',
                'Address 1': u' ',
                'Address 2': u'7904 COASTAL HWY #5',
                'City': u'OCEAN CITY',
                'State': u'MD',
                'Zip': u'21842-6707',
                'Country': u' ',
                'License number': u'1999202399',
            },
        ]

        data = etl.extract_data_from_html(
            TEST_BUSINESS_LICENSE_LIST, headers=headers, skip_rows=2)

        self.maxDiff = None
        for got, expect in zip(list(data), expected):
            self.assertEqual(got, expect)

if __name__ == '__main__':
    unittest.main()


'''
Created on 2012-8-28

@author: zongzong
'''
from core.basehandler import BaseHandler
from tornado.web import authenticated
from views.decorators import route
from views.decorators import role_required
from models.user import User

@route('/get_users')
class OperatersHandler(BaseHandler):

    @authenticated
    @role_required('/get_users')
    def post(self):
        template_values = {}
        template_values['users'] = User.getUsers(2)
        template_values['usertype']=2
        self.render_template('/site/users.html', **template_values)

    @authenticated
    @role_required('/get_users')
    def get(self):
        template_values = {}
        template_values['users'] = User.getUsers(2)
        template_values['usertype']=2
        self.render_template('/site/users.html', **template_values)

@route('/get_admin')
class AdministratorsHandler(BaseHandler):

    @authenticated
    @role_required('/get_admin')
    def post(self):
        template_values = {}
        template_values['users'] = User.getUsers(1)
        template_values['usertype']=1
        self.render_template('/site/users.html', **template_values)

    @authenticated
    @role_required('/get_admin')
    def get(self):
        template_values = {}
        template_values['users'] = User.getUsers(1)
        template_values['usertype']=1
        self.render_template('/site/users.html', **template_values)

@route('/user_del')
class DelUserHandler(BaseHandler):
    def post(self):
        _id = self.get_argument("u_id", None)
        User.updateUser(_id,1)
        self.finish("finished")

@route('/user_active')
class ActiveUserHandler(BaseHandler):
    def post(self):
        _id = self.get_argument("u_id", None)
        User.updateUser(_id,0)
        self.finish("finished")


from QUANTAXIS.QAFetch.QAQuery import QA_fetch_stock_list
from QUANTAXIS.QAFetch import QA_fetch_get_stock_realtime
from QUANTTOOLS.message_func.wechat import send_actionnotice
from QUANTTOOLS.QAStockETL.QAFetch import QA_fetch_stock_fianacial_adv
from QUANTAXIS.QAFetch.QAQuery_Advance import QA_fetch_stock_day_adv
import pandas as pd
import logging
import strategyease_sdk
from QUANTTOOLS.account_manage.setting import yun_ip, yun_port, easytrade_password
import time
import datetime
from QUANTTOOLS.account_manage.trading_message import send_trading_message
from QUANTAXIS.QAUtil import QA_util_get_last_day
from QUANTAXIS.QAFetch.QAQuery import QA_fetch_stock_to_market_date
from QUANTAXIS.QAUtil import QA_util_today_str
import math

def date_func(date):
    if (date is None) or date in ['None', 0, '0']:
        d2 = datetime.datetime.strptime(QA_util_today_str(),"%Y-%m-%d")
    else:
        d2=datetime.datetime.strptime(date,"%Y%m%d")
    d1 = datetime.datetime.strptime(QA_util_today_str(),"%Y-%m-%d")
    diff_days=d1-d2
    return(diff_days.days)

def func1(x,y):
    if x == 0:
        return y
    else:
        return x

def floor_round(x):
    if isinstance(x, int):
        y = math.floor(x/100)*100
    else:
        y = x

    if y > x:
        return(x)
    else:
        return(y)

def re_build(target, positions, sub_accounts, trading_date, percent, exceptions, k=100):
    sub_accounts= sub_accounts -10000
    positions = positions[positions['股票余额'].astype(float) > 0]
    positions = positions[positions['股票余额'].astype(float) > 0]
    positions['上市时间'] = positions['证券代码'].apply(lambda x:date_func(str(QA_fetch_stock_to_market_date(x))))

    if exceptions is not None:
        exceptions = exceptions.extend(list(positions[positions['上市时间'] <= 15].set_index('证券代码').index))
    else:
        exceptions = list(positions[positions['上市时间'] <= 15].set_index('证券代码').index)

    if target is None:
        res = pd.concat([positions.set_index('证券代码'),
                         QA_fetch_stock_fianacial_adv(list(positions.set_index('证券代码').index), trading_date, trading_date).data.reset_index('date')[['NAME','INDUSTRY']],
                         QA_fetch_stock_day_adv(list(positions.set_index('证券代码').index),QA_util_get_last_day(trading_date,60),trading_date).to_qfq().data.loc[trading_date].reset_index('date')['close']],
                        axis=1)
        if exceptions is not None:
            exceptions_list = [i for i in list(res.index) if i not in exceptions]
            res = res.loc[exceptions_list]
        else:
            pass
        avg_account = 0
        res = res.assign(tar=avg_account)
        res['cnt'] = 0
        res['real'] = 0
        res['mark'] = (res['cnt'] - res['股票余额'].apply(lambda x:float(x))).apply(lambda x:floor_round(x))
    else:
        tar1 = target.reset_index().groupby('code').max()
        tar1['double'] = target.reset_index().groupby('code')['RANK'].count()
        target = tar1
        if exceptions is not None:
            exceptions_list = [i for i in list(target.index) if i not in exceptions]
            exceptions_listb = [i for i in list(positions.set_index('证券代码').index) if i not in exceptions]
            r1 = target.loc[exceptions_list].join(positions.set_index('证券代码').loc[exceptions_listb],how='outer')
        else:
            r1 = target.join(positions.set_index('证券代码'),how='outer')
        r1['股票余额'] = r1['股票余额'].fillna(0)
        realtm = QA_fetch_get_stock_realtime('tdx', code=[x for x in list(r1.index) if x in list(QA_fetch_stock_list().index)]).reset_index('datetime')[['ask1','ask_vol1','bid1','bid_vol1']]
        close = QA_fetch_stock_day_adv(list(r1.index),QA_util_get_last_day(trading_date,60),trading_date).data.loc[trading_date].reset_index('date')['close']
        res = r1.join(QA_fetch_stock_fianacial_adv(list(r1.index), trading_date, trading_date).data.reset_index('date')[['NAME','INDUSTRY']],how='left').join(realtm,how='left').join(close,how='left')
        avg_account = (sub_accounts * percent)/target['double'].sum()
        res = res.assign(tar=avg_account)
        res.ix[res['RANK'].isnull(),'tar'] = 0
        res['tar'] = res['tar'] * res['double']
        res['amt'] = res.apply(lambda x: func1(x['ask1'], x['bid1']),axis = 1)
        res['cnt'] = (res['tar']/res['amt']/100).apply(lambda x: round(x, 0)*100)
        res['real'] = res['cnt'] * res['amt']
        res = res.sort_values(by='amt', ascending= False)
        res = res.fillna(0)
        res1 = res[res['tar']>0]
        res2 = res[res['tar']==0]
        res1.ix[-1, 'cnt'] = round((res1['real'][-1]-(res1['real'].sum()-res1['tar'].sum()))/res1['ask1'][-1]/100,0)*100 - k
        res = pd.concat([res1,res2])
        res['real'] = res['cnt'] * res['amt']
        res['mark'] = (res['cnt'] - res['股票余额'].apply(lambda x:float(x))).apply(lambda x:math.floor(x/100)*100)
    return(res)

def build(target, positions, sub_accounts, trading_date, percent, exceptions, k=100):
    res = re_build(target, positions, sub_accounts, trading_date, percent, exceptions,k=k)
    while res['tar'].sum() < res['real'].sum():
        k = k + 100
        res = re_build(target, positions, sub_accounts, trading_date, percent, exceptions,k=k)
    return(res)

def trade_roboot(target, account, trading_date,percent, strategy_id, type='end', exceptions = None):
    logging.basicConfig(level=logging.DEBUG)
    client = strategyease_sdk.Client(host=yun_ip, port=yun_port, key=easytrade_password)
    account1=account
    client.cancel_all(account1)
    account_info = client.get_account(account1)
    sub_accounts = client.get_positions(account1)['sub_accounts']['总 资 产'].values[0]
    try:
        frozen = float(client.get_positions(account1)['positions'].set_index('证券代码').loc[exceptions]['市值'].sum())
    except:
        frozen = 0
    sub_accounts = sub_accounts - frozen
    positions = client.get_positions(account1)['positions'][['证券代码','证券名称','股票余额','可用余额','冻结数量','参考盈亏','盈亏比例(%)']]

    if target is None:
        e = send_trading_message(account1, strategy_id, account_info, None, "触发清仓", None, 0, direction = 'SELL', type='MARKET', priceType=4,price=None, client=client)

    res = build(target, positions, sub_accounts, trading_date, percent, exceptions, 100)
    res1 = res

    while (res[res['mark']<0].shape[0] + res[res['mark']>0].shape[0]) > 0:

        h1 = int(datetime.datetime.now().strftime("%H"))
        if h1 >= 15 or h1 <= 9:
            break

        if res[res['mark']<0].shape[0] == 0:
            pass
        else:
            for i in res[res['mark'] < 0].index:
                if type == 'end':
                    cnt = float(res.at[i, 'cnt'])
                    tar = float(res.at[i, '股票余额'])
                    NAME = res.at[i, 'NAME']
                    INDUSTRY = res.at[i, 'INDUSTRY']
                    mark = abs(float(res.at[i, 'mark']))

                    print('卖出 {code}({NAME},{INDUSTRY}) {cnt}股, 目标持仓:{target},总金额:{tar}'.format(code=i,
                                                                                NAME= NAME,
                                                                                INDUSTRY= INDUSTRY,
                                                                                cnt=abs(mark),
                                                                                target=cnt,
                                                                                tar=tar))
                    e = send_trading_message(account1, strategy_id, account_info, i, NAME, INDUSTRY, mark, direction = 'SELL', type='MARKET', priceType=4, price=None, client=client)
                elif type == 'morning':
                    cnt = float(res.at[i, 'cnt'])
                    tar = float(res.at[i, '股票余额'])
                    NAME = res.at[i, 'NAME']
                    INDUSTRY = res.at[i, 'INDUSTRY']
                    mark = abs(float(res.at[i, 'mark']))
                    price = round(float(res.at[i, 'close']*1.0995),2)
                    print('早盘挂单卖出 {code}({NAME},{INDUSTRY}) {cnt}股, 目标持仓:{target},单价:{price},总金额:{tar}'.format(code=i,
                                                                                NAME= NAME,
                                                                                INDUSTRY= INDUSTRY,
                                                                                cnt=abs(mark),
                                                                                target=cnt,
                                                                                tar=tar,
                                                                                price=price))
                    e = send_trading_message(account1, strategy_id, account_info, i, NAME, INDUSTRY, mark, direction = 'SELL', type='LIMIT', priceType=None, price=price, client=client)
                else:
                    pass
                time.sleep(3)

            time.sleep(15)

        if res[res['mark'] == 0].shape[0] == 0:
            pass
        else:
            for i in res[res['mark'] == 0].index:
                cnt = float(res.at[i, 'cnt'])
                tar = float(res.at[i, 'real'])
                NAME = res.at[i, 'NAME']
                INDUSTRY = res.at[i, 'INDUSTRY']
                mark = abs(float(res.at[i, 'mark']))
                print('继续持有 {code}({NAME},{INDUSTRY}), 目标持仓:{target},总金额:{tar}'.format(code=i,
                                                                                NAME= NAME,
                                                                                INDUSTRY=INDUSTRY,
                                                                                target=cnt,
                                                                                tar=tar))
                send_actionnotice(strategy_id,
                                  account_info,
                                  '{code}({NAME},{INDUSTRY})'.format(code=i,NAME= NAME, INDUSTRY=INDUSTRY),
                                  direction = 'HOLD',
                                  offset='HOLD',
                                  volume=abs(mark)
                                  )
                time.sleep(5)

        if res[res['mark'] > 0].shape[0] == 0:
            pass
        else:
            for i in res[res['mark'] > 0].index:
                if type == 'end':
                    cnt = float(res.at[i, 'cnt'])
                    tar = float(res.at[i, 'real'])
                    NAME = res.at[i, 'NAME']
                    INDUSTRY = res.at[i, 'INDUSTRY']
                    mark = abs(float(res.at[i, 'mark']))
                    print('买入 {code}({NAME},{INDUSTRY}) {cnt}股, 目标持仓:{target},总金额:{tar}'.format(code=i,
                                                                                NAME= NAME,
                                                                                INDUSTRY=INDUSTRY,
                                                                                cnt=abs(mark),
                                                                                target=cnt,
                                                                                tar=tar))
                    e = send_trading_message(account1, strategy_id, account_info, i, NAME, INDUSTRY, mark, direction = 'BUY', type='MARKET', priceType=4, price = None, client=client)
                elif type == 'morning':
                    cnt = float(res.at[i, 'cnt'])
                    tar = float(res.at[i, 'real'])
                    NAME = res.at[i, 'NAME']
                    INDUSTRY = res.at[i, 'INDUSTRY']
                    mark = abs(float(res.at[i, 'mark']))
                    price = round(float(res.at[i, 'close']*(1-0.0995)),2)
                    print('早盘挂单买入 {code}({NAME},{INDUSTRY}) {cnt}股, 目标持仓:{target},单价:{price},总金额:{tar}'.format(code=i,
                                                                                NAME= NAME,
                                                                                INDUSTRY=INDUSTRY,
                                                                                cnt=abs(mark),
                                                                                target=cnt,
                                                                                price=price,
                                                                                tar=tar))
                    e = send_trading_message(account1, strategy_id, account_info, i, NAME, INDUSTRY, mark, direction = 'BUY', type='LIMIT', priceType=None, price=price, client=client)
                else:
                    pass
                time.sleep(3)

            time.sleep(15)

        if type == 'end':
            #sub_accounts = client.get_positions(account1)['sub_accounts']['总 资 产'].values[0] - frozen
            positions = client.get_positions(account1)['positions'][['证券代码','证券名称','股票余额','可用余额','冻结数量','参考盈亏','盈亏比例(%)']]
            res = build(target, positions, sub_accounts, trading_date, percent, exceptions,100)
        elif type == 'morning':
            break
        else:
            break

    return(res1)

if __name__ == '__main__':
    pass


from tezromach.layers import Normalization, Linear, Tanh, Sigmoid, ReLU
from tezromach.network import NeuralNet
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn import preprocessing

encoder = preprocessing.OneHotEncoder(sparse=False)

inputs = np.genfromtxt("mnist_data/inputs.csv", delimiter=',')
targets = np.genfromtxt("mnist_data/targets.csv", delimiter=',')
targets = encoder.fit_transform(np.array(targets).reshape(-1, 1))

inputs_train, inputs_test, targets_train, targets_test = train_test_split(inputs, targets, test_size=0.15)
print("inputs shape =", inputs.shape)
print("targets shape =", targets.shape)

net = NeuralNet([
    Normalization(),

    Linear(input_size=64, output_size=100),
    ReLU(),
    Linear(input_size=100, output_size=80),
    Sigmoid(),
    Linear(input_size=80, output_size=10),
    Sigmoid()
])

print("start fit NeuralNet")
net.fit(inputs_train, targets_train, verbose_print=10, num_epochs=200, learning_rate=0.1)
print("done fit NeuralNet")
print(net)

net.save_model("model_mnist")

predicted = net.predict(inputs_train)
predicted = np.argmax(predicted, axis=1)
predicted_targets_train = np.argmax(targets_train, axis=1)

print("accuracy metrics (train): ", metrics.accuracy_score(predicted, predicted_targets_train))

predicted = net.predict(inputs_test)
predicted = np.argmax(predicted, axis=1)
predicted_targets_test = np.argmax(targets_test, axis=1)

print("accuracy metrics (test): ", metrics.accuracy_score(predicted, predicted_targets_test))


###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Crossbar.io Technologies GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

__all = (
    'sleep',
    'peer2str',
)

def peer2str(peer):
    if isinstance(peer, tuple):
        ip_ver = 4 if len(peer) == 2 else 6
        return u"tcp{2}:{0}:{1}".format(peer[0], peer[1], ip_ver)
    elif isinstance(peer, str):
        return u"unix:{0}".format(peer)
    else:
        return u"?:{0}".format(peer)

def get_serializers():
    from autobahn.wamp import serializer

    serializers = ['CBORSerializer', 'MsgPackSerializer', 'UBJSONSerializer', 'JsonSerializer']
    serializers = list(filter(lambda x: x, map(lambda s: getattr(serializer, s) if hasattr(serializer, s)
                                               else None, serializers)))
    return serializers


import requests
from django.conf import settings
from django.db.models import Q

from company.models import Company
from core.models import Specialization, JobDuties, Duty
from vacancy.models import Skill
from vacancy.models import Vacancy, Job
from .constants import *
from .models import VacancySkills, VacancyJobs

def filter_by_request_types(list_roles, user):
    q = Q()
    if list_roles:
        for role in list_roles:
            if role == 'creator':
                companies_ids = Company.objects.filter(profile=user).values_list('id', flat=True).distinct()
                vacancies_ids = Vacancy.objects.filter(company_id__in=companies_ids).values_list('id',
                                                                                flat=True).distinct()
                q = q | Q(vacancy_id__in=vacancies_ids)
            elif role == 'member':
                q = q | Q(user=user)

    else:
        q = Q(~Q(creator=user) & ~Q(members=user))

    return q

def filter_by_skills(skill_text):
    if skill_text:
        skill = Skill.objects.filter(text__icontains=skill_text.lower()).values_list('id', flat=True)
        return Q(skills__in=skill)
    else:
        return Q()

def filter_by_specializations(list_specializations):
    if list_specializations:
        specializations = Specialization.objects.filter(id__in=list_specializations).values_list('id', flat=True)
        return Q(specializations__in=specializations)
    else:
        return Q()

def filter_by_text(text):
    if text:
        return Q(Q(name__icontains=text) | Q(description__icontains=text))
    else:
        return Q()

def filter_by_experience_type(experience_type):
    if experience_type:
        return Q(experience_type=experience_type)
    else:
        return Q()

def filter_by_type_of_work(type_of_work):
    if type_of_work:
        return Q(type_of_work=type_of_work)
    else:
        return Q()

def setup_vacancy_display(vacancies):
    for vacancy in vacancies:
        setup_single_vacancy_display(vacancy)

    return vacancies

def setup_single_vacancy_display(vacancy):
    vacancy['employment_type'] = {'id': vacancy.get('employment_type'),
                                  'text': Constants().get_employment_types(vacancy.get('employment_type'))}

    vacancy['experience_type'] = {'id': vacancy.get('experience_type'),
                                  'text': Constants().get_experience_types(vacancy.get('experience_type'))}

    vacancy['schedule_type'] = {'id': vacancy.get('schedule_type'),
                                'text': Constants().get_schedule_types(vacancy.get('schedule_type'))}

    return vacancy

def create_vacancy_skills(vacancy, skills):
    VacancySkills.objects.filter(vacancy_id=vacancy.id).delete()
    for string_skill in skills:
        skill = Skill.objects.filter(text=string_skill)

        if not skill:
            skill = Skill.objects.create(text=string_skill)
        else:
            skill = skill.get()

        VacancySkills.objects.create(vacancy=vacancy, skill=skill)

def create_vacancy_jobs(vacancy, jobs):
    VacancyJobs.objects.filter(vacancy_id=vacancy.id).delete()
    for job in jobs:
        duties = job.pop('duties')
        job = Job.objects.create(**job)
        for duty in duties:
            duty = Duty.objects.create(text=duty)
            JobDuties.objects.create(job=job, duty=duty)
        VacancyJobs.objects.create(vacancy=vacancy, job=job)

class ListAsQuerySet(list):

    def __init__(self, *args, model, **kwargs):
        self.model = model
        super().__init__(*args, **kwargs)

    def filter(self, *args, **kwargs):
        return self  # filter ignoring, but you can impl custom filter

    def order_by(self, *args, **kwargs):
        return self

def get_super_job_vacancies(keywords, type_of_work, experience):
    app_url = 'https://api.superjob.ru/2.20/vacancies/'
    period = 0
    town = 'Ставрополь'
    order_field = 'date'
    order_direction = 'desc'
    count = 10

    # , 'type_of_work': type_of_work, 'experience': experience
    headers = {'Content-Type': 'application/json', 'X-Api-App-Id': settings.SUPER_JOB_SECRET_KEY}
    payload = {'period': period, 'town': town, 'order_field': order_field, 'order_direction': order_direction,
               'count': count, 'keywords': keywords, 'type_of_work': type_of_work, 'experience': experience, }

    response = requests.get(app_url, headers=headers, params=payload)
    vacancies = list([])
    for vacancy_json in response.json().get('objects'):
        vacancy = dict({})

        vacancy['id'] = 1000

        vacancy['name'] = vacancy_json.get('profession')
        vacancy['short_description'] = vacancy_json.get('candidat')
        vacancy['description'] = vacancy_json.get('candidat')
        vacancy['salary'] = vacancy_json.get('payment_from')
        vacancy['experience_type'] = vacancy_json.get('experience').get('id')

        if vacancy_json.get('type_of_work').get('id') == 6:
            vacancy['schedule_type'] = 1
        elif vacancy_json.get('type_of_work').get('id') == 10:
            vacancy['schedule_type'] = 2
        elif vacancy_json.get('type_of_work').get('id') == 7:
            vacancy['schedule_type'] = 3

        vacancy['employment_type'] = 1

        vacancy['external'] = True
        vacancy['link'] = vacancy_json.get('link')

        company_json = {'name': vacancy_json.get('firm_name')}
        company = Company(**company_json)
        vacancy['company'] = company

        instance = Vacancy(**vacancy)
        vacancies.append(instance)

    return vacancies


import numpy as np
import pandas as pd
import pickle

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

class FeaturePreparer:
    # when we call the Feature preparer for the first time
    # we initialise it with the training data set, which is
    # then stored as an attribute(raw_data)
    def __init__(self, *, raw_data):
        """
        Initialize the object with a file in the data/raw folder
        :param raw_data:
        """
        self.raw_data = pd.read_csv(f"../data/raw/{raw_data}")
        self.prepared_data = None
        self.X_train = None
        self.X_test = None
        self.y_train = None
        self.y_test = None
        self.categorical_variables = None
        self.low_card_vars = None
        self.high_card_vars = None
        self.label_encoder = LabelEncoder()

    def add_labels(self) -> None:
        """Adds data labels to the raw training set"""
        labels = pd.read_csv("../data/raw/train_labels.csv")

        self.prepared_data = pd.merge(self.prepared_data, labels, how="left", left_on="id", right_on="id")

    def filter_data(self) -> None:
        """Adds extra columns and filters out necessary columns"""

        self.prepared_data.drop("basin", axis=1)

        self.prepared_data["installed_DWE"] = np.where(self.prepared_data["installer"] == "DWE", 1, 0)

        self.prepared_data["has_wpt"] = np.where(self.prepared_data["wpt_name"] != "none", 1, 0)

        pickle_in = open("../parameters/model_features.pkl", "rb")

        columns_to_keep = pickle.load(pickle_in)

        print(f"Columns for modeling {columns_to_keep}")

        self.prepared_data = self.prepared_data[columns_to_keep]

    def separate_variables(self) -> None:
        self.categorical_variables = [var for var in self.prepared_data.columns if self.prepared_data[var].dtypes == "O"]

        self.low_card_vars = [var for var in self.categorical_variables if self.prepared_data[var].nunique() < 20]

        self.high_card_vars = [var for var in self.categorical_variables if var not in self.low_card_vars]

        print(f"There are {len(self.low_card_vars)} low cardinality (n < 20) categorical variables")

        print(f"There are {len(self.high_card_vars)} high cardinality (n > 20) categorical variables")

    def handle_na(self) -> None:

        #TODO: Should this be handled better?

        self.prepared_data.dropna()
        print(f"Shape of DataFrame after cleaning: {self.prepared_data.shape}")

    def encode_categorical_variables(self):
        self.low_card_vars.remove("status_group")

        # TODO: Remove once I find out how to encode high cardinality variables
        self.prepared_data.drop(self.high_card_vars, axis=1, inplace=True)

        dummies = pd.get_dummies(self.prepared_data[self.low_card_vars])

        self.prepared_data = pd.concat([self.prepared_data, dummies], axis=1)

        self.prepared_data.drop(self.low_card_vars, axis=1, inplace=True)

        print(f"Categorical variables encoded, new shape is: {self.prepared_data.shape}")

    def split_data(self) -> None:
        X = self.prepared_data.drop("status_group", axis=1)

        y = self.prepared_data["status_group"]

        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
            X, y, test_size=0.2, random_state=42)

        print(f"X_train shape: {self.X_train.shape}, X_test shape: {self.X_test.shape}")

    def encode_label(self) -> None:
        self.y_train = self.label_encoder.fit_transform(self.y_train)

        self.y_test = self.label_encoder.transform(self.y_test)

        print("Labels encoded")

    def prepare_data(self):
        """Runs all data preparation steps"""
        self.prepared_data = self.raw_data.copy(deep=True)

        self.add_labels()
        self.filter_data()
        self.separate_variables()
        self.handle_na()
        self.encode_categorical_variables()
        self.split_data()
        self.encode_label()

        print("-" * 10, "Data prepared", "-" * 10)

        self.save_data()

    def save_data(self) -> None:
        self.prepared_data.to_csv("../data/interim/df_interim.csv", index = False)
        print("Data saved")

if __name__ == "__main__":
    test = FeaturePreparer(raw_data="train_data.csv")
    test.prepare_data()


import pandas as pd
import numpy as np
import io
import decimal as de
import sys

# import from system
windoww=sys.argv[1]
acttual=sys.argv[2]
predictedd =sys.argv[3]
outtput=sys.argv[4]

# import data
# actual_path=system_path+'/insight_testsuite/tests/my_own_test_1/input/actual.txt'
# predicted_path=system_path+'/insight_testsuite/tests/my_own_test_1/input/predicted.txt'
# system_path= os.getcwd()[:-4]

actual=pd.read_table(acttual, sep='|',header=None)
predicted=pd.read_table(predictedd, sep='|',header=None)

# pick up the right moving time window
# window_path=system_path+'/insight_testsuite/tests/my_own_test_1/input/window.txt'
f = io.open(windoww,'r')
message = f.read()
f.close()
sliding_window=int(message)

# add a header for easier manipulation
actual.columns=['hour','stock_code', 'stock_price']
predicted.columns=['hour','stock_code', 'stock_price']

# construct a moving window of elements to take average
predicted_time=predicted['hour'].tolist()
predicted_time=list(set(predicted_time))
q=len(predicted_time)-sliding_window
begin=[]
end=[]
i=0
while i<=q:
    begin.append(predicted_time[i])
    end.append(predicted_time[i+sliding_window-1])
    i=i+1
begin=np.array(begin).T
end=np.array(end).T

# construct a join pandas dataframe
mergedStuff = pd.merge(actual, predicted, on=['hour','stock_code'], how='outer')
mergedStuff['diff'] = mergedStuff.apply(lambda x: abs(x['stock_price_x'] - x['stock_price_y']) if not np.isnan(x['stock_price_y']) or np.isnan(x['stock_price_x']) else np.nan, axis=1)
mergedStuff.sort_values(by=['hour'])

# extract the difference and the time

desired=mergedStuff[['hour','diff']]
desire=desired.dropna(subset=['diff'])
desire.sort_values(by=['hour'])

# extract the sum of differences in each hour
L=[]
for time in predicted_time:
    net=desire.loc[desired['hour']==time]['diff'].sum()
    weight=desire[desire['hour']==time].shape[0]
    L.append([net,weight])

# extract the moving averages

output=[]
i=0
while i<=q:
    diff=0
    temp=L[i:i+sliding_window]
    weight=0
    for item in temp:
        diff=diff+item[0]
        weight=weight+item[1]
    a=de.Decimal(diff)
    if weight!=0:
        c=a/de.Decimal(weight)
        d=c.quantize(de.Decimal('.00'))
        output.append(d)
    else:
        output.append('NA')
    i=i+1
output=np.array(output)

# create the output dataframe
df_begin=pd.DataFrame(begin)
df_end=pd.DataFrame(end)
value=pd.DataFrame(output.T)
frames=[df_begin, df_end,value]
result=pd.concat(frames,axis=1)

# write the output_file
# output_path=system_path+'/output/comparison.txt'
result.to_csv(outtput, header=None, sep="|", index=False)


from bitarray import bitarray
import sys
import os
def binStr(num, length):
    return bin(num)[2:].zfill(length)

def bytesToBin(bytes):
    if len(bytes) == 0:
        return ''
    else:
        return bin(int.from_bytes(bytes, byteorder='big'))[2:].zfill(8 * len(bytes))

def lz77_enc(buffer, wBits, lBits):
    maxWindow = (2**wBits) -1
    a = bitarray()
    w = i = 0
    while i < len(buffer):
        word = buffer[i:i + 1]
        l = 0
        back = 0
        first = i
        find = buffer.rfind(word, w, i)
        while find != -1:
            back = first - find
            l += 1
            i += 1
            # length can't exceed the end or our bit limit
            if i >= len(buffer) - 1 or i - find >= (2**lBits):
                break
            word += buffer[i:i + 1]
            wl = len(word)
            # only rfind again if next 2 chars not same
            if buffer[i:i + 1] != buffer[find + wl - 1:find + wl]:
                find = buffer.rfind(word, w, i)
        next = buffer[i:i + 1]
        i += 1
        if i - w > maxWindow:  # slide the window if exceeded max size
            w += i - w - maxWindow
        ls = binStr(l, lBits)
        a += binStr(back, wBits) + ls + bytesToBin(next)
    return a

#compresses a given file with the input number of wBits
def main():
    fn = sys.argv[1]
    wBits = int(sys.argv[2])
    lBits = int(sys.argv[3])
    plainFile = open(fn, 'rb')
    plainText = plainFile.read()
    encodedBitArr = lz77_enc(plainText, wBits, lBits)
    compressed_fn = os.path.splitext(fn)[0] + '.lz77'
    with open(compressed_fn, 'wb') as file:
        encodedBitArr.tofile(file)
    print('Your encoded data is stored in file', compressed_fn)
    return False

if __name__ == "__main__":
    main()


#!/usr/bin/python3

# Creates a TXF file for Form 1099-B to populate Form 8949.
# https://turbotax.intuit.com/txf/TXF042.jsp
# Look for codes 321, 711, 712, 323, 713, 714.

# python create-txf-2015.py 1099-b.csv > 1099-b.txf
# If you have multiple accounts, you may combine the .csv files by
# sorting them before creating the .txf file from the combined file.

# Import the .txf file into TurboTax via
# "File > Import > From TXF Files".
# You should see this:
# These Documents Are Now Ready for Import:
# - 1099-B (number of transactions)

# The result DOES NOT look correct in in EasyView.  You can verify it
# like this: "View > Go to Forms". Open "Form 8949". Review the
# transactions. Note that Line 1 in both Part I and II is scrollable
# if you have more than six transactions.  Also, there are multiple
# copies of the form if more than one of A, B, C or D, E, F is
# checked.

# If you don't like what you see, you can remove the imported data via
# "File > Remove Imported Data".

import sys
import csv
import datetime

box_dict = {'A': 321, 'B': 711, 'C': 712, 'D': 323, 'E': 713, 'F': 714}
# Only two taxrefs for wash sales are documented in Section IV.  One
# of those is used in Example 2C.  Example 2G for a long-term wash
# sale uses taxref 713.
wash_box_dict = {'A': 682, 'B': 718, 'C': 712, 'D': 323, 'E': 713, 'F': 714}

print('V042')
print('Aself')
print('D', datetime.date.today().strftime('%m/%d/%y'))
print('^')

with open(sys.argv[1], 'r') as csvfile:
    for row in csv.reader(csvfile):
        symbol = row[0]
        count = row[1]
        acquired = row[2]
        disposed = row[3]
        proceeds = row[4]
        try:
            # TXF042.jsp, example 2B; may be reported as "Missing" in 1099-B.
            float(row[5])
            base = row[5]
        except ValueError:
            base = ''
        adj = row[6]
        gain = row[7]
        adj_code = row[9]
        box = row[10]
        taxref = wash_box_dict[box] if adj_code == 'W' else box_dict[box]
        prefix = ' '
        if '/' in symbol:
            prefix = ' opt '
        else:
            prefix = ' sh '
        if float(count) < 0:
            prefix = ' short' + prefix
            count = count[1:]
            # https://www.irs.gov/pub/irs-pdf/i8949.pdf
            # Column (b)-Date Acquired
            # For a short sale, enter the date you acquired the property
            # delivered to the broker or lender to close the short sale.
            acquired = disposed
        descr = count + prefix + symbol
        print('TD')
        print('N' + str(taxref))
        print('P ' + descr)
        print('D ' + acquired.replace('VARIOUS', 'Various'))
        print('D ' + disposed)
        print('$' + base)
        print('$' + proceeds)
        # TurboTax 2016 ignores the sign of the adjustment and always puts "W"
        # in Box (f) if an adjustment is present.  For other types of
        # adjustments, e.g., "B", one has to override the values in Form 8949
        # in Forms View, including the new value in Box (h).
        if adj != '--' and adj != '0.00':
            print('$' + adj)
        print('^')


# -*- coding: utf-8 -*-

import logging

from odoo import models, fields, api
from odoo.addons.http_routing.models.ir_http import slugify

_logger = logging.getLogger(__name__)

class ActivityArea(models.Model):
    """
    Activity states of a company
    """
    _name = 'ar.activity_area'
    _description = 'Secteurs d\'activités d\'une entreprise'

    name = fields.Char(string='Nom du secteur', required=True)
    slug = fields.Char(compute='_slugify_name', store=True)

    @api.depends('name')
    def _slugify_name(self):
        self.slug = slugify(self.name)

class PartnerTypes(models.Model):
    """
    Types of contacts
    """
    _name = 'ar.partner_type'
    _description = 'Type de contacts'

    name = fields.Char(string='Nom', required=True)
    slug = fields.Char(compute='_slugify_name', store=True)

    @api.depends('name')
    def _slugify_name(self):
        self.slug = slugify(self.name)

class Partner(models.Model):
    """
    Add fields to contacts
    """
    _inherit = 'res.partner'

    def _get_defaultpartner_type(self):
        """
        :return: the default partner type
        """
        return self.env['ar.partner_type'].search([('slug', '=', 'autre')], limit=1)

    referent = fields.Many2one(
        'res.partner',
        string='Référent au sein d\'Aveyron Recrute',
        required=False,
        domain=[["user_ids.groups_id.name", "ilike", "internal"]]
    )

    partner_type = fields.Many2one(
        'ar.partner_type',
        string='Type de contact',
        required=False,
        default=_get_defaultpartner_type
    )

    presentation = fields.Text(
        string='Présentation',
        required=False
    )

    activity_areas = fields.Many2many(
        'ar.activity_area',
        string='Secteurs d\'Activités',
        required=False
    )

    employee_count = fields.Integer(
        string='Nombre d\'employés',
        required=False
    )

    needs = fields.Text(
        string='Mes besoins en recrutement les plus fréquents',
        required=False
    )

    year_birth = fields.Integer(
        string='Année de naissance',
        required=False,
        default=0
    )

    your_profession = fields.Text(
        string='Vos métiers / vos produits en synthèse',
        required=False
    )

    @api.model
    def send_contact_recruteur_email(self, partner_id, subject, message):
        partner_obj = self.env['res.partner'].browse(partner_id)

        if partner_obj is None:
            return False

        lang = partner_obj.user_id.lang
        template = self.env.ref('espace_recruteur.mail_template_contact_recruteur')
        template.with_context(dbname=self._cr.dbname, subject=subject, message=message, lang=lang).send_mail(
            partner_obj.id, force_send=True)

        return True

    @api.model
    def send_contact_candidat_email(self, partner_id, subject, message):
        partner_obj = self.env['res.partner'].browse(partner_id)

        if partner_obj is None:
            return False

        lang = partner_obj.user_id.lang
        template = self.env.ref('espace_candidat.mail_template_contact_candidat')
        template.with_context(dbname=self._cr.dbname, subject=subject, message=message, lang=lang).send_mail(
            partner_obj.id, force_send=True)

        return True

    @api.model
    def get_company(self, partner_id):
        partner = self.env['res.partner'].search_read(
            domain=[('id', '=', partner_id)],
            fields=('id', 'name', 'street', 'street2', 'zip', 'city', 'email', 'employee_count', 'website',
                    'activity_areas', 'presentation')
        )

        if not partner:
            return None

        areas = self.env['ar.activity_area'].search_read(
            domain=[('id', 'in', partner[0]['activity_areas'])],
            fields=('id', 'name')
        )

        partner[0]['activity_areas'] = areas

        return partner

    @api.model
    def get_all_companies(self):
        companies = self.env['res.company'].search_read(
            domain=[('partner_id', 'not ilike', 'aveyron recrute')],
            fields=('id', 'partner_id')
        )

        if len(companies) <= 0:
            return []

        partners = []

        for company in companies:
            if company['partner_id'] and company['partner_id'][0]:
                partners.append(self.get_company(company['partner_id'][0]))

        return partners

    @api.one
    def _prepare_model_for_send_website(self):
        partner = self.read(('id', 'name', 'street', 'street2', 'zip', 'city', 'email', 'employee_count', 'website',
                             'activity_areas', 'presentation'))

        if not partner:
            return None

        areas = self.env['ar.activity_area'].search_read(
            domain=[('id', 'in', partner['activity_areas'])],
            fields=('id', 'name')
        )

        partner['activity_areas'] = areas

        return partner

    @api.model
    def get_partner_with_profiles(self, email):
        base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
        partner = self.env['res.partner'].search([('email', 'ilike', email)], limit=1)

        if not partner:
            return False

        partner_dict = partner.read(['id', 'name', 'email'])
        partner_dict[0]['profiles'] = []
        profiles = self.env['hr.applicant'].search([('is_profile', '=', True), ('partner_id', '=', partner.id)])

        if len(profiles) > 0:
            partner_dict[0]['profiles'] = profiles.read([
                'id', 'name', 'description', 'formations', 'activities', 'attachment_ids',
                'diplomas', 'skills'
            ])

        for profile in partner_dict[0]['profiles']:
            if len(profile['formations']) > 0:
                profile['formations'] = self.env['ar.formation'].search_read(
                    domain=[('id', 'in', profile['formations'])],
                    fields=('id', 'name')
                )

            if len(profile['activities']) > 0:
                profile['activities'] = self.env['ar.activity_area'].search_read(
                    domain=[('id', 'in', profile['activities'])],
                    fields=('id', 'name')
                )

            if len(profile['attachment_ids']) > 0:
                profile['attachment_ids'] = self.env['ir.attachment'].search_read(
                    domain=[('id', 'in', profile['attachment_ids'])],
                    fields=('id', 'website_url')
                )

                for attachment in profile['attachment_ids']:
                    attachment['website_url'] = base_url + attachment['website_url']

            profile['diplomas'] = str(profile['diplomas']).split('|') if profile['diplomas'] else []
            profile['skills'] = str(profile['skills']).split('|') if profile['skills'] else []

        return partner_dict


import typing
import re
import time
import os
import random
import datetime

import jk_utils
import jk_json
import jk_typing

from .ArchivedBackupInfoFile import ArchivedBackupInfoFile
from .BackupDataFile import BackupDataFile

def _toInt(s:str) -> int:
	while (len(s) > 1) and s[0] == "0":
		s = s[1:]
	return int(s)
#

#
# This class represents a backup within the archive.
#
class Backup(object):

	################################################################################################################################
	## Constructors
	################################################################################################################################

	@jk_typing.checkFunctionSignature()
	def __init__(self, dirPath:str):
		self.__dirPath = dirPath
		self.__infoFile = ArchivedBackupInfoFile.loadFromFile(os.path.join(dirPath, "_backup_info.json"))

		self.__files = []
		for fe in os.scandir(dirPath):
			if bool(re.match("[a-zA-Z0-9]", fe.name[0])) and fe.is_file():
				self.__files.append(BackupDataFile(fe))

		self.__files = tuple(self.__files)

		self.__cached_dtBackupStart = None
		self.__cached_sBackupStartDate = None
		self.__cached_sBackupStartDateTime = None
	#

	################################################################################################################################
	## Public Properties
	################################################################################################################################

	@property
	def files(self) -> typing.Tuple[BackupDataFile]:
		return self.__files
	#

	@property
	def dirPath(self) -> str:
		return self.__dirPath
	#

	@property
	def dtBackupStart(self) -> datetime.datetime:
		if self.__cached_dtBackupStart is None:
			#self.__cached_dtBackupStart = datetime.datetime.fromtimestamp(self.__infoFile.getValue("tBackupStart"))
			self.__cached_dtBackupStart = self.__infoFile.getValue("tBackupStart").dateTime
		return self.__cached_dtBackupStart
	#

	#
	# Returns a human readable representation of the start time stamp in the form:
	# "YYYY-MM-DD"
	#
	@property
	def sBackupStartDate(self) -> str:
		dt = self.dtBackupStart
		if self.__cached_sBackupStartDate is None:
			self.__cached_sBackupStartDate = "{:04d}-{:02d}-{:02d}".format(
				dt.year,
				dt.month,
				dt.day,
			)
		return self.__cached_sBackupStartDate
	#

	#
	# Returns a human readable representation of the start time stamp in the form:
	# "YYYY-MM-DD hh:mm:ss"
	#
	@property
	def sBackupStartDateTime(self) -> str:
		dt = self.dtBackupStart
		if self.__cached_sBackupStartDateTime is None:
			self.__cached_sBackupStartDateTime = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}".format(
				dt.year,
				dt.month,
				dt.day,
				dt.hour,
				dt.minute,
				dt.second,
			)
		return self.__cached_sBackupStartDateTime
	#

	#
	# Returns a machine readable representation of the start time stamp in the form:
	# {
	#		"year": ...,
	#		"month": ...,
	#		"day": ...,
	#		"hour": ...,
	#		"minute": ...,
	#		"second": ...,
	# }
	#
	@property
	def jBackupStartDateTime(self) -> dict:
		dt = self.dtBackupStart
		return {
			"year": dt.year,
			"month": dt.month,
			"day": dt.day,
			"hour": dt.hour,
			"minute": dt.minute,
			"second": dt.second,
		}
	#

	#
	# Returns a machine readable representation of the start time stamp in the form:
	# ( ..year.., ..month.., ..day.., ..hour.., ..minute.., ..second.. )
	#
	@property
	def lBackupStartDateTime(self) -> tuple:
		dt = self.dtBackupStart
		return (
			dt.year,
			dt.month,
			dt.day,
			dt.hour,
			dt.minute,
			dt.second,
		)
	#

	@property
	def lBackupStartDate(self) -> tuple:
		dt = self.dtBackupStart
		return (
			dt.year,
			dt.month,
			dt.day,
		)
	#

	@property
	def timeStamp(self) -> jk_utils.TimeStamp:
		return self.__infoFile.getValue("tBackupStart")
	#

	@property
	def sizeInBytesUploaded(self) -> jk_utils.AmountOfBytes:
		return self.__infoFile.getValue("nTotalBytesUploaded")
	#

	@property
	def sizeInBytesArchived(self) -> jk_utils.AmountOfBytes:
		return self.__infoFile.getValue("nTotalBytesArchived")
	#

	@property
	def clientBackupDuration(self) -> float:
		return self.__infoFile.getValue("dClientBackupDuration")
	#

	@property
	def systemName(self) -> str:
		return self.__infoFile.getValue("systemName")
	#

	@property
	def backupUserName(self) -> str:
		return self.__infoFile.getValue("backupUserName")
	#

	@property
	def backupIdentifier(self) -> str:
		return self.__infoFile.getValue("backupIdentifier")
	#

	################################################################################################################################
	## Public Methods
	################################################################################################################################

	################################################################################################################################
	## Static Methods
	################################################################################################################################

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	@jk_typing.checkFunctionSignature()
	def buildRecommendedDirName(dt:datetime.datetime = None, systemName:str = None, backupUserName:str = None, backupIdentifier:str = None, backup = None) -> str:
		if backup is not None:
			assert isinstance(backup, Backup)
			dt = backup.dtBackupStart
			systemName = backup.systemName
			backupUserName = backup.backupUserName
			backupIdentifier = backup.backupIdentifier

		assert dt is not None
		Backup.verifySystemName(systemName)
		Backup.verifyBackupUserName(backupUserName)
		Backup.verifyBackupIdentifier(backupIdentifier)

		return "{}---{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}---{}---{}".format(
			systemName,
			dt.year,
			dt.month,
			dt.day,
			dt.hour,
			dt.minute,
			dt.second,
			backupUserName,
			backupIdentifier)
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def tryParseDirName(dirName:str) -> dict:
		m = re.match(r"^([a-zA-Z_\.0-9-\+%]+)---(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)---([a-zA-Z_\.0-9-\+%]+)---([a-zA-Z_\.0-9-\+%]+)$", dirName)
		if m:
			return {
				"systemName": m.group(1),
				"year": _toInt(m.group(2)),
				"month": _toInt(m.group(3)),
				"day": _toInt(m.group(4)),
				"hour": _toInt(m.group(5)),
				"minute": _toInt(m.group(6)),
				"second": _toInt(m.group(7)),
				"userName": m.group(8),
				"backupIdentifier": m.group(9),
			}
		return None
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def verifySystemName(systemName:str):
		assert re.match(r".*(---|\.\.|\s).*", systemName) is None
		assert re.match(r"^[a-zA-Z_\.0-9-\+%]+$", systemName) is not None
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def verifyBackupUserName(userName:str):
		assert re.match(r".*(---|\.\.|\s).*", userName) is None
		assert re.match(r"^[a-zA-Z_\.0-9-\+%]+$", userName) is not None
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def verifyBackupIdentifier(backupIdentifier:str):
		assert re.match(r".*(---|\.\.|\s).*", backupIdentifier) is None
		assert re.match(r"^[a-zA-Z_\.0-9-\+%]+$", backupIdentifier) is not None
	#

#


import Adafruit_DHT
import RPi.GPIO as GPIO
import time
from datetime import datetime
import temp_sensor

class manage_temp():

    def __init__(self,pin_sensor):
        self.issummer=False
        self.iswinter=False
        GPIO.setmode(GPIO.BCM)
        self.pin_sensor = pin_sensor
        self.sensor_com = 6
        self.sensor_heat = 2
        self.sensor_cool = 3
        GPIO.setup(self.sensor_com, GPIO.OUT)
        GPIO.setup(self.sensor_heat, GPIO.OUT)
        GPIO.setup(self.sensor_cool, GPIO.OUT)
        if datetime.now().month>=5 and datetime.now().month<11:
            self.set_summer_mode()
        elif datetime.now().month>=11 and datetime.now().month<5:
            self.set_winter_mode()

    def set_winter_mode(self):
        self.issummer = False
        self.iswinter = True
        GPIO.output(self.sensor_heat, True)  ## enciendo el pin
        GPIO.output(self.sensor_cool, False)  ## apago el pin

    def set_summer_mode(self):
        self.issummer =True
        self.iswinter = False
        GPIO.output(self.sensor_cool, True)  ## enciendo el pin
        GPIO.output(self.sensor_heat, False)  ## apago el pin

    def set_temp(self, temp_setter):
        self.temp_target = temp_setter
        self.get_actual_temp()
        if self.temperature() < temp_setter and self.iswinter:
            return self.set_on_mode()
        elif self.temperature() >= temp_setter and self.iswinter:
            return self.set_off_mode()
        elif self.temperature() <= temp_setter and self.issummer:
            return self.set_off_mode()
        elif self.temperature() > temp_setter and self.issummer:
            return self.set_on_mode()

    def get_actual_temp(self):
        #temp_sensor = temp_sensor.get_(self.pin_sensor)
        self.humidity, self.temperature = temp_sensor.get_humidity_temp()
        self.temperature=5
        return (200, "La temperatura actual es de %s grados" %self.temperature)

    def set_on_mode(self):
        if GPIO.input(self.sensor_com):
            return (300, "La calefaccion ya esta encendida"  )
        else:
            GPIO.output(self.sensor_com, True)  ## enciendo el pin
            return (200, "encendiendo la caldera a %s grados en modo %s" % (self.temp_target, self.get_mode()))

    def set_off_mode(self):
        if not GPIO.input(self.sensor_com):
            return (300, "La calefaccion ya esta apagada"  )
        else:
            return (200, "Apagando la caldera a %s grados en modo %s" % (self.temp_target, self.get_mode()))
            GPIO.output(self.sensor_com, False)  ## enciendo el pin

    def get_mode(self):
        if self.issummer:
            return "verano"
        if self.iswinter:
            return "invierno"


import threading
import time
from datetime import datetime
from itertools import cycle

import RPi.GPIO as GPIO

import display
import events
import ip
import messages
import threads
import update
from constants import *

def display_clock():
    now = datetime.now()
    device.write_text(1, " {:%H%M%S}".format(now.time()), dots=(3, 5))
    events.change_mode.wait(DISPLAY_RATE_CLOCK)

def display_date():
    now = datetime.now()
    device.write_text(1, "{:%d-%m-%y}".format(now.date()))
    events.change_mode.wait(DISPLAY_RATE_DATE)

def display_weather():
    device.write_text(1, "{:>6d}*C".format(update.temperature))
    events.change_mode.wait(DISPLAY_RATE_WEATHER)

def display_currency():
    device.write_text(1, " {:d} EUR".format(update.eur), dots=[6])
    events.change_mode.wait(DISPLAY_RATE_CURRENCY)
    if not events.change_mode.is_set():
        device.write_text(1, " {:d} USD".format(update.usd), dots=[6])
        events.change_mode.wait(DISPLAY_RATE_CURRENCY)

def display_instagram():
    device.write_text(1, "IG{:>6d}".format(update.followers))
    events.change_mode.wait(DISPLAY_RATE_IG)

def hide_display():
    events.change_mode.wait(5)

def wait_for_message_display():
    time.sleep(0.2)

def button_listener():
    global current_mode
    while True:
        if GPIO.event_detected(BUTTON_1):
            if messages.messages_to_read:
                message = messages.messages_to_read.popleft()
                threads.start_time_dependent()
                show_message(message)
                if messages.messages_to_read:
                    threads.start_flow()
                messages.send_read_id(message['id'])
            else:
                show_message()
            time.sleep(WAIT_TIME_AFTER_CLICK)
        elif GPIO.event_detected(BUTTON_2):
            current_mode = next(mode)
            events.change_mode.set()
            device.clear()
            time.sleep(WAIT_TIME_AFTER_CLICK)
        else:
            time.sleep(0.2)

def show_message(message=None):
    events.reading_message.set()
    device.clear()
    global current_mode
    remember_mode, current_mode = current_mode, 0
    if message:
        device.show_message(text=message['text'].upper(), mw=True)
    else:
        device.write_text(1, "NO MSGS", mw=True)
        time.sleep(2)
    current_mode = remember_mode
    events.reading_message.clear()

def setup_button_listener():
    threading.Thread(target=button_listener, daemon=True).start()

def init():
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(BUTTON_1, GPIO.IN)
    GPIO.setup(BUTTON_2, GPIO.IN)
    GPIO.add_event_detect(BUTTON_1, GPIO.RISING, bouncetime=250)
    GPIO.add_event_detect(BUTTON_2, GPIO.RISING, bouncetime=250)

    global device
    device = display.device

    threads.thread_dependent_on_time.start()

    device.show_message(text=ip.ip)
    device.write_text(1, "LOADING", dots=[1])

    setup_button_listener()
    update.all_modes()
    messages.setup_messages_service()
    ip.send_ip()
    device.clear()

modes = {
    0: wait_for_message_display,
    1: display_clock,
    2: display_date,
    3: display_weather,
    4: display_currency,
    5: display_instagram,
    6: hide_display
}
current_mode = 1
mode = cycle(range(1, 7))
next(mode)

if __name__ == '__main__':
    init()
    while True:
        while not events.change_mode.is_set():
            modes[current_mode]()
        events.change_mode.clear()


import os
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from natsort import natsorted

first = True
# AdamW_10-40sec_Mel_Spectrogram / AdamW_10-40sec_MFCC/ RAdam_10-40sec_Mel_Spectrogram / RAdam_10-40sec_MFCC/history
PATH = 'AdamW_10-40sec_Mel_Spectrogram'
history_list = natsorted(os.listdir(PATH))
print(history_list)
file_name = history_list[0][:-6]

for idx, name in enumerate(history_list):
    df = pd.read_csv(os.path.join(PATH, name))

    globals()[f'loss_{idx+1}'] = df['loss'].to_list()
    globals()[f'accuracy_{idx+1}'] = df['accuracy'].to_list()
    globals()[f'val_loss_{idx+1}'] = df['val_loss'].to_list()
    globals()[f'val_accuracy_{idx+1}'] = df['val_accuracy'].to_list()

total_loss = list(zip(loss_1, loss_2, loss_3, loss_4, loss_5))
total_acc = list(zip(accuracy_1, accuracy_2, accuracy_3, accuracy_4, accuracy_5))
total_val_loss = list(zip(val_loss_1, val_loss_2, val_loss_3, val_loss_4, val_loss_5))
total_val_acc = list(zip(val_accuracy_1, val_accuracy_2, val_accuracy_3, val_accuracy_4, val_accuracy_5))

loss_lower_bound = [min(x) for x in total_loss]
loss_upper_bound = [max(x) for x in total_loss]
acc_lower_bound = [min(x) for x in total_acc]
acc_upper_bound = [max(x) for x in total_acc]

val_loss_lower_bound = [min(x) for x in total_val_loss]
val_loss_upper_bound = [max(x) for x in total_val_loss]
val_acc_lower_bound = [min(x) for x in total_val_acc]
val_acc_upper_bound = [max(x) for x in total_val_acc]

loss_avg = [sum(x)/len(x) for x in total_loss]
acc_avg = [sum(x)/len(x) for x in total_acc]
val_loss_avg = [sum(x)/len(x) for x in total_val_loss]
val_acc_avg = [sum(x)/len(x) for x in total_val_acc]

index = range(len(acc_avg))
df = pd.DataFrame(data=list(zip(loss_avg, acc_avg, val_loss_avg, val_acc_avg)),
                  columns=['loss', 'accuracy', 'val_loss', 'val_accuracy'], index=index)
df.to_csv(os.path.join(PATH, file_name+'_avg.csv'), mode='w')

x_axis = list(index)
plt.plot(x_axis, acc_avg)
plt.plot(x_axis, val_acc_avg)
plt.fill_between(x_axis, acc_lower_bound, acc_upper_bound, alpha=0.2)
plt.fill_between(x_axis, val_acc_lower_bound, val_acc_upper_bound, alpha=0.2)
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='best')
plt.grid()
plt.savefig(os.path.join(PATH, file_name+'_avg.png'), dpi=300)
plt.show()

# ============================ Trying to plot LOSS for checking convergence ===================
x_axis = list(index)
plt.plot(x_axis, loss_avg)
plt.plot(x_axis, val_loss_avg)
plt.fill_between(x_axis, loss_lower_bound, loss_upper_bound, alpha=0.2)
plt.fill_between(x_axis, val_loss_lower_bound, val_loss_upper_bound, alpha=0.2)
plt.title('Model accuracy')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='best')
plt.grid()
plt.savefig(os.path.join(PATH, file_name+'_loss.png'), dpi=300)
plt.show()


import sys
import time

from PyQt5 import QtCore, QtWidgets
from PyQt5.QtGui import QColor

# https://evileg.com/ru/post/579/
# + something renamed...

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(453, 408)
        self.verticalLayout = QtWidgets.QVBoxLayout(MainWindow)
        self.verticalLayout.setObjectName("verticalLayout")
        self.verticalLayout_2 = QtWidgets.QVBoxLayout()
        self.verticalLayout_2.setObjectName("verticalLayout_2")
        self.textBrowser = QtWidgets.QTextBrowser(MainWindow)
        self.textBrowser.setObjectName("textBrowser")
        self.verticalLayout_2.addWidget(self.textBrowser)
        self.verticalLayout.addLayout(self.verticalLayout_2)
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")
        spacerItem = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout.addItem(spacerItem)
        self.pushButton = QtWidgets.QPushButton(MainWindow)
        self.pushButton.setObjectName("pushButton")
        self.horizontalLayout.addWidget(self.pushButton)
        spacerItem1 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout.addItem(spacerItem1)
        self.verticalLayout.addLayout(self.horizontalLayout)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "--- Пример ---"))
        self.pushButton.setText(_translate("MainWindow", "-- ВВОД --"))

# Объект, который будет перенесён в другой поток для выполнения кода
class BrowserHandler(QtCore.QObject):
    # running = False
    newTextAndColor = QtCore.pyqtSignal(str, object)

    # метод, который будет выполнять алгоритм в другом потоке
    def run(self):
        while True:
            # посылаем сигнал из второго потока в GUI поток
            self.newTextAndColor.emit(f'{time.strftime("%Y-%m-%d %H:%M:%S")} - thread 2 var 1.\n', QColor(0, 0, 150))
            QtCore.QThread.msleep(1000)

            # посылаем сигнал из второго потока в GUI поток
            self.newTextAndColor.emit(f'{time.strftime("%Y-%m-%d %H:%M:%S")} - thread 2 var 2.\n', QColor(150, 0, 0))
            QtCore.QThread.msleep(1000)

#class MyWindow():
class MyWindow(QtWidgets.QWidget):
#class MyWindow(QtWidgets.QMainWindow):

    def __init__(self, parent=None):
        super().__init__()
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        # используем кнопку для добавления текста с цветом в главном потоке
        self.ui.pushButton.clicked.connect(self.addAnotherTextAndColor)

        # создадим поток
        self.thread = QtCore.QThread()
        # создадим объект для выполнения кода в другом потоке
        self.browserHandler = BrowserHandler()
        # перенесём объект в другой поток
        self.browserHandler.moveToThread(self.thread)
        # после чего подключим все сигналы и слоты
        self.browserHandler.newTextAndColor.connect(self.addNewTextAndColor)
        # подключим сигнал старта потока к методу run у объекта, который должен выполнять код в другом потоке
        self.thread.started.connect(self.browserHandler.run)
        # запустим поток
        self.thread.start()

    @QtCore.pyqtSlot(str, object)
    def addNewTextAndColor(self, string, color):
        self.ui.textBrowser.setTextColor(color)
        self.ui.textBrowser.append(string)

    def addAnotherTextAndColor(self):
        self.ui.textBrowser.setTextColor(QColor(0, 200, 0))
        self.ui.textBrowser.append(f'Нажата кл.{time.strftime("%Y-%m-%d %H:%M:%S")} - thread 2 var 3.\n')

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    window = MyWindow()
    window.show()
    sys.exit(app.exec())


from datetime import timedelta
import requests
from provider.utils import on_workdays

URL = "https://www.kajahu.com/etlap/d"
API = "https://appif.kajahu.com/jdmenu?jlang=hu&jseat=-"

@on_workdays
def getMenu(today):
    date = today.strftime("%Y-%m-%d")
    menujson = requests.get(API).json()
    menu = []
    for day in menujson['jdata']:
        if day['ddate'] == date:
            menu = [day['line1'], day['line2'], day['line3']]
            break

    return menu

menu = {
    'name': 'KAJAHU',
    'id': 'kjh',
    'url': URL,
    'get': getMenu,
    'ttl': timedelta(hours=23),
    'cards': []
}


import html
import re
import sys
from functools import lru_cache
from pathlib import Path
from typing import Dict

import pymorphy2

PATH = Path("../..")
sys.path.append(str(PATH))

cache: Dict = {}
morph = pymorphy2.MorphAnalyzer()

# read stopwords for RU
try:
    with open(PATH / "data/features/stopwords_ru.txt", "r") as file:
        stopwords = file.read().splitlines()
except FileNotFoundError:
    print("can't load /data/features/stopwords_ru.txt")
    stopwords = []

def clean_text(text: str = None) -> str:
    """
    clean text, leaving only tokens for clustering
    parameters
    ----------
        text : string
            input text
    returns
    -------
        cleaned string text without lower case
    """

    if not isinstance(text, str):
        text = str(text)

    text = html.unescape(text)

    text = re.sub(r"https?:\/\/.*[\r\n]*", "", text)  # remove urls
    text = re.sub(r"\S+@\S+", "", text)  # remove emails
    text = re.sub(r"ё", "е", text)
    text = re.sub(r"\!|\"|\:|\;|\.|\,|[<>]|\?|\@|\[|\]|\^|\_|\`|[{}]|\~|[—–-]|[«»]|[()]", " ", text)  # remove punctuation
    text = re.sub(r"\s+", " ", text)  # remove the long blanks

    text = text.strip()

    if len(text) < 3:
        return "TOREMOVE"
    else:
        return text

@lru_cache()
def get_morph4token(token: str = None) -> str:
    """
    get lemma for one tokens with decorator `@lru_cache`
    """

    return morph.parse(token)[0].normal_form

def lemmatize(text: str = None, char4split: str = " ") -> str:
    """
    lemmatizations text with cache
    parameters
    ----------
    input_text : string
        cleaned text
    char4split : string (default = " ")
        char-symbol how to split text
    returns
    -------
        lemmatized text
    """

    # get tokens from input text
    # in this case it's normal approach because we hard cleaned text
    if not isinstance(text, str):
        text = str(text)

    # get tokens from input text
    # in this case it's normal approach because we hard cleaned text
    list_tokens = text.split(char4split)

    words_lem = [
        get_morph4token(token) for token in list_tokens
        if token not in stopwords
    ]

    if len(words_lem) < 3:
        return "TOREMOVE"
    else:
        return " ".join(words_lem)


#File used to communicate with esp8266 to gather json location data
# REV 1.1

import time
import serial
import json
import os
from pix2coord import *
from Mqtt import *
ser=None
sully = 0
##class definition of objects on field
##------------------------------------
ser = serial.Serial('/dev/serial0',115200) ##declare ser object for serial comms
def updatejson(ser):
            # Serial begin with esp8266--------------------------------------------------
            if ser.inWaiting() > 0:
                rawdata = ser.readline()
                rawdata = rawdata.decode('ASCII')
                data = json.loads(rawdata)

##                parse data and save into objects
                # ------------------------------------------------------------
                cor_br = corners(data["Corners"][2]["X"],data["Corners"][2]["Y"])
                cor_tr = corners(data["Corners"][3]["X"],data["Corners"][3]["Y"])
                cor_bl = corners(data["Corners"][0]["X"],data["Corners"][0]["Y"])
                cor_tl = corners(data["Corners"][1]["X"],data["Corners"][1]["Y"])
                cval = maxmin_xy(cor_tl.x, cor_tr.x, cor_bl.x, cor_br.x, cor_tl.y, cor_tr.y, cor_bl.y, cor_br.y) #find the max and min values of the corners

                redcircle = objpos(data["Red Team Data"]["Circle"]["Object Center"]["X"],data["Red Team Data"]["Circle"]["Object Center"]["Y"],cval.xmin,cval.xmax,cval.ymin,cval.ymax)
                redsquare = objpos(data["Red Team Data"]["Square"]["Object Center"]["X"],data["Red Team Data"]["Square"]["Object Center"]["Y"],cval.xmin,cval.xmax,cval.ymin,cval.ymax)
                redtriangle = objpos(data["Red Team Data"]["Triangle"]["Object Center"]["X"],data["Red Team Data"]["Triangle"]["Object Center"]["Y"],cval.xmin,cval.xmax,cval.ymin,cval.ymax)

                bluecircle = objpos(data["Blue Team Data"]["Circle"]["Object Center"]["X"],data["Blue Team Data"]["Circle"]["Object Center"]["Y"],cval.xmin,cval.xmax,cval.ymin,cval.ymax)
                bluesquare = objpos(data["Blue Team Data"]["Square"]["Object Center"]["X"],data["Blue Team Data"]["Square"]["Object Center"]["Y"],cval.xmin,cval.xmax,cval.ymin,cval.ymax)
                bluetriangle = objpos(data["Blue Team Data"]["Triangle"]["Object Center"]["X"],data["Blue Team Data"]["Triangle"]["Object Center"]["Y"],cval.xmin,cval.xmax,cval.ymin,cval.ymax)

                ball = objpos(data["Ball"]["Object Center"]["X"],data["Ball"]["Object Center"]["Y"],cval.xmin,cval.xmax,cval.ymin,cval.ymax)
                print("ball")
                ball.printobj()
                print("red circle")
                redcircle.printobj()
                # if ((abs(ball.x-redcircle.x))<15 and abs((ball.y-redcircle.y)<15)):
                #     sully = 1
                #     pubspeeds("Rover1",0,0,0,1)
                # else:
                #     sully = 0
                # print(sully)
                return
while True:
    updatejson(ser)
#

##        except KeyboardInterrupt:
##            print ("Exiting Program")
##        ##except:
##        ##    print ("Error Occurs, Exiting Program")
##
##        finally:
##            ser.close()
##            pass


#
# @author:charlotte.Song
# @file: lsvt_eval.py
# @Date: 2019/3/4 17:12
# @description:
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import argparse
import torch
import os
import os.path as osp
import json
from collections import OrderedDict
import Polygon as plg
import cv2

""" load submit.json and compare with eval annotations.
    using scripts like ctw_script.py
    in submit.json the key name is 'res_{image_id}'
    and in eval_ann_file(xxx.json) the key name is 'res_{image_id}'

    load the eval settings from json file. 'eval_setting.json'
"""

eval_ann_file = '../data/LSVT/annotations/sp_val_eval_labels.json'
submit_path = '../submit/lsvt/'
art_eval_ann_file = '../data/ArT/annotations/sp_val_eval_labels.json'
art_submit_path = '../submit/art/'
art_val_data_root = '/home/data1/IC19/ArT/sp_val_art_images/'

debug_path = '../visualization/debug/'
stastic_path = '../visualization/stastics/'

def parse_args():
    parser = argparse.ArgumentParser(description='art postmodule')
    parser.add_argument('--submit_file', type=str, default=None)
    parser.add_argument('--gt_file', type=str, default=None)
    parser.add_argument('--nms_thr', type=float, default=0.5)
    parser.add_argument('--con_thr', type=float, default=0.0)
    parser.add_argument('--eval_f', type=str, default='ctw',
                        help='ctw or xez')
    parser.add_argument('--stastics', action='store_true')
    parser.add_argument('--search', action='store_true')        # search best h1 score
    # for search mode
    parser.add_argument('--low_score', default=0.5, type=float)
    parser.add_argument('--high_score', default=0.95, type=float)
    parser.add_argument('--step', type=float, default=0.05)
    args = parser.parse_args()
    return args

def get_union(pa, pb):
    pa_area = pa.area()
    pb_area = pb.area()
    return pa_area + pb_area - get_intersection(pa, pb)

def get_intersection(pa, pb):
    pInt = pa & pb
    if len(pInt) == 0:
        return 0
    else:
        return pInt.area()

# get scales of polygon
def get_rescale_factor(name, long_size=2560, short_size=800, keep_ratio=True):
    name = osp.join(art_val_data_root, name.replace('res_', 'gt_') + '.jpg')
    img = cv2.imread(name)
    h, w = img.shape[:2]
    if keep_ratio:
        scale_factor = min(
            long_size * 1.0 / max(h, w),
            short_size * 1.0 / min(h, w),
        )
    else:
        scale_factor = [
            long_size * 1.0 / max(h, w),
            short_size * 1.0 / min(h, w)
        ]
        scale_factor = [scale_factor[0], scale_factor[1]] if h >= w else [scale_factor[1], scale_factor[0]]
        scale_factor = np.array(scale_factor).astype(np.float32)
    return scale_factor

def evaluation(submit_file, eval_ann, threshold=0.5, confidence=0.0,
               debug_mode=False, stastics_mode=False):
    assert osp.isfile(submit_file)
    assert osp.isfile(eval_ann)
    with open(eval_ann, 'r', encoding='utf-8') as f:
        gt_annotations = json.loads(f.read(), object_pairs_hook=OrderedDict)
    with open(submit_file, 'r', encoding='utf-8') as f:
        preds = json.loads(f.read(), object_pairs_hook=OrderedDict)

    tp, fp, npos = 0, 0, 0
    lost_gt_polygon = []
    """ for validation the annotation is same as the trainset. """
    for name, pred_ann in preds.items():
        gt_annotation = gt_annotations[name]    # is a list
        # npos should filter the ignored box.
        for gt_polygon_tran in gt_annotation:
            if gt_polygon_tran["illegibility"]:
                continue
            npos += 1

        cover = set()
        # compare each pred_polygon with each gt_polygon
        for pred_polygon_pro in pred_ann:
            pred_polygon = np.array(pred_polygon_pro["points"])   # shape: [n, 2]
            pred_polygon = plg.Polygon(pred_polygon)
            # set the score threshold
            if pred_polygon_pro["confidence"] < confidence:
                continue
            flag = False
            is_ignore = False

            for gt_id, gt_polygon_tran in enumerate(gt_annotation):
                gt_illegibility = gt_polygon_tran["illegibility"]
                gt_polygon = np.array(gt_polygon_tran["points"]).reshape(-1, 2).astype(np.int64)
                gt_polygon = plg.Polygon(gt_polygon)

                union = get_union(pred_polygon, gt_polygon)
                intersection = get_intersection(pred_polygon, gt_polygon)
                if (flag is False) and (intersection * 1.0 / union) >= threshold:
                    if gt_id not in cover:
                        flag = True
                        cover.add(gt_id)
                        if gt_illegibility:
                            is_ignore = True

            if flag:
                tp += 0.0 if is_ignore else 1.0
            else:
                fp += 1.0

        if stastics_mode:
            """ calculate all lost gt bboxes """
            scale_factor = get_rescale_factor(name)
            for gt_id, gt_polygon_tran in enumerate(gt_annotation):
                gt_polygon = np.array(gt_polygon_tran["points"]).reshape(-1, 2).astype(np.float32)
                # needed to
                gt_polygon = (gt_polygon * scale_factor).astype(np.int64)
                # gt_polygon_area = plg.Polygon(gt_polygon).area()
                ltop = np.min(gt_polygon, axis=0)
                rbottom = np.max(gt_polygon, axis=0)
                gt_polygon_area = (rbottom[0] - ltop[0]) * (rbottom[1] - ltop[1])
                if (gt_id not in cover) and (gt_polygon_tran["illegibility"] is False):
                    lost_gt_polygon.append(gt_polygon_area)

        if debug_mode:
            precision = tp / (fp + tp)
            recall = tp / npos
            hmean = 0 if (precision + recall) == 0 else 2 * precision * recall / (precision + recall)
            print("p: {:f}  r: {:f}  h: {:f}".format(precision, recall, hmean))
            return name

    precision = tp / (fp + tp)
    recall = tp / npos      # recall do not calculate the ignored ones.
    hmean = 0 if (precision + recall) == 0 else 2 * precision * recall / (precision + recall)

    print("p: {:f}  r: {:f}  h: {:f}".format(precision, recall, hmean))

    if stastics_mode:
        """ print """
        import matplotlib as mpl
        mpl.use('Agg')
        import matplotlib.pyplot as plt
        if not os.path.exists(stastic_path):
            os.makedirs(stastic_path)
        if len(lost_gt_polygon) > 0:
            lost_gt_polygon = np.asarray(lost_gt_polygon)
            plt.hist(lost_gt_polygon[lost_gt_polygon < 10000], bins='auto', histtype='bar')
            plt.title('lost gt bbox area')
            plt.xlabel('area')
            plt.ylabel('frequency')
            plt.savefig(os.path.join(stastic_path, 'stastics_plot.jpg'))
            print('totalnum: {:d}; min: {:f}; max: {:f}'.format(len(lost_gt_polygon), min(lost_gt_polygon), max(lost_gt_polygon)))
            print('write down {}'.format(os.path.join(stastic_path, 'stastics_plot.jp')))

    return hmean

def sigma_calculation(det_p, gt_p):
    """
    sigma = inter_area / gt_area
    """
    return get_intersection(det_p, gt_p) / gt_p.area()

def tau_calculation(det_p, gt_p):
    """ tau = inter_area / det_area """
    return get_intersection(det_p, gt_p) / det_p.area()

# detection_filtering
# filter out the pred bboxes which has a IoU > thr with ignored gt bboxes.
def detection_filtering(pred_ann, gt_ann, thr=0.5):
    filtered_pred_anns = []
    filtered_gt_anns = []
    filtered = []
    for gt_indx in range(len(gt_ann)):
        if not gt_ann[gt_indx]["illegibility"]:
            filtered_gt_anns.append(gt_ann[gt_indx])
            continue
        gt_points = np.array(gt_ann[gt_indx]["points"]).reshape(-1, 2).astype(np.int64)
        gt_polygon = plg.Polygon(gt_points)

        for pred_indx in range(len(pred_ann)):
            pred_points = np.array(pred_ann[pred_indx]["points"]).reshape(-1, 2).astype(np.int64)
            pred_polygon = plg.Polygon(pred_points)

            pred_gt_iou = get_intersection(pred_polygon, gt_polygon) / (pred_polygon.area() + 1e-5)
            if pred_gt_iou >= thr:
                # keep the boxes that have small iou with ignored gt box
                # filtered_pred_anns.append(pred_ann[pred_indx])
                filtered.append(pred_indx)
    for pred_indx in range(len(pred_ann)):
        if pred_indx not in filtered:
            filtered_pred_anns.append(pred_ann[pred_indx])
    return filtered_pred_anns, filtered_gt_anns

def one2one(local_sigma_table, local_tau_table, local_accumulative_recall,
            local_accumulative_precision, global_accumulative_recall, global_accumulative_precision,
            gt_flag, pred_flag, num_gt, tr, tp):
    """
    :param local_sigma_table: local sigma iou positive
    :param local_tau_table: local tau iou positive
    :param local_accumulative_recall: local recall
    :param local_accumulative_precision: local prec
    :param global_accumulative_recall: global recall
    :param global_accumulative_precision: global prec
    :param gt_flag:
    :param det_flag:
    :return:
    """
    for gt_id in range(num_gt):
        # sigma and tau are different iou results.
        # select the ones that iou > thr.
        qualified_sigma_candidiates = np.where(local_sigma_table[gt_id, :] > tr)
        num_qualified_sigma_candidates = qualified_sigma_candidiates[0].shape[0]
        qualified_tau_candidates = np.where(local_tau_table[gt_id, :] > tp)
        num_qualified_tau_candidates = qualified_tau_candidates[0].shape[0]

        if (num_qualified_sigma_candidates == 1) and (num_qualified_tau_candidates == 1):
            global_accumulative_recall = global_accumulative_recall + 1.0
            global_accumulative_precision = global_accumulative_precision + 1.0
            local_accumulative_recall = local_accumulative_recall + 1.0
            local_accumulative_precision = local_accumulative_precision + 1.0

            gt_flag[0, gt_id] = 1       # one2one
            matched_pred_id = np.where(local_sigma_table[gt_id, :] > tr)
            pred_flag[0, matched_pred_id] = 1       # one2one
    return local_accumulative_recall, local_accumulative_precision, \
           global_accumulative_recall, global_accumulative_precision, \
           gt_flag, pred_flag

def one2many(local_sigma_table, local_tau_table, local_accumulative_recall,
            local_accumulative_precision, global_accumulative_recall, global_accumulative_precision,
            gt_flag, pred_flag, num_gt, tr, tp, k, fsc_k):
    for gt_id in range(num_gt):
        if gt_flag[0, gt_id] > 0:   # one2one.
            continue

        non_zero_in_sigma = np.where(local_sigma_table[gt_id, :] > 0)
        num_non_zero_in_sigma = non_zero_in_sigma[0].shape[0]

        if num_non_zero_in_sigma >= k:
            ####search for all detections that overlaps with this groundtruth
            qualified_tau_candidates = np.where((local_tau_table[gt_id, :] >= tp) & (pred_flag[0, :] == 0))
            num_qualified_tau_candidates = qualified_tau_candidates[0].shape[0]

            if num_qualified_tau_candidates == 1:
                if ((local_tau_table[gt_id, qualified_tau_candidates] >= tp) and (
                        local_sigma_table[gt_id, qualified_tau_candidates] >= tr)):
                    # became an one-to-one case
                    global_accumulative_recall = global_accumulative_recall + 1.0
                    global_accumulative_precision = global_accumulative_precision + 1.0
                    local_accumulative_recall = local_accumulative_recall + 1.0
                    local_accumulative_precision = local_accumulative_precision + 1.0

                    gt_flag[0, gt_id] = 1   # a good pair.
                    pred_flag[0, qualified_tau_candidates] = 1
            elif (np.sum(local_sigma_table[gt_id, qualified_tau_candidates]) >= tr):
                gt_flag[0, gt_id] = 1
                pred_flag[0, qualified_tau_candidates] = 1

                global_accumulative_recall = global_accumulative_recall + fsc_k
                global_accumulative_precision = global_accumulative_precision + num_qualified_tau_candidates * fsc_k

                local_accumulative_recall = local_accumulative_recall + fsc_k
                local_accumulative_precision = local_accumulative_precision + num_qualified_tau_candidates * fsc_k

    return local_accumulative_recall, local_accumulative_precision, \
           global_accumulative_recall, global_accumulative_precision, \
           gt_flag, pred_flag

def many2many(local_sigma_table, local_tau_table, local_accumulative_recall,
            local_accumulative_precision, global_accumulative_recall, global_accumulative_precision,
            gt_flag, pred_flag, num_pred, tr, tp, k, fsc_k):

    for pred_id in range(num_pred):
        if pred_flag[0, pred_id] > 0:
            continue

        non_zero_in_tau = np.where(local_tau_table[:, pred_id] > 0)
        num_non_zero_in_tau = non_zero_in_tau[0].shape[0]

        if num_non_zero_in_tau >= k:
            ####search for all detections that overlaps with this groundtruth
            qualified_sigma_candidates = np.where((local_sigma_table[:, pred_id] >= tp) & (gt_flag[0, :] == 0))
            num_qualified_sigma_candidates = qualified_sigma_candidates[0].shape[0]

            if num_qualified_sigma_candidates == 1:
                if ((local_tau_table[qualified_sigma_candidates, pred_id] >= tp) and (
                        local_sigma_table[qualified_sigma_candidates, pred_id] >= tr)):
                    # became an one-to-one case
                    global_accumulative_recall = global_accumulative_recall + 1.0
                    global_accumulative_precision = global_accumulative_precision + 1.0
                    local_accumulative_recall = local_accumulative_recall + 1.0
                    local_accumulative_precision = local_accumulative_precision + 1.0

                    gt_flag[0, qualified_sigma_candidates] = 1
                    pred_flag[0, pred_id] = 1
            elif (np.sum(local_tau_table[qualified_sigma_candidates, pred_id]) >= tp):
                pred_flag[0, pred_id] = 1
                gt_flag[0, qualified_sigma_candidates] = 1

                global_accumulative_recall = global_accumulative_recall + num_qualified_sigma_candidates * fsc_k
                global_accumulative_precision = global_accumulative_precision + fsc_k

                local_accumulative_recall = local_accumulative_recall + num_qualified_sigma_candidates * fsc_k
                local_accumulative_precision = local_accumulative_precision + fsc_k
    return local_accumulative_recall, local_accumulative_precision, \
           global_accumulative_recall, global_accumulative_precision, \
           gt_flag, pred_flag

def det_evaluation(submit_file, eval_ann, iou_thr=0.5, debug_mode=False, debug_name=None):
    """ settings """
    global_sigma, global_tau = [], []
    tr, tp, fsc_k, k = 0.7, 0.6, 0.8, 2
    global_accumulative_recall = 0
    global_accumulative_precision = 0
    total_num_gt = 0
    total_num_pred = 0

    """ read pred ann and gt ann """
    assert osp.isfile(submit_file)
    assert osp.isfile(eval_ann)
    with open(eval_ann, 'r', encoding='utf-8') as f:
        gt_annotations = json.loads(f.read(), object_pairs_hook=OrderedDict)
    with open(submit_file, 'r', encoding='utf-8') as f:
        pred_annotations = json.loads(f.read(), object_pairs_hook=OrderedDict)

    """ eval """
    if debug_mode:
        pred_ann = pred_annotations[debug_name]
        # first get the corresponding gt ann
        gt_ann = gt_annotations[debug_name]
        # filtered_pred_ann, and filtered_gt_ann has filtered out all
        # bboxes that overlap with ignored bboxes.
        filtered_pred_ann, filtered_gt_ann = detection_filtering(pred_ann, gt_ann)
        local_sigma_table = np.zeros((len(filtered_gt_ann), len(filtered_pred_ann)))
        local_tau_table = np.zeros((len(filtered_gt_ann), len(filtered_pred_ann)))

        """ compare each pred box with each gt box """
        for gt_idx in range(len(filtered_gt_ann)):
            if len(filtered_pred_ann) > 0:
                gt_points = np.array(filtered_gt_ann[gt_idx]["points"]).reshape(-1, 2).astype(np.int64)
                gt_polygon = plg.Polygon(gt_points)
                for pred_idx in range(len(filtered_pred_ann)):
                    pred_points = np.array(filtered_pred_ann[pred_idx]["points"]).reshape(-1, 2).astype(np.int64)
                    pred_polygon = plg.Polygon(pred_points)
                    # store each img's information of gt_idx and pred_idx.
                    local_sigma_table[gt_idx, pred_idx] = sigma_calculation(pred_polygon, gt_polygon)
                    local_tau_table[gt_idx, pred_idx] = tau_calculation(pred_polygon, gt_polygon)
        global_sigma.append(local_sigma_table)
        global_tau.append(local_tau_table)
    else:
        for name, pred_ann in pred_annotations.items():
            # first get the corresponding gt ann
            gt_ann = gt_annotations[name]
            # filtered_pred_ann, and filtered_gt_ann has filtered out all
            # bboxes that overlap with ignored bboxes.
            filtered_pred_ann, filtered_gt_ann = detection_filtering(pred_ann, gt_ann)
            local_sigma_table = np.zeros((len(filtered_gt_ann), len(filtered_pred_ann)))
            local_tau_table = np.zeros((len(filtered_gt_ann), len(filtered_pred_ann)))

            """ compare each pred box with each gt box """
            for gt_idx in range(len(filtered_gt_ann)):
                if len(filtered_pred_ann) > 0:
                    gt_points = np.array(filtered_gt_ann[gt_idx]["points"]).reshape(-1, 2).astype(np.int64)
                    gt_polygon = plg.Polygon(gt_points)
                    for pred_idx in range(len(filtered_pred_ann)):
                        pred_points = np.array(filtered_pred_ann[pred_idx]["points"]).reshape(-1, 2).astype(np.int64)
                        pred_polygon = plg.Polygon(pred_points)
                        # store each img's information of gt_idx and pred_idx.
                        local_sigma_table[gt_idx, pred_idx] = sigma_calculation(pred_polygon, gt_polygon)
                        local_tau_table[gt_idx, pred_idx] = tau_calculation(pred_polygon, gt_polygon)
            global_sigma.append(local_sigma_table)
            global_tau.append(local_tau_table)

    """ calculate recall and prec """
    for idx in range(len(global_sigma)):
        local_sigma_table = global_sigma[idx]
        local_tau_table = global_tau[idx]

        num_gt = local_sigma_table.shape[0]
        num_pred = local_sigma_table.shape[1]

        total_num_gt = total_num_gt + num_gt
        total_num_pred = total_num_pred + num_pred

        local_accumulative_recall = 0
        local_accumulative_precision = 0
        gt_flag = np.zeros((1, num_gt))
        pred_flag = np.zeros((1, num_pred))

        #######first check for one-to-one case##########
        """ see what one2one do """
        local_accumulative_recall, local_accumulative_precision, \
        global_accumulative_recall, global_accumulative_precision, \
        gt_flag, pred_flag = one2one(local_sigma_table, local_tau_table, local_accumulative_recall,
                                     local_accumulative_precision, global_accumulative_recall, global_accumulative_precision,
                                     gt_flag, pred_flag, num_gt, tr, tp)
        ######then check for one-to-many case##########
        local_accumulative_recall, local_accumulative_precision, \
        global_accumulative_recall, global_accumulative_precision, \
        gt_flag, pred_flag = one2many(local_sigma_table, local_tau_table, local_accumulative_recall,
                                      local_accumulative_precision, global_accumulative_recall, global_accumulative_precision,
                                      gt_flag, pred_flag, num_gt, tr, tp, k, fsc_k)
        #######then check for many-to-many case##########
        local_accumulative_recall, local_accumulative_precision, global_accumulative_recall, global_accumulative_precision, \
        gt_flag, det_flag = many2many(local_sigma_table, local_tau_table,
                                         local_accumulative_recall, local_accumulative_precision,
                                         global_accumulative_recall, global_accumulative_precision,
                                         gt_flag, pred_flag, num_pred, tr, tp, k, fsc_k)

        try:
            local_precision = local_accumulative_precision / num_pred
        except ZeroDivisionError:
            local_precision = 0

        try:
            local_recall = local_accumulative_recall / num_gt
        except ZeroDivisionError:
            local_recall = 0

        # temp = '%s______/Precision:_%s_______/Recall:_%s\n' % (idx, str(local_precision), str(local_recall))
        # print("local:" + temp)
    try:
        recall = global_accumulative_recall / total_num_gt
    except ZeroDivisionError:
        recall = 0

    try:
        precision = global_accumulative_precision / total_num_pred
    except ZeroDivisionError:
        precision = 0

    try:
        f_score = 2 * precision * recall / (precision + recall)
    except ZeroDivisionError:
        f_score = 0

    hmean = 2 * precision * recall / (precision + recall)
    temp = ('Precision:_%s_______/Recall:_%s/Hmean:_%s\n' % (str(precision), str(recall), str(hmean)))
    print(temp)

if __name__ == '__main__':
    args = parse_args()
    if args.eval_f == 'ctw':
        if not args.search:
            hmean = evaluation(submit_file=args.submit_file, eval_ann=args.gt_file,
                               threshold=args.nms_thr, confidence=args.con_thr,
                               stastics_mode=args.stastics)
        else:
            best = 0
            best_conf = args.low_score
            for conf in np.arange(args.low_score, args.high_score, args.step):
                hmean = evaluation(submit_file=args.submit_file, eval_ann=args.gt_file,
                                   threshold=args.nms_thr, confidence=conf,
                                   stastics_mode=args.stastics)
                if hmean > best:
                    best = hmean
                    best_conf = conf
            print('conf {:.4f}: best-hmean:{:.6f}'.format(best_conf, best))

    elif args.eval_f == 'debug':
        name = evaluation(submit_file=args.submit_file, eval_ann=args.gt_file,
                          threshold=args.nms_thr, confidence=args.con_thr, debug_mode=True)
        print(name)
        det_evaluation(submit_file=args.submit_file, eval_ann=args.gt_file, iou_thr=args.nms_thr,
                       debug_mode=True, debug_name=name)

    else:
        eval_tool = det_evaluation(submit_file=args.submit_file, eval_ann=args.gt_file,
                                   iou_thr=args.nms_thr)


import pathlib

from setuptools import setup

# The directory containing this file
HERE = pathlib.Path(__file__).parent

# The text of the README file
README = (HERE / "README.md").read_text()

# This call to setup() does all the work
setup(
    name="pyPreservicaSearch",
    version="0.1.0",
    description="Simple Report Designer for Preservica",
    long_description=README,
    long_description_content_type="text/markdown",
    url="",    author="James Carr",
    author_email="james.carr@preservica.com",
    license="Apache License 2.0",
    packages=["pyPreservicaSearch"],
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
        "Topic :: System :: Archiving",
    ],
    install_requires=["requests", "certifi", "pyPreservica >= 0.7.3", "PySide2 == 5.15.0", "shiboken2 == 5.15.0"]
)