import argparse
import codecs
import multiprocessing as mp
from pathlib import Path
from typing import *

import dask.bag as db
import pandas as pd
from dask.diagnostics import ProgressBar

parser = argparse.ArgumentParser(
    description="Converts tagged files by tagger toyger to dataframe with an id and lemmas")
parser.add_argument("--dir", help="Str path to zip file")
parser.add_argument("--out", help="Output file name with `p` extension")

args = parser.parse_args()

def parse_path(path: Path) -> Dict[str, List[str]]:
    tweet_id = path.name.split(".")[0]
    with codecs.open(path.as_posix(), encoding='utf-8') as f:
        file_content = f.read().strip().replace("'", "").replace('"', "")
    try:
        frame = pd.read_csv(pd.compat.StringIO(file_content), quotechar="'", sep="\t", header=None)
        frame = frame[frame[4] != "interp"]
        frame = frame.drop_duplicates(subset=[1])
        frame[3] = frame[3].map(lambda x: str(x).lower().split(':')[0])
        return {
            "id_str": tweet_id,
            "lemmas": frame[3].tolist()
        }
    except:
        print(file_content)
        print(tweet_id)
        return {
            "id_str": "",
            "lemmas": []
        }

def main():
    paths = list(Path(args.dir).rglob("*.txt"))
    pbar = ProgressBar()
    pbar.register()
    a_bag = db.from_sequence(paths, npartitions=mp.cpu_count())
    a_bag = a_bag.map(lambda a_path: parse_path(a_path))
    frame_data = a_bag.compute()
    pbar.unregister()

    frame = pd.DataFrame(frame_data)
    frame.to_pickle(args.out)

if __name__ == '__main__':
    main()


import torchvision
import torch
import gym
import subprocess as sp

from networks import *
from wrappers import make_env
import random

import sys

import pyglet

path = "checkpoints/%s.pt"%sys.argv[1]

q = OCNN()
#q.load_state_dict(torch.load(path,map_location=torch.device('cpu')))
q.eval()
env = gym.make('BreakoutNoFrameskip-v0')
env = make_env(env,noop_max=10) # sometimes it no-ops until the ball is unsaveable

obs = env.reset()
action = 0

kold = None

def update(dt):
    global obs, action, kold
    env.render()
    k = q(torch.cat(obs).unsqueeze(0))
    kk = k.tolist()
    if kold != kk:
        print(*('- '[i>0]+str(round(abs(i),3)).ljust(5,'0') for i in kk[0]))
        kold = kk
    if random.random() < 0.9:
        if random.random() < 0.05:
            action = env.action_space.sample()
            #print('Random',action)
        else:
            action = k.max(1)[1].item()
            #print('Chosen',action)
    obs,_,done,_ = env.step(action)
    if done:
        obs = env.reset()

pyglet.clock.schedule_interval(update, 1./60.)
try:
    pyglet.app.run()
except:
    env.close()


##try other way to compute cos-similarity
import json
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import math

with open('sample-relationshipTable.json', 'r', encoding='utf-8') as f:
    relationship = json.load(f)
f.close()

sumSim = []
for item in similarities:
    if round(item.sum(), 3) > 1:
        sumSim.append(item.sum())
plt.hist(sumSim, 100)

recordList = []
for item in similarities:
    for record in item:
        if round(record, 3) != 0 and round(record, 3) != 1:
            recordList.append(record)
plt.hist(recordList, 100)


import sys
from collections import deque
from collections import defaultdict
input = sys.stdin.readline

def bfs(start, minimum):
    visited = [False] * (N+1)
    visited[start] = True

    que = deque([start])
    cnt = 0

    while que:
        cur_node = que.popleft()

        for q, r in mootube[cur_node]:
            if not visited[q] and r >= minimum:
                visited[q] = True
                que.append(q)
                cnt += 1
    return cnt

if __name__ == "__main__":
    N, Q = list(map(int, input().split()))
    mootube = [[0] * (N+1) for _ in range(N+1)]

    mootube = defaultdict(list)
    for i in range(N-1):
        p, q, r = list(map(int, input().split()))
        mootube[p].append((q, r))
        mootube[q].append((p, r))

    for _ in range(Q):
        K, start = list(map(int, input().split()))
        print(bfs(start, K))


from pymavlink import mavutil
from pymavlink.dialects.v20 import common as mavlink
import struct
import array
import time
import os
import sys
import pickle
import numpy as np
import matplotlib.pyplot as plt
import VFControl
import utm
import copy
ignore_list = ["ATTITUDE_QUATERNION", "HIGHRES_IMU", "ATTITUDE", "GLOBAL_POSITION_INT", "LOCAL_POSITION_NED",
               "POSITION_TARGET_LOCAL_NED", "HIGHRES_IMU", "ATTITUDE_TARGET", "POSITION_TARGET_GLOBAL_INT", "VFR_HUD",
               "SYS_STATUS", "BATTERY_STATUS", "HEARTBEAT", "GPS_RAW_INT", "ALTITUDE", "WIND_COV", "EXTENDED_SYS_STATE",
               "ESTIMATOR_STATUS", "VIBRATION", "HOME_POSITION","SERVO_OUTPUT_RAW"]

target_system = 1  # 255
target_component = 0  # 0
mission_type = 0

def LoadWPFile(filename):
    fileread = open(filename, 'rb')
    wpCountMsg = pickle.load(fileread)
    wpList = []
    for i in range(0, wpCountMsg.count):
        wpList.append(pickle.load(fileread))

    fileread.close()
    return wpList

def GetWPList(mav, savefile):
    mav.mav.mission_request_list_send(target_system, target_component, mission_type)

    if (savefile is not None):
        fileout = open(savefile, 'wb')

    num_wp = -1
    print("Requesting WP List")
    while True:
        msg = mav.recv_msg()
        if msg is None:
            continue
        skip_msg = False
        for strnames in ignore_list:
            if (msg.name is strnames):
                skip_msg = True
        if (skip_msg is True):
            continue

        if msg.name is "MISSION_COUNT":
            num_wp = msg.count
            print("#WP:" + str(num_wp))
            if (num_wp == 0):
                mav.mav.mission_ack_send(target_system, target_component,
                                         type=0)  # , mission_type=0)#type=msg.seq, mission_type=0)
                break
            else:
                mav.mav.mission_request_send(target_system, target_component, seq=0)  # , mission_type=0)
            if (savefile is not None):
                pickle.dump(msg, fileout)
        elif msg.name is "MISSION_ITEM":
            print(msg)
            if (savefile is not None):
                pickle.dump(msg, fileout)

            if (msg.seq >= 0 and msg.seq + 1 < num_wp):
                print("--> REQ # " + str(msg.seq))
                mav.mav.mission_request_send(target_system, target_component, seq=msg.seq + 1)  # , mission_type=0)
            elif (msg.seq + 1 == num_wp):
                mav.mav.mission_ack_send(target_system, target_component,
                                         type=0)  # , mission_type=0)#type=msg.seq, mission_type=0)
                if (savefile is not None):
                    fileout.close()
                break
        elif msg.name is "MISSION_CURRENT":
            # print(msg)
            i = 1
        else:
            print(msg)

def WriteWPtoMAV(mavlink, WPmsg):
    count = len(WPmsg)
    target_component = 0  # 0

    mav.mav.mission_count_send(target_system, target_component, count)
    print("Send count " + str(count))
    MAV_MISSION_ACCEPTED = 0  # mission accepted OK
    seqNUM = 0
    while True:
        msg = mav.recv_msg()
        if msg is None:
            continue
        skip_msg = False
        for strnames in ignore_list:
            if (msg.name is strnames):
                skip_msg = True
        if (skip_msg is True):
            continue
        if msg.name is "MISSION_ACK":
            print(msg.name + "->GOOD")
            break
        elif msg.name is "MISSION_REQUEST":
            print(msg.name)
            # mav.mav.mission_ack_send(target_system, target_component, type=MAV_MISSION_ACCEPTED)  # , mission_type=0)#type=msg.seq, mission_type=0)
            msg = WPmsg[seqNUM]

            # if (msg.command == 22):
            #    msg.x = globalpos.lat / 1e7
            #    msg.y = globalpos.lon / 1e7
            #    msg.z = globalpos.alt / 1000

            # target_system = msg.target_system
            target_component = msg.target_component
            seq = seqNUM
            frame = msg.frame
            # if(SendFile=='Set2'):
            #    command = 16
            # else:
            command = msg.command
            current = msg.current
            autocontinue = msg.autocontinue
            param1 = msg.param1
            param2 = msg.param2
            param3 = msg.param3
            param4 = msg.param4
            x = msg.x
            y = msg.y
            z = 10  # msg.z
            print("Send SEQ " + str(seqNUM) + " " + str(msg))
            mav.mav.mission_item_send(target_system, target_component, seq, frame, command, current,
                                      autocontinue, param1, param2,
                                      param3, param4, x, y, z)
            seqNUM = seqNUM + 1

        elif msg.name is "STATUSTEXT":
            print(msg.name + " " + str(msg.text))
            break
        elif msg.name is "MISSION_CURRENT":
            # print(msg)
            z = 1
        else:
            print(msg.name)

def GetHomePositionWait(mav):
    return GetMessageData(mav, "HOME_POSITION")

def GetCurrentGlobalPositionWait(mav):
    return GetMessageData(mav, "GLOBAL_POSITION_INT")

def GetCurrentLocalPositionWait(mav):
    # print("Waiting on Local Position")
    while True:
        msg = mav.recv_msg()
        if msg is None:
            continue

        if msg.name is "LOCAL_POSITION_NED":
            return msg

def GetPositionTargetLocalNEDWait(mav):
    return GetMessageData(mav, 'POSITION_TARGET_LOCAL_NED')

def GetAttitudeWait(mav):
    return GetMessageData(mav, 'ATTITUDE')

def GetMessageData(mav, msg_name):
    while True:
        msg = mav.recv_msg()
        if msg is None:
            continue

        if msg.name is msg_name:
            return msg

mav = mavutil.mavlink_connection('udpin::14552')
# mav = mavutil.mavlink_connection("COM7",baud=115200)
print(mav.address)
print('wait')
mav.wait_heartbeat()
print('got beat')

# SET_POSITION_TARGET_LOCAL_NED
# set_position_target_local_ned_send(self, time_boot_ms, target_system, target_component, coordinate_frame, type_mask, x,
#                                   y, z, vx, vy, vz, afx, afy, afz, yaw, yaw_rate, force_mavlink1=False):

# mav.mav.set_position_target_local_ned_send()
dt = 0.01
VFTarget = [500, 500]
cvf = VFControl.CircleVectorField('Gradient')
cvf.mCircleRadius = 20
cvf.xc = VFTarget[0]
cvf.yc = VFTarget[1]
# cvf.bUsePathFunc = True
# cvf.velPathFunc = expPathFunc
# MAV_FRAME_LOCAL_NED

# Save the WP (not used)
#GetWPList(mav,'twoWPtemplate.pickl')
#sys.exit(1)
wpBlank = LoadWPFile('twoWPtemplate.pickl')

HP = GetHomePositionWait(mav)
print(HP)
CGP = GetCurrentGlobalPositionWait(mav)
print(CGP)
# WPList = GetWPList(mav)
plt.figure()
plt.ion()
uav = VFControl.VFUAV(dt)
BasePosLatLon = GetCurrentGlobalPositionWait(mav)

while True:
    CHP = GetHomePositionWait(mav)
    CLP = GetCurrentLocalPositionWait(mav)
    CGP = GetCurrentGlobalPositionWait(mav)

    ATT = GetAttitudeWait(mav)

    # print(CLP)
    plt.cla()
    plt.scatter(CLP.x, CLP.y)
    TargetLocal = GetPositionTargetLocalNEDWait(mav)

    # plt.scatter(TargetLocal.x,TargetLocal.y)
    # uav.SetPosition([CLP.x,CLP.y])
    curr_theta = ATT.yaw  # np.arctan2(CLP.vy,CLP.vx)

    new_vector = {'pos': [CLP.x, CLP.y], 'theta': curr_theta, 'velocity_vec': [CLP.vx, CLP.vy]}
    dir_theta = uav.ExportNewTurnAngleFromVF(new_vector, cvf, t=0)
    new_arm = 25.0  # m

    target_x = CLP.x + new_arm * np.cos(dir_theta)
    target_y = CLP.y + new_arm * np.sin(dir_theta)
    print("Now: " + str(curr_theta) + ' [' + str(CLP.x) + ', ' + str(CLP.y) + ']')
    print("New: " + str(dir_theta) + ' [' + str(target_x) + ', ' + str(target_y) + ']')

    vf = cvf.GetVF_XYUV(0, dt, uav, IncludeUAVPos=True)
    Q = plt.quiver(vf['x'], vf['y'], vf['u'], vf['v'])

    plt.scatter(target_x, target_y, marker='x')

    circle1 = plt.Circle((vf['xc'], vf['yc']), cvf.mCircleRadius, color='b', fill=False)
    ax = plt.gca()

    ax.add_artist(circle1)
    ax.add_artist(plt.scatter(vf['xc'], vf['yc']))
    plt.scatter(cvf.xc, cvf.yc)
    plt.axis('equal')
    plt.pause(0.000000000001)

    #continue #use to avoid waypoint changes

    # get vehicle XYZ (on earth)

    vlat = BasePosLatLon.lat / 1e7
    vlon = BasePosLatLon.lon / 1e7

    Gxy = utm.from_latlon(vlat, vlon)

    x = CLP.x
    y = CLP.y
    z = CLP.z

    #test for same coor
    #CPLL = utm.to_latlon(x + Gxy[0], y + Gxy[1], Gxy[2], Gxy[3])
    #actual data needed
    #CPLL = utm.to_latlon(Gxy[0], Gxy[1], Gxy[2], Gxy[3])

    CTLL = utm.to_latlon(target_x + Gxy[0], target_y + Gxy[1], Gxy[2], Gxy[3])
    CirCenter = utm.to_latlon(cvf.xc + Gxy[0], cvf.yc + Gxy[1], Gxy[2], Gxy[3])

    print("Base Pos",vlat, vlon)
    print("My Pos", CGP.lat/1e7, CGP.lon/1e7)
    print("Target Pos",CTLL[0],CTLL[1])
    print("Circ Pos",CirCenter[0],CirCenter[1])

    #print(CPLL[0], CPLL[1])

    wpBlank = LoadWPFile('twoWPtemplate.pickl')
    wpOut = wpBlank[:]
    wpOut = []
    wpOut.append([])
    wpOut[0] = copy.deepcopy(wpBlank[0])
    wpOut.append([])
    wpOut[1] = copy.deepcopy(wpBlank[1])
    wpOut.append([])
    wpOut[2] = copy.deepcopy(wpOut[1])
    #print(str(len(wpOut))+"#######################################"+str(len(wpBlank)))
    # sys.exit(1)
    wpOut[0].x = CGP.lat/1e7#CPLL[0]
    wpOut[0].y = CGP.lon/1e7#CPLL[1]
    wpOut[0].z = CLP.z
    wpOut[0].current = 0

    # wpOut[0].frame = 1 #MAV_FRAME_LOCAL_NED

    wpOut[1].x = CTLL[0]
    wpOut[1].y = CTLL[1]
    wpOut[1].z = CLP.z
    wpOut[1].seq = 1
    wpOut[1].current = 1

    # wpOut[1].frame = 1

    #wpOut.append([])
    #wpOut[2] = copy.deepcopy(wpOut[1])
    wpOut[2].x = CirCenter[0]
    wpOut[2].y = CirCenter[1]
    wpOut[2].z = CLP.z
    wpOut[2].seq = 2
    wpOut[2].current = 0

    WriteWPtoMAV(mav, wpOut)
    print('Sent WPs')
    plt.pause(2.0)

sys.exit(1)


#import pandas as pd #pip install pandas
#import xlwt #pip install xlwt
from openpyxl import load_workbook #pip install openpyxldd
from tqdm import tqdm #pip install tqdm
#import datetime

rb = load_workbook('./workedjournal.xlsx')
wb = load_workbook('./DataBase.xlsx')
print(wb.sheetnames)

def refresh_names():
    not_RAO_array = ['АРХПОИНТ', 'Библиотечно-информационный отдел', 'Информационный центр "Библиотека имени К.Д.Ушинского"',
                 'Сектор библиотечно-информационного обслуживания РАО', 'Сектор комплектования и обработки фонда', 'Сектор обслуживания читателей',
                 'Сектор хранения фонда']
    work_sheet_wb, work_sheet_wb_RAO, work_sheet_wb_other = wb['Сотрудники'], wb['РАО'], wb['Прочие']
    work_sheet_rb = rb['visitorReport']
    flag = False
    RAO_row, oth_row = 2, 2
    for rows in work_sheet_rb.iter_rows(values_only=True):
        if flag and rows[0][0] != '2':
            if rows[3] in not_RAO_array:
                work_sheet_wb_other.cell(row=oth_row, column=1).value = rows[0]
                work_sheet_wb_other.cell(row=oth_row, column=2).value = rows[3]
                oth_row += 1
            else:
                work_sheet_wb_RAO.cell(row=RAO_row, column=1).value = rows[0]
                work_sheet_wb_RAO.cell(row=RAO_row, column=2).value = rows[3]
                RAO_row += 1
        if rows[0] == 'Сотрудник':
            flag = True

# refresh_names() # Только при добавлении новых имён в БД
# wb.save('./DataBase.xlsx')

work_sheet_wb = wb['Сотрудники']
work_sheet_wb_RAO = wb['РАО']
work_sheet_wb_main = wb['Отделы']
work_sheet_wb_other = wb['Прочие']
work_sheet_rb = rb['visitorReport']

def search_data_by_name(name):
    sheet = work_sheet_rb
    n_row, n_title_row = 0, 0
    title_row = str()
    worker_dict, by_days_info = {}, {}
    in_name_flag = False

    for rows in work_sheet_rb.iter_rows(values_only=True):
        if str(rows[0])[:1] != '2':
            in_name_flag = False
        if in_name_flag:
            for i in range(len(title_row)):
                try:
                    by_days_info[title_row[i]].append(rows[i])
                except:
                    by_days_info[title_row[i]] = [rows[i]]
        if rows[0] == 'Сотрудник':
            n_title_row = n_row
            title_row = rows

        if title_row and rows[0] == name:
            in_name_flag = True
            for i in range(len(title_row)):
                worker_dict[title_row[i]] = rows[i]
        n_row += 1
    return worker_dict, by_days_info

n_row = 0
counter = -1
days_info = []
title_dict = {
    'График работы': 3,
    'Присутствие': 4,
    'Рабочее время': 5,
    'Отсутствие': 6,
    'Переработка': 7,
    'Баланс отработанного времени': 8
}
title_dict_1 = {
    'Сотрудник': 3,
    'Присутствие': 4,
    'Рабочее время': 5,
    'Отсутствие': 6,
    'Переработка': 7,
    'Баланс отработанного времени': 8
}

for rows in tqdm(work_sheet_wb_main.iter_rows(values_only=True)):
    n_row += 1

    if counter >= 0:
        for key, value in days_info.items():
            try:
                work_sheet_wb_main.cell(row=n_row, column=title_dict_1[key]).value = value[-counter - 1]
            except:
                pass

    elif rows[1]: # Не пустая строка
        worker_dict, days_info = search_data_by_name(rows[1])
        for key, value in worker_dict.items():
            try:
                work_sheet_wb_main.cell(row=n_row, column=title_dict[key]).value = value
                counter = len(title_dict) + 1
            except:
                pass

    counter -= 1
    if n_row > 2000:
        break

wb.save('./DataBase.xlsx')

RAO_structure = {
    'Отдел Информатизации' : [],
    'РУКОВОДСТВО': [],
    'ОТДЕЛЕНИЕ ФИЛОСОФИИ ОБРАЗОВАНИЯ И ТЕОРЕТИЧЕСКОЙ ПЕДАГОГИКИ': [],
    'ОТДЕЛЕНИЕ ПСИХОЛОГИИ И ВОЗРАСТНОЙ ФИЗИОЛОГИИ': [],
    'ОТДЕЛЕНИЕ ОБЩЕГО СРЕДНЕГО ОБРАЗОВАНИЯ': [],
    'ОТДЕЛЕНИЕ ПРОФЕССИОНАЛЬНОГО ОБРАЗОВАНИЯ': [],
    'ОТДЕЛЕНИЕ ОБРАЗОВАНИЯ И КУЛЬТУРЫ': [],
    'ОТДЕЛЕНИЕ РОССИЙСКОЙ СЛОВЕСНОСТИ': [],
    'СЕКРИТАРИАТ ПОПЕЧИТЕЛЬСКОГО СОВЕТА': [],
    'УПРАВЛЕНИЕ СЕКРИТАРИАТА И ПРОТОКОЛА': [],
    'ФЕДЕРАЛЬНЫЙ РЕСУРСНЫЙ ЦЕНТР ПСИХОЛОГИЧЕСКОЙ СЛУЖБЫ В СИСТЕМЕ ВЫСШЕГО ОБРАЗОВАНИЯ': [],
    'ЦЕНТР РАЗВИТИЯ ОБРАЗОВАНИЯ': [
        'ЛАБОРАТОРИЯ НАУЧНЫХ ПОДХОДОВ В УПРАВЛЕНИИ ОБРАЗОВАТЕЛЬНЫМИ СИСТЕМАМИ',
        'ЛАБОРАТОРИЯ РАЗВИТИЯ СОДЕРЖАНИЯ ОБРАЗОВАНИЯ',
        'ЛАБОРАТОРИЯ СОВРЕМЕННЫХ ФОРМ И МЕТОДОВ ОБУЧЕНИЯ',
        'ЛАБОРАТОРИЯ АНТРОПОЛОГИИ И ПЕДАГОГИЧЕСКОЙ КОМПАРАТИВИСТИКИ',
        'ЛАБОРАТОРИЯ СОЦИОЛОГИЧЕСКИХ ИССЛЕДОВАНИЙ В ОБРАЗОВАНИИ',
        'ЛАБОРАТИРИЯ РАЗВИТИЯ ЦИФРОВОЙ ОБРАЗОВАТЕЛЬНОЙ СРЕДЫ'],
    'ЦЕНТР РАЗВИТИЯ ВЫСШЕГО И СРЕДНЕГО ПРОФЕССИОНАЛЬНОГО ОБРАЗОВАНИЯ': [
        'ЛАБОРАТОРИЯ РАЗВИТИЯ СРЕДНЕГО ПРОФЕССИОНАЛЬНОГО ОБРАЗОВАНИЯ',
        'ЛАБОРАТОРИЯ РАЗВИТИЯ ВЫСШЕГО ПРОФЕССИОНАЛЬНОГО ОБРАЗОВАНИЯ',
        'ЛАБОРАТОРИЯ КООРДИНАЦИИ И МОНИТОРИНГА ДИССЕРТАЦИОННЫХ ИССЛЕДОВАНИЙ ПО ПЕДАГОГИЧЕСКИМ И ПСИХОЛОГИЧЕСКИМ НАУКАМ'
    ],
    'ЦЕНТР ОБРАЗОВАТЕЛЬНОГО ЗАКОНОДАТЕЛЬСТВА И ПРОБЛЕМ ПРАВОПРИМЕНЕНИЯ': [],
    'УЧЕБНЫЙ ЦЕНТР': [],
    'ЦЕНТР ДУХОВНОГО-НРАВСТВЕННОГО ВОСПИТАНИЯ И ТЕОЛОГИИ': [],
    'ЦЕНТР РАЗВИТИЯ ПЕДАГОГИЧЕСКОГО ОБРАЗОВАНИЯ': [
        'Лаборатория продвижения и пиар сопровождения педагогического образования'
    ],
    'ЦЕНТР РАЗВИТИЯ ИНКЛЮЗИВНОГО ОБРАЗОВАНИЯ': [
        'Лаборатория обучающихся с особыми образовательными потребностями',
        'Лаборатория содержания и форм подготовки специалистов с детьми с ОВЗ и инвалидностью'
    ],
    'ЦЕНТР ВОСПИТАНИЯ И РАЗВИТИЯ ЛИЧНОСТИ': [
        'ЛАБОРАТОРИЯ РАЗВИТИЯ ПРОСВЕТИТЕЛЬНОЙ ДЕЯТЕЛЬНОСТИ',
        'ЛАБОРАТОРИЯ ГУМАНИТАРНОГО ОБРАЗОВАНИЯ И ФОРМИРОВАНИЯ МИРОВОЗРЕНИЯ',
        'ЛАБОРАТОРИЯ СОВРЕМЕННЫХ ФОРМ И МЕТОДОВ ВОСПИТАНИЯ',
        'ЛАБОРАТОРИЯ СОВРЕМЕННЫХ ФОРМ И МЕТОДОВ ПРОФЕССИОНАЛЬНОГО САМООПРЕДЕЛЕНИЯ И ПРОФОРИЕНТАЦИИ',
        'Лаборатория научных основ изучения личности развития ребенка',
        'Лаборатория развития содержания воспитания и социализации',
        'Лаборатория психологического сопровождения развития личности в образовательном процессе'
    ],
    'ЦЕНТР РОДНЫХ ЯЗЫКОВ И КУЛЬТУР НАРОДОВ РФ': [],
    'ЦЕНТР РУССКОГО ЯЗЫКА И СЛАВИСТИКИ': [],
    'ЦЕНТР ЭКСПЕРТИЗЫ В ОБРАЗОВАНИИ': [],
    'ИНФОРМАЦИОННЫЙ ЦЕНТР "БИБЛИОТЕКА им. К.Д. УШИНСКОГО"': [
        'МЕТОДИЧЕСКИЙ ОТДЕЛ'
    ],
    'УПРАВЛЕНИЕ КООРДИНАЦИИ НАУЧНЫХ ИССЛЕДОВАНИЙ И ПОДГОТОВКИ КАДРОВ ВЫСШЕЙ КВАЛИФИКАЦИИ': [
        'ОТДЕЛ КООРДИНАЦИИ НАУЧНЫХ ИССЛЕДОВАНИЙ',
        'ОТДЕЛ КООРДИНАЦИИ ДЕЯТЕЛЬНОСТИ ПО ПОДГОТОВКЕ КАДРОВ ВЫСШЕЙ КВАЛИФИКАЦИИ',
        'ОТДЕЛ ПО РАБОТЕ С ПРОЕКТАМИ',
        'Отдел редакционно - издательской работы'
    ],
    'УПРАВЛЕНИЕ МЕЖДУНАРОДНОГО СОТРУДНИЧЕСТВА, СВЯЗЕЙ С ОБЩЕСТВЕННОСТЬЮ И СМИ': [
        'ОТДЕЛ МЕЖДУНАРОГО СОТРУДНИЧЕСТВА',
        'ОТДЕЛ ПО СВЯЗЯМ С ОБЩЕСТВЕННОСТЬЮ И СМИ'
    ],
    'ДЕПАРТАМЕНТ УПРАВЛЕНИЯ ДЕЛАМИ РАО': [
        'ОТДЕЛ КАДРОВ И ОХРАНЫ ТРУДА'
    ],
    'УПРАВЛЕНИЕ ПРАВОВОГО ОБЕСПЕЧЕНИЯ И ДЕЛОПРОИЗВОДСТВА': [
        'ОТДЕЛ ПРАВОВОГО ОБЕСПЕЧЕНИЯ И ИМУЩЕСТВЕННЫХ ОТНОШЕНИЙ',
        'ОТДЕЛ ДЕЛОПРОИЗВОДСТВА',
        'СЕКТОР НАУЧНОГО АРХИВА'
    ],
    'УПРАВЛЕНИЕ ФИНАНСОВОГО ПЛАНИРОВАНИЯ И БУХГАЛТЕРСКОГО УЧЕТА': [
        'ПЛАНОВЫЙ ОТДЕЛ',
        'ОТДЕЛ БУХГАЛТЕРСКОГО УЧЕТА И ОТЧЕТНОСТИ',
        'КОНТРАКТНАЯ СЛУЖБА',
        'ХОЗЯЙСТВЕННЫЙ ОТДЕЛ'
    ]
}


import unittest
from datetime import datetime
from airflow.models import DagBag, TaskInstance

class TestXComExamplesDag(unittest.TestCase):

    def setUp(self):
        self.dagbag = DagBag()

    def test_xcoms(self):
        dag_id = 'hello_world_xcoms'
        dag = self.dagbag.get_dag(dag_id)
        push_to_xcoms_task = dag.get_task('push_to_xcoms')
        pull_from_xcoms_task = dag.get_task('pull_from_xcoms')

        execution_date = datetime.now()

        push_to_xcoms_ti = TaskInstance(task=push_to_xcoms_task, execution_date=execution_date)
        context = push_to_xcoms_ti.get_template_context()
        push_to_xcoms_task.execute(context)

        pull_from_xcoms_ti = TaskInstance(task=pull_from_xcoms_task, execution_date=execution_date)

        result = pull_from_xcoms_ti.xcom_pull(key="dummyKey")
        self.assertEqual(result, 'dummyValue')

    def test_xcom_in_templated_field(self):
        dag_id = 'hello_world_xcoms'
        dag = self.dagbag.get_dag(dag_id)
        push_to_xcoms_task = dag.get_task('push_to_xcoms')

        execution_date = datetime.now()

        push_to_xcoms_ti = TaskInstance(task=push_to_xcoms_task, execution_date=execution_date)
        context = push_to_xcoms_ti.get_template_context()
        push_to_xcoms_task.execute(context)

        templated_xcoms_value_task = dag.get_task('templated_xcoms_value')
        templated_xcoms_value_ti = TaskInstance(task=templated_xcoms_value_task, execution_date=execution_date)
        context = templated_xcoms_value_ti.get_template_context()

        bash_operator_templated_field = 'bash_command'

        rendered_template = templated_xcoms_value_task.render_template

        bash_command_value = getattr(templated_xcoms_value_task, bash_operator_templated_field)

        bash_command_rendered_value = rendered_template(bash_operator_templated_field, bash_command_value, context)

        self.assertEqual(bash_command_rendered_value, 'echo dummyValue')

suite = unittest.TestLoader().loadTestsFromTestCase(TestXComExamplesDag)
unittest.TextTestRunner(verbosity=2).run(suite)


import numpy as np
import matplotlib.pyplot as plt

def simple_plot():
    plt.figure(figsize=(8, 6), dpi=80)
    plt.ion()
    for index in range(100):
        plt.cla()
        plt.title("Graph")
        plt.grid(True)
        x = np.linspace(-1.5 + 0.1*index, 1.5+0.1*index, 256, endpoint=True)
        y = 3 * np.pi * np.exp(-5 * np.sin(2 * np.pi * x))
        # set x axis
        plt.xlabel("X")
        plt.xlim(-3 + 0.1*index, 3 + 0.1*index)
        plt.xticks(np.linspace(-3 + 0.1*index, 3+0.1*index, 9, endpoint=True))
        # set y axis
        plt.ylabel("Y")
        plt.ylim(-1.0, 1400.0)
        plt.yticks(np.linspace(0, 1400, 20, endpoint=True))
        plt.plot(x, y, "g-", linewidth=2.0, label="h(t)")
        plt.legend(loc="upper left", shadow=True)
        plt.pause(0.1)
    plt.ioff()
    plt.show()
    return

simple_plot()


"""
Script automating backtest on kryll.io and sending results to an API
Made by Thomas with the help of torkium
"""
import time
import random
import sys
from selenium.webdriver.support.ui import Select
from custom.user_config import UserConfig
from custom.utilities import UtilityTools
from custom.css_const import CssConst
from custom.selenium_utilities import SeleniumUtilities
from custom.backtest_config import BacktestConfig
from custom.api import Api
from getpass import getpass

def set_input_date(start_input_date, end_input_date):
    """
    Takes a start date, end date and set them in inputs
    """
    dates_inputs = sel_tools.get_elements(css.DATES_INPUTS)
    start_input = dates_inputs[0]
    end_input = dates_inputs[1]
    sel_tools.driver.execute_script('arguments[0].removeAttribute("readonly")', start_input)
    sel_tools.driver.execute_script('arguments[0].removeAttribute("readonly")', end_input)
    start_input.clear()
    start_input.send_keys(start_input_date)
    end_input.clear()
    end_input.send_keys(end_input_date)

def exec_backtest(backtest_config):
    """
    Takes a strategy name, a strategy id, a pair, and a backtest date
    return True if no errors else return False
    """
    if not api.backtest_has_failed(backtest_config) and not api.backtest_already_did(backtest_config):
        tools.log("[ℹ] Test in progress, Please Wait...")
        # set date into input
        set_input_date(backtest_config.getStart(), backtest_config.getEnd())
        test_btn = sel_tools.get_element(css.BACKTEST_START_BTN)
        sel_tools.clean_network_calls()
        sel_tools.click_on_element(test_btn)

        error_during_backtest = sel_tools.check_error_during_backtest()
        if error_during_backtest:
            raise Exception("Error during backtest")
        sel_tools.clean_network_calls()
        hold = sel_tools.get_element_double(css.ANALYSE_TAB_HOLD)
        # click on depth analysis button
        sel_tools.click_on_element(sel_tools.get_element(css.ANALYSE_TAB_DEEP_ANALYSE_LINK))
        windows_handle = sel_tools.wait_for_windows_handle(300)
        if not windows_handle:
            tools.log("[❌] depth analysis button is break on kryll side, period canceled")
            raise Exception("depth analysis button is break on kryll side, period canceled")
        send_ok = False
        try:
            sel_tools.driver.switch_to.window(sel_tools.driver.window_handles[1])
            # wait for the advanced bt page to load
            depth_analysis_page_loaded = sel_tools.wait_for_element(css.ADVANCED_ANALYSE_TRADE, 300)
            if not depth_analysis_page_loaded:
                tools.log("[⚠] depth analysis tab seems to don't load, refresh and retry...")
                # retry
                sel_tools.refresh()
                depth_analysis_page_loaded = sel_tools.wait_for_element(css.ADVANCED_ANALYSE_TRADE, 120)
            if depth_analysis_page_loaded:
                send_ok = api.send_result(backtest_config, {"hold": hold})

        except Exception as error:
            tools.log("[❌] Sending backtest : Exception occured : " + str(error))

        if not send_ok:
            screenshot_name = "backtest_fail_deep_analysis_" + str(backtest_config.getId()) + ".png"
            sel_tools.save_screenshot(screenshot_name)
            tools.log("[❌] You can see the screenshot on this file : " + screenshot_name)
        sel_tools.driver.close()
        sel_tools.driver.switch_to.window(sel_tools.driver.window_handles[0])
        if send_ok:
            return True
        tools.log("[❌] Error during sending the result, period canceled.")
        raise Exception("Error during backtest")
    return False

def install_strat_if_needed():
    try:
        tools.log("[ℹ] Checking if strategy is installed...")
        sel_tools.get_element(css.BACKTEST_BTN)
    except Exception:
        tools.log("[ℹ] Install in progress...")
        install_btn = sel_tools.get_element(css.INSTALL_BTN)
        sel_tools.click_on_element(install_btn)
    return True

def update_strat_if_needed():
    tools.log("[ℹ] Checking if strategy needs to be updated")
    time.sleep(5)
    if "btn-warning" in sel_tools.get_element(css.UPDATE_BTN).get_attribute("class").split():
        tools.log("[ℹ] Updating the strategy...")
        update_btn = sel_tools.get_element(css.UPDATE_BTN)
        sel_tools.click_on_element(update_btn)
    else:
        tools.log("[ℹ] Strategy is up to date")

def get_recommended_pairs():
    try:
        recommended_pairs = sel_tools.get_elements(css.RECOMMEND_PAIRS)
    except Exception:
        tools.log("[⚠] No recommanded pair")
        recommended_pairs = {}
    recommended_pairs_list = []
    for i in recommended_pairs:
        recommended_pairs_list.append(i)
    return recommended_pairs_list

def get_exchanges_to_test():
    if "exchanges" in user.config_file:
        exchange_options = sel_tools.get_element(css.EXCHANGE).find_elements_by_tag_name("option")
        user_exchanges = user.config_file["exchanges"]
        final_exchanges = []
        for exchange in exchange_options:
            if exchange.text in user_exchanges:
                final_exchanges.append(exchange)
    else:
        final_exchanges = [sel_tools.get_element(css.BINANCE_EXCHANGE)]
    return final_exchanges

def get_pairs_to_test():
    recommended_pairs_list = get_recommended_pairs()
    final_pairs = []
    pairs_input = sel_tools.get_element(css.PAIRS_INPUT)
    pairs_list = pairs_input.find_elements_by_tag_name("option")
    random.shuffle(pairs_list)
    # Backtest recommended first
    for i in pairs_list:
        if i in recommended_pairs_list:
            pairs_list.remove(i)
    total_pairs_list = recommended_pairs_list + pairs_list
    for i in total_pairs_list:
        # Get infos
        pair = i.text.strip()
        recommended = 0
        force_pair = 0
        if i in recommended_pairs_list:
            recommended = 1

        if "pair" in user.config_file:
            if not pair.replace(" / ", "/") in user.config_file["pair"]:
                continue
            if pair.replace(" / ", "/") in user.config_file["pair"]:
                force_pair = 1
        if "accu" in user.config_file and not pair.split(" / ")[1] in user.config_file["accu"]:
            continue

        # check if user want to test every pairs
        if force_pair == 0 and recommended == 0 and user.config_file["every_pairs"] == "n":
            continue
        if recommended == 1 and "skip_recommended_pair" in user.config_file and user.config_file["skip_recommended_pair"] == "y" and force_pair == 0:
            continue
        final_pairs.append(i)
    return final_pairs

def select_exchange(exchange):
    sel_tools.clean_network_calls()
    exchange_text = exchange.text
    exchange_select = Select(sel_tools.get_element(css.EXCHANGE))
    exchange_select.select_by_visible_text(exchange_text)
    sel_tools.wait_network_calls_loaded(5)
    return True

def select_pair(pair):
    sel_tools.clean_network_calls()
    pair_text = pair.text
    pairs_input = Select(sel_tools.get_element(css.PAIRS_INPUT))
    pairs_input.select_by_value(pair_text.replace(" / ", "-"))
    sel_tools.wait_network_calls_loaded(5)

def is_pair_listed(pair):
    pairs_input = Select(sel_tools.get_element(css.PAIRS_INPUT))
    try:
        pairs_input.select_by_value(pair.text.strip().replace(" / ", "-"))
    except Exception:
        return False
    return True

def get_periods(backtest_config):
    dates_inputs = sel_tools.get_elements(css.DATES_INPUTS)
    start_input = dates_inputs[0]
    min_date = sel_tools.wait_for_attribute_value(start_input, "min")
    tools.log(f"[ℹ] min date : {min_date}", verbose=True)
    backtest_dates = api.get_backtest_dates(min_date=min_date, backtest_config=backtest_config)
    return backtest_dates

def click_on_backtest_button():
    backtest_page_loaded = False
    while not backtest_page_loaded:
        try:
            sel_tools.click_on_element(sel_tools.get_element(css.BACKTEST_BTN), True)
            sel_tools.get_element_text(css.STRAT_NAME_BACKTEST)
            sel_tools.get_element_text(css.STRAT_VERSION)
            backtest_page_loaded = True
        except:
            tools.log("[ℹ] White page... Retry")
            sel_tools.refresh()
            continue

def is_recommended_pair(pair):
    recommended_pairs_list = get_recommended_pairs()
    return pair in recommended_pairs_list

def run_backtest(backtest_config):
    backtest_done = False
    backtest_failed = False
    backtest_failed_screenshot = True
    retry_order_skipped = 0
    while True not in (backtest_done, backtest_failed):
        # Try to exec the backtest
        try:
            exec_backtest(backtest_config)
            backtest_done = True
        # If backtest failed, check if order too small or too many order skipped. If yes, retry x3 with more start wallet
        except:
            log = ""
            try:
                log = sel_tools.get_element_text(css.LOGS_LAST_LINE)
                if retry_order_skipped <= 3 and ("order too small" in log or "Too many orders skipped" in log):
                    backtest_failed_screenshot = False
                    retry_order_skipped = retry_order_skipped + 1
                    if retry_order_skipped > 3:
                        backtest_failed = True
                        tools.log(f"[❌] {log}")
                    else:
                        sel_tools.click_on_element(sel_tools.get_element(sel_tools.css.BALANCE_BUTTON, 2))
                        multiplicator = 1000 * retry_order_skipped
                        try:
                            amount_one_input = sel_tools.get_element(sel_tools.css.STARTING_AMOUNT_ONE)
                            amount_one = amount_one_input.get_attribute("value")
                            amount_one = float(amount_one)
                            amount_one = amount_one * multiplicator
                            amount_one_input.clear()
                            amount_one_input.send_keys(str(amount_one))
                        except:
                            pass
                        try:
                            amount_two_input = sel_tools.get_element(sel_tools.css.STARTING_AMOUNT_TWO)
                            amount_two = amount_two_input.get_attribute("value")
                            amount_two = float(amount_two)
                            amount_two = amount_two * multiplicator
                            amount_two_input.clear()
                            amount_two_input.send_keys(str(amount_two))
                        except:
                            pass
                        tools.log(f"[❌] Too many order skipped, retry with x{multiplicator} amount {retry_order_skipped}/3.")
                else:
                    backtest_failed = True
                    tools.log(f"[❌] {log}")
            except:
                backtest_failed = True
            if "missing data" in log:
                tools.log(f"[❌] {log}")
                backtest_failed_screenshot = False
                backtest_failed = True
            if backtest_failed:
                api.backtest_add_failed(backtest_config=backtest_config, log=log)
                tools.log("[❌] Backtest Failed.")
                if backtest_failed_screenshot:
                    screenshot_name = "backtest_fail_" + str(backtest_config.getId()) + ".png"
                    sel_tools.save_screenshot(screenshot_name)
                    tools.log("[❌] You can see the screenshot on this file : " + screenshot_name)

def run():
    # If more 1 tab open, closed useless tabs
    sel_tools.close_unused_tabs()
    for strat_id in strat_ids:
        backtest_config = BacktestConfig()
        backtest_config.setStratId(strat_id)
        # Go to strategy page
        tools.log("[ℹ] Go to strategy page...")
        sel_tools.get("https://platform.kryll.io/marketplace/" + strat_id)
        # Check if strategy is installed, and install it
        install_strat_if_needed()
        # Check if strategy needs to be updated
        update_strat_if_needed()
        # Click on backtest button
        tools.log("[ℹ] Click on backtest button...", True)
        click_on_backtest_button()
        # Get strategy name, strategy version
        tools.log("[ℹ] Get strategy name and version...", True)
        backtest_config.setStratName(sel_tools.get_element_text(css.STRAT_NAME_BACKTEST).strip())
        backtest_config.setStratVersion(sel_tools.get_element_text(css.STRAT_VERSION).split(" ")[1])
        # Get recommended pairs list
        tools.log("[ℹ] Get recommended pair list...", True)
        recommended_pairs = get_recommended_pairs()
        # Get Exchanges to test
        tools.log("[ℹ] Get exchanges to test...", True)
        exchanges = get_exchanges_to_test()
        # For each exchange
        for exchange in exchanges:
            backtest_config.setExchange(exchange.text)
            # Select exchange and wait network pair load
            tools.log(f"[ℹ] Select exchange {backtest_config.getExchange()}...")
            select_exchange(exchange)
            # Get pairs to test
            tools.log("[ℹ] Get pairs...", True)
            pairs = get_pairs_to_test()
            # For each pairs
            for pair in pairs:
                backtest_config.setPair(pair.text)
                backtest_config.setRecommended(pair in recommended_pairs)
                # Check if pair exist on exchange
                if not is_pair_listed(pair):
                    tools.log(f"[ℹ] Pair {backtest_config.getPair()} not listed...")
                    continue
                # Select pair wait network dates load and get min date
                tools.log(f"[ℹ] Select pair {backtest_config.getPair()}...")
                select_pair(pair)
                # Get periods to test
                tools.log("[ℹ] Get periods to test...", True)
                periods = get_periods(backtest_config)
                # For each period
                for period in periods:
                    backtest_config.setPeriod(period["period"])
                    backtest_config.setStart(period["start"])
                    backtest_config.setEnd(period["end"])
                    tools.log("[ℹ] **************************************")
                    tools.log("[ℹ] *         Running backtest           *")
                    tools.log("[ℹ] **************************************")
                    tools.log(f"[ℹ] * Strategy name : {backtest_config.getStratName()}")
                    tools.log(f"[ℹ] * Strategy version : {backtest_config.getStratVersion()}")
                    tools.log(f"[ℹ] * Exchange : {backtest_config.getExchange()}")
                    tools.log(f"[ℹ] * Backtest pair : {backtest_config.getPair()}")
                    tools.log(f"[ℹ] * Backtest period : {backtest_config.getPeriod()}")
                    tools.log(f"[ℹ] * From : {backtest_config.getStart()} to {backtest_config.getEnd()}")
                    tools.log("[ℹ] ***************************************")
                    # Run backtest
                    run_backtest(backtest_config)
        # strategy backtested, next
        tools.log("[ℹ] strategy backtested !")
    return True

# ----------------------
# Start of the program
# ----------------------
configFile = "config.yaml"
if len(sys.argv) > 1:
    configFile = sys.argv[1] + ".yaml"
    # print("Use config file " + configFile)
user = UserConfig(configFile)
tools = UtilityTools(user_config_file=user.config_file)
if configFile != "config.yaml":
    tools.log("[ℹ] Config File: " + configFile)
css = CssConst()

if "browser" in user.config_file:
    config_browser = user.config_file["browser"].lower()
else:
    config_browser = False
client_driver = tools.detect_browsers(user.config_file["headless"], config_browser)
api = Api(user_config_file=user.config_file, driver=client_driver)
sel_tools = SeleniumUtilities(user_config_file=user.config_file, driver=client_driver)
sel_tools.driver.get("https://platform.kryll.io/login")
if user.login:
    tools.log("[ℹ] Login using account :" + user.login["email"])
    sel_tools.get_element(css.EMAIL_INPUT).send_keys(user.login["email"])
    sel_tools.get_element(css.PASSWORD_INPUT).send_keys(user.login["password"])
else:
    sel_tools.get_element(css.EMAIL_INPUT).send_keys(input("email :"))
    sel_tools.get_element(css.PASSWORD_INPUT).send_keys(getpass("password :"))

if user.config_file["ask_2fa"] == "n":
    sel_tools.click_on_element(sel_tools.get_element(css.LOG_IN_BTN))
else:
    twofa = input("2FA :").lower()
    sel_tools.get_element(css.TWO_FA_INPUT).send_keys(twofa)

sel_tools.wait_for_element(css.USER_DROPDOWN, 100000)

if user.config_file["get_strategies_from_marketplace"] == "y":
    tools.log("[ℹ] Getting strategies from the marketplace...")
    sel_tools.driver.get("https://platform.kryll.io/marketplace/top")
    ids = sel_tools.get_elements(
        "div.col-sm-12 > app-card-strategy-user:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(2) > div:nth-child(1) > h3:nth-child(1) > a:nth-child(1)"
    )
    strat_ids = []
    for strategy_id in ids:
        strat_ids.append(strategy_id.get_attribute("href").split("/")[4])
else:
    if "strat_ids" in user.config_file:
        strat_ids = user.config_file["strat_ids"]
    else:
        strat_ids = tools.ask_strat()

COUNT_QUICK_FAIL = 0
EXECUTION_ID = 0
while COUNT_QUICK_FAIL < 3:
    EXECUTION_ID = EXECUTION_ID + 1
    start = time.time()
    try:
        random.shuffle(strat_ids)
        run()
    except Exception as error:
        SCREENSHOT_NAME = "screen_fail_" + str(EXECUTION_ID) + ".png"
        sel_tools.save_screenshot(SCREENSHOT_NAME)
        tools.log("[❌] Exception occured on execution " + str(EXECUTION_ID) + " : " + str(error))
        tools.log("[❌] You can see the screenshot on this file : " + SCREENSHOT_NAME)
        tools.log("[❌] Retry...")
    end = time.time()
    elapsed = end - start
    if elapsed < 30:
        COUNT_QUICK_FAIL = COUNT_QUICK_FAIL + 1
        tools.log("[❌] Quick fail : " + str(COUNT_QUICK_FAIL) + "/3")
    else:
        COUNT_QUICK_FAIL = 0


import pyspark.pandas as ps
from flask import current_app
from pyspark.conf import SparkConf
from pyspark.sql import SparkSession
from pyspark.sql import functions as fn
from pyspark.sql.types import StructField, IntegerType, DateType, StringType, FloatType
from app import db

def _common_config(conf):
    jars = [
        "%s/%s" % (current_app.config["PACKAGE_DIRECTORY"], _)
        for _ in ["sqlite-jdbc-3.34.0.jar", "mysql-connector-j-8.0.32.jar"]
    ]
    conf = (
        conf.set("spark.python.worker.memory", "128m")
        .set("spark.task.maxFailures", "10")
        .set("spark.rdd.compress", "true")
        .set("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version", "2")
        .set("spark.jars", ",".join(jars))
        .set("spark.sql.legacy.timeParserPolicy", "LEGACY")
        .set("spark.sql.parquet.datetimeRebaseModeInWrite", "LEGACY")
        .set("spark.sql.adaptive.enabled", "true")
        .set("spark.sql.adaptive.skewJoin.enabled", "true")
        .set("spark.shuffle.spill.compress", "true")
        .set("spark.sql.broadcastTimeout", 600)
        .set("spark.network.timeout", "240s")
        .set("spark.driver.extraClassPath", current_app.config["SPARK_DB_DRIVER"])
        .set("spark.executor.extraClassPath", current_app.config["SPARK_DB_DRIVER"])
    )
    return conf

def _spark_config(app_name, num_cores, driver_memory):
    conf = (
        SparkConf()
        .setAppName(app_name)
        .setMaster("local[%s,4]" % num_cores)
        .set("spark.driver.memory", driver_memory)
    )
    return _common_config(conf)

def get_spark(
    app_name="weatherdata-api",
    num_cores=2,
    driver_memory="8g",
):
    conf = _spark_config(app_name, num_cores, driver_memory)
    spark_builder = SparkSession.builder.config(conf=conf)
    spark = spark_builder.getOrCreate()
    spark.sparkContext.setLogLevel("WARN")

    return spark

class StringField(StructField):
    def __init__(self, field_name, nullable=True):
        super().__init__(name=field_name, dataType=StringType())
        self.nullable = nullable

class IntField(StructField):
    def __init__(self, field_name, nullable=True):
        super().__init__(name=field_name, dataType=IntegerType())
        self.nullable = nullable

class DateField(StructField):
    def __init__(self, field_name, nullable=True):
        super().__init__(name=field_name, dataType=DateType())
        self.nullable = nullable

class FloatField(StructField):
    def __init__(self, field_name, nullable=True):
        super().__init__(name=field_name, dataType=FloatType())
        self.nullable = nullable

def table_to_df(tablename, custom_schema=None):
    """Lesson learned:
    Since sqlite was chosen as a test option, this gets messy here because
    spark/pandas can't import date fields from sqlite.
    TODO: drop sqlite and deploy with mysql container even for tests and dev
    """
    options = (
        {
            "customSchema": custom_schema,
        }
        if custom_schema
        else None
    )

    if custom_schema:
        psdf = ps.read_sql(
            tablename, con=current_app.config["SPARK_DB_CON"], options=options
        )
    else:
        psdf = ps.read_sql(tablename, con=current_app.config["SPARK_DB_CON"])

    df = psdf.to_spark(index_col=None)
    if custom_schema and "date" in custom_schema:
        df = df.withColumn("date", fn.to_date(fn.col("date"), "yyyy-MM-dd"))

    return df

def df_to_db(df, tablename, if_exists="append"):
    """
    This looks like and antipattern because it is.
    It's needed for this demo because I'm using sqlite which
    won't allow multiple write connections
    """

    df.toPandas().to_sql(tablename, con=db.engine, if_exists=if_exists, index=False)


import os
from celery import Celery
from celery.signals import worker_process_shutdown, worker_ready

from .db_client import connect_to_db_sync, shutdown_db_sync, db

# def find_tasks() -> list[str]:
#     curr_dir =

@worker_ready.connect
def db_conn_init(*args, **kwargs):
    """ Initialize a clean DB connection. """
    if db.cluster is not None:
        shutdown_db_sync()
    connect_to_db_sync()

@worker_process_shutdown.connect
def cassandra_shutdown(*args, **kwargs):
    """Close DB connection"""
    shutdown_db_sync()

CELERY_BROKER: str = (
    f"pyamqp://user:Pass1234@rabbitmq:5672//"
)
CELERY_BACKEND: str = f"redis://redis:6379"
celery_app = Celery(
    __name__, broker=CELERY_BROKER, backend=CELERY_BACKEND
)

celery_app.conf.broker_pool_limit = 500
celery_app.conf.worker_prefetch_multiplier = 5
celery_app.conf.worker_send_task_events = False
celery_app.conf.task_send_sent_event = False
celery_app.conf.task_ignore_result = True
celery_app.conf.task_store_errors_even_if_ignored = True
celery_app.conf.result_expires = 30

celery_app.autodiscover_tasks(packages=['app'])


# -*- coding:utf8 -*-

import time
import json
import requests
import tkinter
from tkinter import ttk, scrolledtext
from AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTClient

"""
 How to compile
 py2applet --make-setup test.py
 python setup.py py2app

"""

class GUI:
    def __init__(self):
        self.main_window = tkinter.Tk()
        self.main_window.title('hue controller')
        self.main_window.geometry('500x800')
        self.press = False
        self.press2 = False

        self.canvas = tkinter.Canvas(
        self.main_window, # キャンバスの作成先アプリ
        width = 400, # キャンバスの横サイズ
        height = 400, # キャンバスの縦サイズ
        bg = "white" # キャンバスの色
        )
        self.canvas.pack()

        self.canvas2 = tkinter.Canvas(
        self.main_window, # キャンバスの作成先アプリ
        width = 400, # キャンバスの横サイズ
        height = 50, # キャンバスの縦サイズ
        bg = "blue" # キャンバスの色
        )
        self.canvas2.pack()

        self.button1 = ttk.Button(text="ON")
        self.button1.pack()
        self.button1.bind("<Button-1>",self.clickButton1)

        self.button2 = ttk.Button(text="OFF")
        self.button2.pack()
        self.button2.bind("<Button-1>",self.clickButton2)

        self.buttonAlertOn = ttk.Button(text="Alert ON")
        self.buttonAlertOn.pack()
        self.buttonAlertOn.bind("<Button-1>",self.clickAlertOn)

        self.buttonAlertOff = ttk.Button(text="Alert OFF")
        self.buttonAlertOff.pack()
        self.buttonAlertOff.bind("<Button-1>",self.clickAlertOff)

        self.button3 = ttk.Button(text="Exit")
        self.button3.pack()
        self.button3.bind("<Button-1>",self.clickButton3)

        self.canvas.bind(
            "<Motion>", # 受付けるイベント
            self.mouse_move_func # そのイベント時に実行する関数
        )

        self.canvas.bind(
            "<ButtonPress>", # 受付けるイベント
            self.mouse_click_func # そのイベント時に実行する関数
        )

        self.canvas.bind(
            "<ButtonRelease>", # 受付けるイベント
            self.mouse_release_func # そのイベント時に実行する関数
        )

        self.canvas2.bind(
            "<Motion>", # 受付けるイベント
            self.mouse_move_func2 # そのイベント時に実行する関数
        )

        self.canvas2.bind(
            "<ButtonPress>", # 受付けるイベント
            self.mouse_click_func2 # そのイベント時に実行する関数
        )

        self.canvas2.bind(
            "<ButtonRelease>", # 受付けるイベント
            self.mouse_release_func2 # そのイベント時に実行する関数
        )

        self.setupAWSIoT()

    def setupAWSIoT(self):
        self.myMQTTClient = AWSIoTMQTTClient('myClientID2') # 適当な値でOK
        self.myMQTTClient.configureEndpoint('a1gbwzf7otibfp-ats.iot.us-west-2.amazonaws.com', 8883) # 管理画面で確認
        self.myMQTTClient.configureCredentials('data/rootCA.pem', 'data/fd35544055-private.pem.key', 'data/fd35544055-certificate.pem.crt')
        self.myMQTTClient.configureOfflinePublishQueueing(-1) # Infinite offline Publish queueing
        self.myMQTTClient.configureDrainingFrequency(2) # Draining: 2 Hz
        self.myMQTTClient.configureConnectDisconnectTimeout(10) # 10 sec
        self.myMQTTClient.configureMQTTOperationTimeout(5) # 5 sec
        self.myMQTTClient.connect()

    def sendMessage(self,message):
        self.myMQTTClient.publish("myTopic2", json.dumps({'message' : message}), 1)

    def mouse_move_func(self,event):
        # 現在のマウスの位置
        x = event.x
        y = event.y

        # マウスボタンが押されている時だけ円を描画
        if self.press:
            self.canvas.create_oval(x - 3, y - 3, x + 3, y + 3, fill = "blue", width=0)
            print("drag "+str(x)+":"+str(y))
            self.sendMessage("drag:"+str(x)+":"+str(y))

    def mouse_click_func(self,event):
        # マウスボタンが押された
        self.press = True

    def mouse_release_func(self,event):
        # マウスボタンが離された
        self.press = False

    def mouse_move_func2(self,event):
        if self.press2:
            x = event.x
            print("volume "+str(x))
            self.sendMessage("volume:"+str(x))
        return

    def mouse_click_func2(self,event):
        self.press2 = True
        return

    def mouse_release_func2(self,event):
        self.press2 = False
        return

    def clickButton1(self,event):
        print("send clickButton1")
        self.sendMessage("clickButton1")
        return "break"

    def clickButton2(self,event):
        print("send clickButton2")
        self.sendMessage("clickButton2")
        return "break"

    def clickAlertOn(self,event):
        print("send clickAlertOn")
        self.sendMessage("clickAlertOn")
        return "break"

    def clickAlertOff(self,event):
        print("send clickAlertOff")
        self.sendMessage("clickAlertOff")
        return "break"

    def clickButton3(self,event):
        exit()
        return "break"

    def start(self):
        self.main_window.mainloop()
        return

def main():
    W = GUI()
    W.start()

if __name__ == "__main__":
    main()


import pandas as pd
from datasets import load_dataset
from apirequest import summary_api_request
import re
from graph import get_overall_graph,get_all_graphs
from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction

results = []

def calculate_rouge_scores(hypothesis, reference):
    # Calculate ROUGE-2 and ROUGE-3 scores
    smoothing = SmoothingFunction().method4  # Smoothing function for short sentences
    rouge2_s = corpus_bleu([[reference]], [hypothesis], weights=(0.5, 0.5), smoothing_function=smoothing)
    rouge3_s = corpus_bleu([[reference]], [hypothesis], weights=(1 / 3, 1 / 3, 1 / 3), smoothing_function=smoothing)
    return rouge2_s, rouge3_s

dataset = load_dataset("cnn_dailymail", '3.0.0')
train_data = dataset['train']
first_example = train_data[2]

for i in range(300):
    example = train_data[i]
    article = example["article"]
    highlights = example["highlights"]
    total_sentences = len(re.split(r'(?<=[.!?])\s', highlights))
    json_response = summary_api_request(article, length=total_sentences)

    rouge2_score, rouge3_score = calculate_rouge_scores(json_response["TextRank"], highlights)
    lsa_rouge2, lsa_rouge3 = calculate_rouge_scores(json_response["LSA"], highlights)
    relevance_rouge2, relevance_rouge3 = calculate_rouge_scores(json_response["Relevance"], highlights)

    results.append({
        "Index": i,
        "TextRank ROUGE-2": rouge2_score,
        "TextRank ROUGE-3": rouge3_score,
        "LSA ROUGE-2": lsa_rouge2,
        "LSA ROUGE-3": lsa_rouge3,
        "Relevance ROUGE-2": relevance_rouge2,
        "Relevance ROUGE-3": relevance_rouge3
    })

df = pd.DataFrame(results)
df.to_csv('results.csv', index=False)

get_overall_graph(df)
get_all_graphs(df)


from unittest import TestCase
from numpy import loadtxt
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

class TestFinetune_class(TestCase):
    def test_finetune_class(self):
        from build import finetune_class
        dataset = loadtxt('./data/diabetes.csv', delimiter=',', skiprows=1)
        X = dataset[:, 0:8]
        y = dataset[:, 8]
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=13)

        param_grid = {"criterion": ['gini', 'entropy'],
                      "n_estimators": [10, 20, 30],
                      "max_depth": [None, 6, 8, 10],
                      "max_leaf_nodes": [None, 5, 10, 20],
                      "min_impurity_split": [0.1, 0.2, 0.3]}

        y_pred, _ = finetune_class(X_train, X_test, y_train, param_grid)
        acc = accuracy_score(y_pred, y_test)

        self.assertGreaterEqual(acc, 0.7)


# -*- coding: utf-8 -*-
"""
Created on Mon Mar  8 19:32:41 2022
@author: Aakash Tuteja
"""
# Initalise a Default Dict
from collections import OrderedDict
d = OrderedDict()

# Read inputs into Dict
m = int(input())

# create dictionary
for i in range(m):
    temp = input().split(' ')
    num = int(temp[-1])
    del temp[-1]
    key = ' '.join(temp)
    try:
        num += d[key]
        d[key] = num
    except:
        d[key] = num

# Print the output
for a in d:
    print('{} {}'.format(a, d[a]))


from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.shortcuts import render, get_object_or_404
from django.urls import reverse
from webargs.fields import Str
from webargs.djangoparser import use_args

from teachers.forms import CreateTeacherForm, UpdateTeacherForm, FilterTeacherForm
from teachers.models import Teacher

def index(request):
    return HttpResponse('Welcome to LMS!')

@use_args(
    {
        'first_name': Str(required=False),
        'last_name': Str(required=False),
    },
    location='query',
)
def get_teachers(request, args):
    teachers = Teacher.objects.all().order_by('birth_date')
    filter_form = FilterTeacherForm(data=request.GET, queryset=teachers)
    return render(
        request=request,
        template_name='teachers/list.html',
        context={
            'filter_form': filter_form,
        }
    )

def detail_teacher(request, pk):
    teacher = get_object_or_404(Teacher, pk=pk)
    return render(request, 'teachers/detail.html', {'title': 'Detail of teacher', 'teacher': teacher})

def create_teacher_view(request):
    if request.method == 'GET':
        form = CreateTeacherForm()
    elif request.method == 'POST':
        form = CreateTeacherForm(request.POST)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(reverse('teachers:list'))

    return render(request, 'teachers/create.html', {'form': form})

def update_teacher(request, pk):
    teacher = get_object_or_404(Teacher, pk=pk)
    if request.method == 'GET':
        form = UpdateTeacherForm(instance=teacher)
    elif request.method == 'POST':
        form = UpdateTeacherForm(request.POST, instance=teacher)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(reverse('teachers:list'))
    return render(request, 'teachers/update.html', {'form': form})

    return HttpResponse(html_form)

def delete_teacher(request, pk):
    st = get_object_or_404(Teacher, pk=pk)
    if request.method == 'POST':
        st.delete()
        return HttpResponseRedirect(reverse('teachers:list'))
    if request.method == 'GET':
        return render(request, 'teachers/delete.html', {'teacher': st})