import pygame, random, time
import gym
from gym import spaces
import numpy as np

'''
    Функция получения выжившей популяции
        Входные параметры:
        - popul - наша популяция
        - val - текущие значения
        - nsurv - количество выживших
        - reverse - указываем требуемую операцию поиска результата: максимизация или минимизация
'''
agent_size = 9
def getSurvPopul(
        popul,
        val,
        nsurv,
        reverse
):
    newpopul = []  # Двумерный массив для новой популяции

    for i in range(len(val)):
        if val[i] > 0:
            val[i] = val[i] + np.random.random() * 0.001

    sval = sorted(val, reverse=reverse)  # Сортируем зачения в val в зависимости от параметра reverse

    for i in range(nsurv):  # Проходимся по циклу nsurv-раз (в итоге в newpopul запишется nsurv-лучших показателей)
        if sval[i] == 0:
            continue
        index = val.index(sval[i])  # Получаем индекс i-того элемента sval в исходном массиве val
        newpopul.append(popul[index])  # В новую папуляцию добавляем элемент из текущей популяции с найденным индексом
    while len(newpopul) < nsurv:
        bot = np.random.random((agent_size, 3))
        newpopul.append(bot)

    return newpopul, sval  # Возвращаем новую популяцию (из nsurv элементов) и сортированный список

'''
    Функция получения родителей
        Входные параметры:
        - curr_popul - текущая популяция
        - nsurv - количество выживших
'''

def getParents(
        curr_popul,
        nsurv
):
    indexp1 = random.randint(0, nsurv - 1)  # Случайный индекс первого родителя в диапазоне от 0 до nsurv - 1
    indexp2 = random.randint(0, nsurv - 1)  # Случайный индекс второго родителя в диапазоне от 0 до nsurv - 1
    botp1 = curr_popul[indexp1]  # Получаем первого бота-родителя по indexp1
    botp2 = curr_popul[indexp2]  # Получаем второго бота-родителя по indexp2

    return botp1, botp2  # Возвращаем обоих полученных ботов

'''
    Функция смешивания (кроссинговера) двух родителей
        Входные параметры:
        - botp1 - первый бот-родитель
        - botp2 - второй бот-родитель
        - j - номер компонента бота
'''

def crossPointFrom2Parents(
        botp1,
        botp2,
        j
):
    pindex = np.random.random()  # Получаем случайное число в диапазоне от 0 до 1

    # Если pindex меньше 0.5, то берем значения от первого бота, иначе от второго
    if pindex < 0.5:
        x = botp1[j]
    else:
        x = botp2[j]
    return x  # Возвращаем значние бота

class Snake(gym.Env):
    """Custom Environment that follows gym interface"""
    metadata = {'render.modes': ['human']}

    def __init__(self, width=50, thickness=10, bot=None):
        super(Snake, self).__init__()
        # necessary internal gym things
        self.id = "Snake"
        self.reward_threshold = 0.0
        self.trials = 100
        self.max_episode_steps = 2000
        self.mode = "auto"
        self.current_step = 0
        self.energy = 50
        self.score = 0

        # make it square for the sake of defining the state space
        self.height = width
        self.possible_actions = ['LEFT', 'FORWARD', 'RIGHT']
        # Define action and observation spaces following openai standard
        self.action_space = spaces.Discrete(len(self.possible_actions))
        # Define other internal variables
        self.width = width
        self.thickness = thickness
        self.gameMatrix = np.zeros((self.width, self.width))
        self.agent = bot
        self.pygame = pygame
        self.pygame.init()
        self.fpsController = self.pygame.time.Clock()
        # np.zeros((self.width, len(self.possible_actions)))
        # whether it should render anything
        # self.display = display
        # Important varibles
        self.reset()

        # Colors
        self.red = pygame.Color(255, 0, 0)  # gameover
        self.green = pygame.Color(0, 255, 0)  # snake
        self.black = pygame.Color(0, 0, 0)  # score
        self.white = pygame.Color(255, 255, 255)  # background
        self.brown = pygame.Color(165, 42, 42)  # food
        self.yellow = pygame.Color(255, 255, 0)
        self.blue = pygame.Color(0, 0, 255)

    def close_render(self):
        self.pygame.quit()

    def get_states(self):

        isfood_forward = 0.0
        isfood_left = 0.0
        isfood_right = 0.0
        cellfree_left = 1.0
        cellfree_right = 1.0
        cellfree_forward = 1.0
        cell2free_left = 1.0
        cell2free_right = 1.0
        cell2free_forward = 1.0

        if self.direction == 'RIGHT':
            if self.snakePos[0] < self.foodPos[0]:
                isfood_forward = 1.0
            elif self.snakePos[0] > self.foodPos[0]:
                isfood_right = 1.0
                isfood_left = 1.0
            if self.snakePos[1] < self.foodPos[1]:
                isfood_right = 1.0
            elif self.snakePos[1] > self.foodPos[1]:
                isfood_left = 1.0
            if self.snakePos[0] + 1 >= self.height:
                cellfree_forward = 0.0
            else:
                cellfree_forward = self.gameMatrix[self.snakePos[0] + 1][self.snakePos[1]]
            if self.snakePos[1] - 1 < 0:
                cellfree_left = 0.0
            else:
                cellfree_left = self.gameMatrix[self.snakePos[0]][self.snakePos[1] - 1]
            if self.snakePos[1] + 1 >= self.height:
                cellfree_right = 0.0
            else:
                cellfree_right = self.gameMatrix[self.snakePos[0]][self.snakePos[1] + 1]
            if self.snakePos[0]+2 >= self.height:
                cell2free_forward = 0.0
            else:
                cell2free_forward = self.gameMatrix[self.snakePos[0]+2][self.snakePos[1]]
            if self.snakePos[1] - 2 < 0:
                cell2free_left = 0.0
            else:
                cell2free_left = self.gameMatrix[self.snakePos[0]][self.snakePos[1] - 2]
            if self.snakePos[1] + 2 >= self.height:
                cell2free_right = 0.0
            else:
                cell2free_right = self.gameMatrix[self.snakePos[0]][self.snakePos[1] + 2]

        if self.direction == 'LEFT':
            if self.snakePos[0] > self.foodPos[0]:
                isfood_forward = 1.0
            else:
                isfood_right = 1.0
                isfood_left  = 1.0
            if self.snakePos[1] > self.foodPos[1]:
                isfood_right = 1.0
            elif self.snakePos[1] < self.foodPos[1]:
                isfood_left = 1.0
            if self.snakePos[0] - 1 < 0:
                cellfree_forward = 0.0
            else:
                cellfree_forward = self.gameMatrix[self.snakePos[0]-1][self.snakePos[1]]
            if self.snakePos[1] + 1 >= self.height:
                cellfree_left = 0.0
            else:
                cellfree_left = self.gameMatrix[self.snakePos[0]][self.snakePos[1] + 1]
            if self.snakePos[1] - 1 < 0:
                cellfree_right = 0.0
            else:
                cellfree_right = self.gameMatrix[self.snakePos[0]][self.snakePos[1] - 1]
            if self.snakePos[0] - 2 < 0:
                cell2free_forward = 0.0
            else:
                cell2free_forward = self.gameMatrix[self.snakePos[0]-2][self.snakePos[1]]
            if self.snakePos[1] + 2 >= self.height:
                cell2free_left = 0.0
            else:
                cell2free_left = self.gameMatrix[self.snakePos[0]][self.snakePos[1] + 2]
            if self.snakePos[1] - 2 < 0:
                cell2free_right = 0.0
            else:
                cell2free_right = self.gameMatrix[self.snakePos[0]][self.snakePos[1] - 2]

        if self.direction == 'UP':
            if self.snakePos[1] > self.foodPos[1]:
                isfood_forward = 1.0
            else:
                isfood_right = 1.0
                isfood_left  = 1.0
            if self.snakePos[0] < self.foodPos[0]:
                isfood_right = 1.0
            elif self.snakePos[0] > self.foodPos[0]:
                isfood_left = 1.0
            if self.snakePos[1] - 1 < 0:
                cellfree_forward = 0.0
            else:
                cellfree_forward = self.gameMatrix[self.snakePos[0]][self.snakePos[1]-1]
            if self.snakePos[0] - 1 < 0:
                cellfree_left = 0.0
            else:
                cellfree_left = self.gameMatrix[self.snakePos[0] - 1][self.snakePos[1]]
            if self.snakePos[0] + 1 >= self.height:
                cellfree_right = 0.0
            else:
                cellfree_right = self.gameMatrix[self.snakePos[0] + 1][self.snakePos[1]]
            if self.snakePos[1] - 2 < 0:
                cell2free_forward = 0.0
            else:
                cell2free_forward = self.gameMatrix[self.snakePos[0]][self.snakePos[1]-2]
            if self.snakePos[0] - 2 < 0:
                cell2free_left = 0.0
            else:
                cell2free_left = self.gameMatrix[self.snakePos[0] - 2][self.snakePos[1]]
            if self.snakePos[0] + 2 >= self.height:
                cell2free_right = 0.0
            else:
                cell2free_right = self.gameMatrix[self.snakePos[0] + 2][self.snakePos[1]]

        if self.direction == 'DOWN':
            if self.snakePos[1] < self.foodPos[1]:
                isfood_forward = 1.0
            else:
                isfood_right = 1.0
                isfood_left  = 1.0
            if self.snakePos[0] > self.foodPos[0]:
                isfood_right = 1.0
            elif self.snakePos[0] < self.foodPos[0]:
                isfood_left = 1.0
            if self.snakePos[1] + 1 >= self.height:
                cellfree_forward = 0.0
            else:
                cellfree_forward = self.gameMatrix[self.snakePos[0]][self.snakePos[1]+1]
            if self.snakePos[0] + 1 >= self.height:
                cellfree_left = 0.0
            else:
                cellfree_left = self.gameMatrix[self.snakePos[0] + 1][self.snakePos[1]]
            if self.snakePos[0] - 1 < 0:
                cellfree_right = 0.0
            else:
                cellfree_right = self.gameMatrix[self.snakePos[0] - 1][self.snakePos[1]]
            if self.snakePos[1] + 2 >= self.height:
                cell2free_forward = 0.0
            else:
                cell2free_forward = self.gameMatrix[self.snakePos[0]][self.snakePos[1]+2]
            if self.snakePos[0] + 2 >= self.height:
                cell2free_left = 0.0
            else:
                cell2free_left = self.gameMatrix[self.snakePos[0] + 2][self.snakePos[1]]
            if self.snakePos[0] - 2 < 0:
                cell2free_right = 0.0
            else:
                cell2free_right = self.gameMatrix[self.snakePos[0] - 2][self.snakePos[1]]

        state_dict = np.array([isfood_left,
                               isfood_forward,
                               isfood_right,
                               cellfree_left,
                               cellfree_right,
                               cellfree_forward,
                               cell2free_left,
                               cell2free_right,
                               cell2free_forward
                                ])

        return state_dict

    def render_game_over(self):
        if self.game_over:
            myFont = self.pygame.font.SysFont('monaco', self.height)
            GOsurf = myFont.render('Game over!', True, self.red)
            GOrect = GOsurf.get_rect()
            GOrect.midtop = ((self.thickness * self.width / 2), 1.5 * self.thickness)
            self.playSurface.blit(GOsurf, GOrect)
            self.render_score(0)
            self.pygame.display.flip()
            time.sleep(4)
            self.close_render()

    def render_score(self, choice=1):
        sFont = self.pygame.font.SysFont('monaco', 24)
        Ssurf = sFont.render('Score : {0}'.format(self.score), True, self.black)
        Srect = Ssurf.get_rect()
        if choice == 1:
            Srect.midtop = (8 * self.thickness, self.thickness)
        else:
            Srect.midtop = (self.thickness * (self.width / 2), 12 * self.thickness)
        self.playSurface.blit(Ssurf, Srect)

    def get_pressed_key(self):
        ''' Gets the key that was pressed by the user
        '''
        change_to = 'FORWARD'  # self.direction
        for event in self.pygame.event.get():
            if event.type == self.pygame.QUIT:
                self.game_over = True
            elif event.type == self.pygame.KEYDOWN:
                if event.key == self.pygame.K_right or event.key == ord('d'):
                    change_to = 'RIGHT'
                if event.key == self.pygame.K_left or event.key == ord('a'):
                    change_to = 'LEFT'
                if event.key == self.pygame.K_ESCAPE:
                    self.pygame.event.post(self.pygame.event.Event(self.pygame.QUIT))
        return change_to

    def move_snake(self, change_to):
        ''' Move the snake
        '''
        self.spawn_food()
        wasChanged = False
        if not wasChanged and change_to == 'RIGHT' and self.direction == 'UP':
            self.direction = 'RIGHT'
            wasChanged = True
        if not wasChanged and change_to == 'RIGHT' and self.direction == 'RIGHT':
            self.direction = 'DOWN'
            wasChanged = True
        if not wasChanged and change_to == 'RIGHT' and self.direction == 'DOWN':
            self.direction = 'LEFT'
            wasChanged = True
        if not wasChanged and change_to == 'RIGHT' and self.direction == 'LEFT':
            self.direction = 'UP'
            wasChanged = True
        if not wasChanged and change_to == 'LEFT' and self.direction == 'UP':
            self.direction = 'LEFT'
            wasChanged = True
        if not wasChanged and change_to == 'LEFT' and self.direction == 'LEFT':
            self.direction = 'DOWN'
            wasChanged = True
        if not wasChanged and change_to == 'LEFT' and self.direction == 'DOWN':
            self.direction = 'RIGHT'
            wasChanged = True
        if not wasChanged and change_to == 'LEFT' and self.direction == 'RIGHT':
            self.direction = 'UP'
            wasChanged = True

        # Update snake position [x,y]
        if self.direction == 'RIGHT':
            self.snakePos[0] += 1  # self.thickness
        if self.direction == 'LEFT':
            self.snakePos[0] -= 1  # self.thickness
        if self.direction == 'UP':
            self.snakePos[1] -= 1  # self.thickness
        if self.direction == 'DOWN':
            self.snakePos[1] += 1  # self.thickness

        # Snake body mechanism
        self.snakeBody.insert(0, list(self.snakePos))
        if self.is_in_food_position():
            self.score += 1
            self.energy += 25
            self.foodSpawn = False
            # test not growing snake
        #             self.snakeBody.pop()
        else:
            self.snakeBody.pop()
        self.check_game_over()

    def is_in_food_position(self):
        return self.snakePos[0] == self.foodPos[0] and self.snakePos[1] == self.foodPos[1]

    def spawn_food(self):
        # Food Spawn
        if self.foodSpawn == False:
            self.foodPos = [random.randrange(1, self.height - 1),  # *self.thickness,
                            random.randrange(1, self.width - 1)  # *self.thickness
                            ]
        self.foodSpawn = True

    def check_game_over(self):
        # Bound
        if self.snakePos[0] > (self.height - 1) or self.snakePos[0] < 0:
            self.game_over = True
            self.score -= 5
        if self.snakePos[1] > (self.width - 1) or self.snakePos[1] < 0:
            self.game_over = True
            self.score -= 5
        # Self hit
        for block in self.snakeBody[1:]:
            if self.snakePos[0] == block[0] and self.snakePos[1] == block[1]:
                self.game_over = True
                self.score -= 5
        self.energy -= 1
        if self.current_step > self.max_episode_steps or self.energy <= 0:
            self.game_over = True

    def play_game(self):
        #self.init_interface()
        while self.game_over == False:
            #direction = self.get_pressed_key()
            direction = self.get_direction_from_GA()
            self.move_snake(direction)
            if self.mode != "auto":
                self.render()
        # if self.mode != "auto":
        #    self.render_game_over()

    def reset(self):
        ''' reset method for openai
        '''
        self.snakePos = [10, 5]
        self.snakeBody = [[10, 5], [9, 5], [8, 5]]
        self.foodSpawn = False
        self.spawn_food()
        self.game_over = False
        self.direction = 'RIGHT'
        self.energy = 50
        self.score = 0
        self.current_step = 0
        self.setFPSTick(5000)

    def setFPSTick(self, FPS):
        self.fpsController = self.pygame.time.Clock()
        self.fpsController.tick(FPS)

    def render(self, mode='human', close=False):
        ''' Render method for openai
        '''
        self.init_interface()
        # Background
        self.playSurface.fill(self.yellow)
        # Draw Snake
        firstElement = True
        for pos in self.snakeBody:
            if firstElement:
                self.pygame.draw.rect(self.playSurface, self.blue,
                                      self.pygame.Rect(self.thickness * pos[0], self.thickness * pos[1], self.thickness,
                                                       self.thickness))
                firstElement = False
            else:
                self.pygame.draw.rect(self.playSurface, self.green,
                                      self.pygame.Rect(self.thickness * pos[0], self.thickness * pos[1], self.thickness,
                                                       self.thickness))

        self.pygame.draw.rect(self.playSurface, self.brown,
                              self.pygame.Rect(self.thickness * self.foodPos[0], self.thickness * self.foodPos[1],
                                               self.thickness, self.thickness))
        self.render_score()
        self.pygame.display.flip()
        # self.draw_toColab()

    def init_interface(self):
        # Play surface
        self.playSurface = self.pygame.display.set_mode((self.height * self.thickness, self.width * self.thickness))
        self.pygame.display.set_caption('Snake game!')

    def updateGameMatrix(self):
        for i in range(self.gameMatrix.shape[0]):
            for j in range(self.gameMatrix.shape[1]):
                self.gameMatrix[i][j] = 0.75
        # update snake pos
        for i in self.snakeBody:
            self.gameMatrix[i[0]][i[1]] = 0.40;
        self.gameMatrix[self.snakePos[0]][self.snakePos[1]] = 0.00
        self.gameMatrix[self.foodPos[0]][self.foodPos[1]] = 1.0

    def get_direction_from_GA(self):
        self.updateGameMatrix()
        states = self.get_states()
        # result = np.dot(self.gameMatrix, self.agent)
        # result = np.sum(result, axis=0)
        # result = np.dot(result, np.ones((20,3)))
        result = np.dot(states, self.agent)
        action = np.argmax(result)
        return self.possible_actions[action]

if __name__ == '__main__':
    numBots = 40
    popul = []  # здесь будет лежать популяция
    reward_list = []  # здесь будет сумма вознаграждений для каждого эпизода
    for i in range(numBots):
        bot = np.random.random((agent_size, 3))
        popul.append(bot)
    nsurv = 20  # количество выживших
    nnew = numBots - nsurv  # количество новых
    epohs = 500  # количество эпох
    mut = 0.01  # коэфициент мутаций
    mutStrength = 1e-1
    curr_time = time.time()
    snake_game = Snake(width=20, thickness=20)
    snake_game.init_interface()

    for it in range(epohs):  # создали список списков всех значений по эпохам
        reward_list = []
        showGame = False
        for bot in popul:
            snake_game.agent = bot
            snake_game.reset()
            if showGame:
                snake_game.mode = 'render'
                snake_game.setFPSTick(FPS=1)
                showGame = False
            else:
                snake_game.mode = 'auto'
            snake_game.play_game()
            reward_list.append(snake_game.score)
        newpopul, sval = getSurvPopul(popul = popul, val = reward_list, nsurv = nsurv, reverse=1)  # получили популяцию выживших, нас интересует бот с максимальным успехом, поэтому reverse = 1
        print(it, time.time() - curr_time, " ", sval[0:3])  # Выводим время на операцию, среднее значение и 20 лучших ботов
        # проходимся по новой популяции
        for k in range(nnew):
            # вытаскиваем новых родителей
            botp1, botp2 = getParents(newpopul, nsurv)
            newbot = []  # здесь будет новый бот
            for j in range(len(botp1)):  # боты-родители одинаковой длины, будем проходиться по каждому элементу родителя
                x = crossPointFrom2Parents(botp1, botp2, j)  # скрещиваем
                for t in range(x.shape[0]):
                    if random.random() < mut:
                        if random.random() < 0.5:
                            x[t] -= random.random() * mutStrength
                        else:
                            x[t] += random.random() * mutStrength
                newbot.append(x)  # закидываем элемент в бота
            newpopul.append(newbot)  # добавляем бота
        popul = newpopul # вывести список на эпоху
        #popul = np.array(popul)  # для того, чтобы можно было легко вытащить индексы условием, преобразуем в numpy массив


#使用线程库
import threading
#队列
from queue import Queue
#解析库
from lxml import etree
import json
import jsonpath
import time
import requests

exitFlag_Parser = False
lock = threading.Lock()

class ThreadCrawl(threading.Thread):
    def __init__(self,threadName,pageQueue,dataQueue):
        #调用父类初始化 第一个参数自己类的名字，第二个参数self
        super(ThreadCrawl,self).__init__()
        #线程名
        self.threadName = threadName
        #页码队列
        self.pageQueue = pageQueue
        #数据队列
        self.dataQueue = dataQueue
        #头信息
        self.headers = {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
                    'Accept-Language': 'zh-CN,zh;q=0.8'
        }

    #线程执行
    def run(self):
        #队列为空循环终止
        print("启动"+self.threadName)
        while not self.pageQueue.empty():
            try:
                #取出1个页面
                #可选参数block,默认值是true
                #1.如果队列为空,block为true，就会进入阻塞状态，直到队列有新数据
                #2.如果队列为空，block为false,就会弹出1个Queue.empty()异常
                page = self.pageQueue.get(False)
                url = "https://www.qiushibaike.com/8hr/page/" + str(page) +"/"
                content = requests.get(url,headers = self.headers)
                self.dataQueue.put(content.text)
            except:
                pass
        print("结束"+self.threadName)

#解析线程
class ThreadParse(threading.Thread):
    def __init__(self,threadName,dataQueue,filename):
        super(ThreadParse,self).__init__()
         #线程名
        self.threadName = threadName
        #数据解析队列
        self.dataQueue = dataQueue
        #文件名
        self.f = filename

    def run(self):
        global exitFlag_Parser
        while not exitFlag_Parser:
            try:
                html = self.dataQueue.get(False)
                self.parse(html)
            except:
                pass

    #解析类
    def parse(self,html):
        html = etree.HTML(html)
        #查找根节点下 id属性包含qiushi_tag字符的div节点 -> @id属性id ， contains包含
        result = html.xpath('//div[contains(@id,"qiushi_tag")]')

        for site in result:

            #当前节点下的div下的a下的img节点的属性src
            imgUrl  = site.xpath('./div/a/img/@src')[0]
            username = site.xpath('.//h2')[0].text
            #获取标签里面的所有text   ab
cd
   .xpath('string(.)').strip() -> abcd
            #如果单纯使用 text 只能获得ab
            content = site.xpath('.//div[@class="content"]/span')[0].xpath('string(.)').strip()

            # 投票次数
            vote = site.xpath('.//i')[0].text
            # 评论数
            comments = site.xpath('.//i')[1].text

            result = {
                    'imgUrl': imgUrl,
                    'username': username,
                    'content': content,
                    'vote': vote,
                    'comments': comments,
            }

            result = json.dumps(result, ensure_ascii=False)
            global lock
            #获取锁
            lock.acquire()
            self.f.write(result+"\n")
            #释放锁
            lock.release()

def main():
    #存放页数的队列,10表示10个页面
    pageQueue = Queue(10)
    # 放入1-10个数字，先进先出
    for i in range(1,11):
        pageQueue.put(i)

    #采集数据结果队列.参数为空表示无限制
    dataQueue = Queue()

    filename = open("duanzi.json","a")

    #采集线程的名字
    crawlList = ['采集线程1号','采集线程2号','采集线程3号']

    #存储三个采集线程
    threadcrawl = []
    #创建线程
    for threadName in crawlList:
        #创建线程对象
        thread = ThreadCrawl(threadName,pageQueue,dataQueue)
        thread.start()
        threadcrawl.append(thread)

    #解析线程
    parseList = ['解析线程1号','解析线程2号','解析线程3号']
    threadparse = []

    for threadName in parseList:
        thread = ThreadParse(threadName,dataQueue,filename)
        thread.start()
        threadparse.append(thread)

    #while not pageQueue.empty():
    #    pass

    #子线程加入阻塞 （主要作用的 阻塞掉主线程）
    for thread in threadcrawl:
        thread.join()
        print('x')

    global exitFlag_Parser
    exitFlag_Parser = True

    '''
    for thread in threadparse:
        thread.join()
        print('x')
    '''

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""Constants go here!"""
from typing import Dict

LAGER_PORT = 52437

TORNADO_FMT = "".join(
    [
        "<level>",
        "[{level.name[0]} ",
        "{time:YYMMDDTHH:mm:ss} ",
        "{name}:{module}:{line}]",
        "</level> ",
        "{message}",
    ]
)

LOGURU_DEFAULT_FMT = "".join(
    [
        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>",
        " | ",
        "<level>{level: <8}</level>",
        " | ",
        "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan>",
        " - ",
        "<level>{message}</level>",
    ]
)

LOG_LEVELS: Dict[str, str] = {
    "notset": "NOTSET",
    "n": "NOTSET",
    "debug": "DEBUG",
    "d": "DEBUG",
    "info": "INFO",
    "i": "INFO",
    "s": "SUCCESS",
    "success": "SUCCESS",
    "warning": "WARNING",
    "warn": "WARNING",
    "w": "WARNING",
    "error": "ERROR",
    "e": "ERROR",
    "critical": "CRITICAL",
    "fatal": "CRITICAL",
    "c": "CRITICAL",
    # enum/enum-strings
    "0": "NOTSET",
    "10": "DEBUG",
    "20": "INFO",
    "25": "SUCCESS",
    "30": "WARNING",
    "40": "ERROR",
    "50": "CRITICAL",
}


import pytest
from rest_thingy import Thingy, parse_response

def test_parse_response():
    class Foo:
        @classmethod
        def extract(cls, response):
            cls.extract_called = True
            return response

        @classmethod
        def deserialize(cls, response):
            cls.deserialize_called = True
            return response

        @classmethod
        def bind(cls, response):
            cls.bind_called = True
            return response

        @classmethod
        @parse_response
        def bar(cls):
            return "baz"

    assert Foo.bar() == "baz"
    assert Foo.extract_called
    assert Foo.deserialize_called
    assert Foo.bind_called

def test_thingy_bind_dict():
    thingy = Thingy.bind({"foo": "bar"})
    assert isinstance(thingy, Thingy)
    assert thingy.foo == "bar"
    assert thingy.__dict__ == {"foo": "bar"}

def test_thingy_bind_list():
    thingies = Thingy.bind([{"foo": "bar"}, {"baz": "qux"}])
    assert isinstance(thingies, list)
    assert len(thingies) == 2

    assert isinstance(thingies[0], Thingy)
    assert thingies[0].foo == "bar"
    assert isinstance(thingies[1], Thingy)
    assert thingies[1].baz == "qux"

def test_thingy_bind_unknown():
    with pytest.raises(TypeError):
        Thingy.bind("")


from keras import initializers, regularizers, constraints
from keras.layers import *
import keras.backend as K
from keras import regularizers
from keras import Model
import numpy as np

from keras.engine.topology import Layer

def MultiHeadsAttModel(l=8 * 8, d=512, dv=64, dout=512, nv=8):
    v1 = Input(shape=(l, d))
    q1 = Input(shape=(l, d))
    k1 = Input(shape=(l, d))

    v2 = Dense(dv * nv, activation="relu")(v1)
    q2 = Dense(dv * nv, activation="relu")(q1)
    k2 = Dense(dv * nv, activation="relu")(k1)

    v = Reshape([l, nv, dv])(v2)
    q = Reshape([l, nv, dv])(q2)
    k = Reshape([l, nv, dv])(k2)

    att = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[-1, -1]) / np.sqrt(dv),
                 output_shape=(l, nv, nv))([q, k])  # l, nv, nv
    att = Lambda(lambda x: K.softmax(x), output_shape=(l, nv, nv))(att)

    out = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[4, 3]), output_shape=(l, nv, dv))([att, v])
    out = Reshape([l, d])(out)

    out = Add()([out, q1])

    out = Dense(dout, activation="relu")(out)

    return Model(inputs=[q1, k1, v1], outputs=out)

class NormL(Layer):

    def __init__(self, **kwargs):
        super(NormL, self).__init__(**kwargs)

    def build(self, input_shape):
        # Create a trainable weight variable for this layer.
        self.a = self.add_weight(name='kernel',
                                      shape=(1,input_shape[-1]),
                                      initializer='ones',
                                      trainable=True)
        self.b = self.add_weight(name='kernel',
                                      shape=(1,input_shape[-1]),
                                      initializer='zeros',
                                      trainable=True)
        super(NormL, self).build(input_shape)  # Be sure to call this somewhere!

    def call(self, x):
        eps = 0.000001
        mu = K.mean(x, keepdims=True, axis=-1)
        sigma = K.std(x, keepdims=True, axis=-1)
        ln_out = (x - mu) / (sigma + eps)
        return ln_out*self.a + self.b

    def compute_output_shape(self, input_shape):
        return input_shape

def zero_loss(y_true, y_pred):
    return 0.5 * K.sum(y_pred, axis=0)
class CenterLossLayer(Layer):

    def __init__(self, alpha=0.5, **kwargs):
        super().__init__(**kwargs)
        self.alpha = alpha

    def build(self, input_shape):
        self.centers = self.add_weight(name='centers',
                                       shape=(19, 1024),
                                       initializer='uniform',
                                       trainable=False)
        # self.counter = self.add_weight(name='counter',
        #                                shape=(1,),
        #                                initializer='zeros',
        #                                trainable=False)  # just for debugging
        super().build(input_shape)

    def call(self, x, mask=None):
        # x[0] is Nx2, x[1] is Nx10 onehot, self.centers is 10x2
        delta_centers = K.dot(K.transpose(x[1]), (K.dot(x[1], self.centers) - x[0]))  # 10x2
        center_counts = K.sum(K.transpose(x[1]), axis=1, keepdims=True) + 1  # 10x1
        delta_centers /= center_counts
        new_centers = self.centers - self.alpha * delta_centers
        self.add_update((self.centers, new_centers), x)

        # self.add_update((self.counter, self.counter + 1), x)

        self.result = x[0] - K.dot(x[1], self.centers)
        self.result = K.sum(self.result ** 2, axis=1, keepdims=True)  # / K.dot(x[1], center_counts)
        return self.result  # Nx1

    def compute_output_shape(self, input_shape):
        return K.int_shape(self.result)

def residual_block(input, input_channels=None, output_channels=None, kernel_size=(3, 3), stride=1):
    """
    full pre-activation residual block
    https://arxiv.org/pdf/1603.05027.pdf
    """
    if output_channels is None:
        output_channels = input.get_shape()[-1].value
    if input_channels is None:
        input_channels = output_channels // 4

    strides = (stride, stride)

    #x = BatchNormalization(epsilon=0.001, momentum=0.99)(input)

    x = Conv2D(input_channels, kernel_size, padding='same', strides=stride, use_bias=False
               #kernel_regularizer=regularizers.l2(0.0005)
               )(input)
    #x = LeakyReLU(0.01)(x)
    x = BatchNormalization(epsilon=1e-5)(x)
    x = ReLU()(x)
    x = Dropout(0.3)(x)

    #x = BatchNormalization(epsilon=0.001, momentum=0.99)(x)
    x = Conv2D(output_channels, kernel_size, padding='same', strides=stride, use_bias=False,
               #kernel_regularizer=regularizers.l2(0.0005)
               )(x)
    x = BatchNormalization(epsilon=1e-5)(x)
    #x = ReLU()(x)

    if input_channels != output_channels or stride != 1:
        input = Conv2D(output_channels, (1, 1), padding='same', strides=strides, use_bias=False,
                       #kernel_regularizer=regularizers.l2(0.0005)
                       )(input)
        input = BatchNormalization(epsilon=1e-5)(input)

    x = Add()([x, input])
    x = ReLU()(x)
    #x = BatchNormalization()(x)
    x = Dropout(0.3)(x)
    return x

class Attention(Layer):
    def __init__(self, step_dim,
                 W_regularizer=None, b_regularizer=None,
                 W_constraint=None, b_constraint=None,
                 bias=True, **kwargs):
        self.supports_masking = True
        self.init = initializers.get('glorot_uniform')

        self.W_regularizer = regularizers.get(W_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        self.step_dim = step_dim
        self.features_dim = 0
        super(Attention, self).__init__(**kwargs)

    def build(self, input_shape):
        assert len(input_shape) == 3

        self.W = self.add_weight(shape=(input_shape[-1],),
                                 initializer=self.init,
                                 name='{}_W'.format(self.name),
                                 regularizer=self.W_regularizer,
                                 constraint=self.W_constraint)
        self.features_dim = input_shape[-1]

        if self.bias:
            self.b = self.add_weight(shape=(input_shape[1],),
                                     initializer='zero',
                                     name='{}_b'.format(self.name),
                                     regularizer=self.b_regularizer,
                                     constraint=self.b_constraint)
        else:
            self.b = None

        self.built = True

    def compute_mask(self, input, input_mask=None):
        return None

    def call(self, x, mask=None):
        features_dim = self.features_dim
        step_dim = self.step_dim

        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),
                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))

        if self.bias:
            eij += self.b

        eij = K.tanh(eij)

        a = K.exp(eij)

        if mask is not None:
            a *= K.cast(mask, K.floatx())

        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1)

    def compute_output_shape(self, input_shape):
        return input_shape[0],  self.features_dim
# class attentionLayer(Layer):
#     def __init__(self, **kwargs):
#         """"
#         Class-wise attention pooling layer
#                 Args:
#                 Attributes:
#             kernel: tensor
#             bias: tensor
#
#         """
#         super(attentionLayer, self).__init__(**kwargs)
#
#     def build(self, input_shape):
#         kernel_shape = [1] * len(input_shape)
#         bias_shape = tuple([1] * (len(input_shape) - 1))
#
#         kernel_shape[-1] = input_shape[-1]
#         kernel_shape = tuple(kernel_shape)
#
#         self.kernel = self.add_weight(
#             shape=kernel_shape,
#             initializer=Zeros(),
#             name='%s_kernel' % self.name)
#
#         self.bias = self.add_weight(
#             shape=bias_shape,
#             initializer=Zeros(),
#             name='%s_bias' % self.name)
#
#         super(attentionLayer, self).build(input_shape)
#
#     def call(self, inputs):
#         weights = K.sum(inputs * self.kernel, axis=-1) + self.bias
#         return weights
#
#     def compute_output_shape(self, input_shape):
#         out_shape = []
#         for i in range(len(input_shape) - 1):
#             out_shape += [input_shape[i]]
#         return tuple(out_shape)

# def attention_3d_block(hidden_states):
#     """
#     Many-to-one attention mechanism for Keras.
#     @param hidden_states: 3D tensor with shape (batch_size, time_steps, input_dim).
#     @return: 2D tensor with shape (batch_size, 128)
#     @author: felixhao28.
#     """
#     hidden_size = int(hidden_states.shape[2])
#     # Inside dense layer
#     #              hidden_states            dot               W            =>           score_first_part
#     # (batch_size, time_steps, hidden_size) dot (hidden_size, hidden_size) => (batch_size, time_steps, hidden_size)
#     # W is the trainable weight matrix of attention Luong's multiplicative style score
#     score_first_part = Dense(hidden_size, use_bias=False, name='attention_score_vec')(hidden_states)
#     #            score_first_part           dot        last_hidden_state     => attention_weights
#     # (batch_size, time_steps, hidden_size) dot   (batch_size, hidden_size)  => (batch_size, time_steps)
#     h_t = Lambda(lambda x: x[:, -1, :], output_shape=(hidden_size,), name='last_hidden_state')(hidden_states)
#     score = dot([score_first_part, h_t], [2, 1], name='attention_score')
#     attention_weights = Activation('softmax', name='attention_weight')(score)
#     # (batch_size, time_steps, hidden_size) dot (batch_size, time_steps) => (batch_size, hidden_size)
#     context_vector = dot([hidden_states, attention_weights], [1, 1], name='context_vector')
#     pre_activation = concatenate([context_vector, h_t], name='attention_output')
#     attention_vector = Dense(128, use_bias=False, activation='tanh', name='attention_vector')(pre_activation)
#     return attention_vector
#
# class attention(Layer):
#     def __init__(self,**kwargs):
#         super(attention,self).__init__(**kwargs)
#
#     def build(self,input_shape):
#         self.W=self.add_weight(name="att_weight",shape=(input_shape[-1],1),initializer="normal")
#         self.b=self.add_weight(name="att_bias",shape=(input_shape[1],1),initializer="zeros")
#         super(attention, self).build(input_shape)
#
#     def call(self,x):
#         et=K.squeeze(K.tanh(K.dot(x,self.W)+self.b),axis=-1)
#         at=K.softmax(et)
#         at=K.expand_dims(at,axis=-1)
#         output=x*at
#         return K.sum(output,axis=1)
#
#     def compute_output_shape(self,input_shape):
#         return (input_shape[0],input_shape[-1])
#
#     def get_config(self):
#         return super(attention,self).get_config()
#
# def attention_block(input, input_channels=None, output_channels=None, encoder_depth=1):
#     """
#     attention block
#     https://arxiv.org/abs/1704.06904
#     """
#
#     p = 1
#     t = 2
#     r = 1
#
#     if input_channels is None:
#         input_channels = input.get_shape()[-1].value
#     if output_channels is None:
#         output_channels = input_channels
#
#     # First Residual Block
#     for i in range(p):
#         input = residual_block(input)
#
#     # Trunc Branch
#     output_trunk = input
#     for i in range(t):
#         output_trunk = residual_block(output_trunk)
#
#     # Soft Mask Branch
#
#     ## encoder
#     ### first down sampling
#     output_soft_mask = MaxPool2D(padding='same')(input)  # 32x32
#     for i in range(r):
#         output_soft_mask = residual_block(output_soft_mask)
#
#     skip_connections = []
#     for i in range(encoder_depth - 1):
#
#         ## skip connections
#         output_skip_connection = residual_block(output_soft_mask)
#         skip_connections.append(output_skip_connection)
#         # print ('skip shape:', output_skip_connection.get_shape())
#
#         ## down sampling
#         output_soft_mask = MaxPool2D(padding='same')(output_soft_mask)
#         for _ in range(r):
#             output_soft_mask = residual_block(output_soft_mask)
#
#             ## decoder
#     skip_connections = list(reversed(skip_connections))
#     for i in range(encoder_depth - 1):
#         ## upsampling
#         for _ in range(r):
#             output_soft_mask = residual_block(output_soft_mask)
#         output_soft_mask = UpSampling2D()(output_soft_mask)
#         ## skip connections
#         output_soft_mask = Add()([output_soft_mask, skip_connections[i]])
#
#     ### last upsampling
#     for i in range(r):
#         output_soft_mask = residual_block(output_soft_mask)
#     output_soft_mask = UpSampling2D()(output_soft_mask)
#
#     ## Output
#     output_soft_mask = Conv2D(input_channels, (1, 1))(output_soft_mask)
#     output_soft_mask = Conv2D(input_channels, (1, 1))(output_soft_mask)
#     output_soft_mask = Activation('sigmoid')(output_soft_mask)
#
#     # Attention: (1 + output_soft_mask) * output_trunk
#     output = Lambda(lambda x: x + 1)(output_soft_mask)
#     output = Multiply()([output, output_trunk])  #
#
#     # Last Residual Block
#     for i in range(p):
#         output = residual_block(output)
#
#     return output


import serial
import time
ser = serial.Serial('COM5', baudrate = 9600, timeout = 5, xonxoff=True)

if ser.isOpen():
    #do nothing
    print("Serial is open")
else:
    try:
            ser.open()
    except Exception as e:
            print("error opening Serial Port: " + str(e))
            exit()

if ser.isOpen():
    o = "hello serial port\n\r"
    o_out = o.encode()
    ser.write(o_out)
    for x in range(10):
       ser.write(b'\x06')
    ser.write(o_out)
    youcan = "you can stop\n\r"
    ser.write(youcan.encode())
else:
    print("Serial is not open")
ser.close()
print("End")


from bots.abstract_bot import AbstractBot
from bots.action import Action
import sendgrid
from sendgrid.helpers.mail import *
import settings

class MailBot(AbstractBot):
    def __init__(self, id):
        actions = ['send']
        super().__init__(id, actions)
        # REQUIRED
        self.mail_from = None
        self.mail_to = None
        self.subject = None
        self.body = None

    def extract_attr(self, intent):
        if intent.parameters['from']:
            self.mail_from = intent.parameters['from']
        if intent.parameters['to']:
            self.mail_to = intent.parameters['to']
        if intent.parameters['subject']:
            self.subject = intent.parameters['subject']
        if intent.parameters['body']:
            self.body = intent.parameters['body']

    def execute(self):
        sg = sendgrid.SendGridAPIClient(apikey=settings.SENDGRID_ACCESS_TOKEN)
        from_email = Email(self.mail_from)
        to_email = Email(self.mail_to)
        subject = self.subject
        content = Content("text/plain",self.body)
        mail = Mail(from_email, subject, to_email, content)
        response = sg.client.mail.send.post(request_body=mail.get())
        print(response.status_code)
        print(response.body)
        print(response.headers)
        return Action(
            action_type='message',
            body='email sent successfully',
            bot=self.id,
            keep_context=False)

    def is_long_running(self):
        return False

    def has_missing_attr(self):
        return (self.mail_from == None or self.mail_to == None or self.subject == None or self.body == None)

    def request_missing_attr(self):
        if self.mail_from == None:
            return Action(
                action_type='inquiry',
                body='Please specify the sender email address',
                bot=self.id,
                keep_context=True)

        if self.mail_to == None:
            return Action(
                action_type='inquiry',
                body='Please specify the recipient email address',
                bot=self.id,
                keep_context=True)

        if self.subject == None:
            return Action(
                action_type='inquiry',
                body='What is the mail subject?',
                bot=self.id,
                keep_context=True)

        if self.body == None:
            return Action(
                action_type='inquiry',
                body='What is the mail body ?',
                bot=self.id,
                keep_context=True)


import os
import json
import torch
import random
import logging
import argparse
import itertools
import numpy as np
from collections import defaultdict
from torch.utils.data import DataLoader

from transformers import pipeline
from transformers import AdamW, AutoTokenizer, AutoModelForTokenClassification

from seqeval.metrics import classification_report
from dataset import CustomizedDataset

def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)

def get_f1(result_str, category=None):
    if category:
        if category not in result_str:
            return None
        else:

            return result_str.split(category)[1].split('      ')[3]
    else:
        return result_str.split('weighted avg')[1].split('      ')[3]

def encode_tags(tags, encodings, tag2id):
    labels = [[tag2id[tag] for tag in doc] for doc in tags]
    encoded_labels = []
    for doc_labels, doc_offset in zip(labels, encodings.offset_mapping):
        # create an empty array of -100
        doc_enc_labels = np.ones(len(doc_offset), dtype=int) * -100
        arr_offset = np.array(doc_offset)

        # set labels whose first offset position is 0 and the second is not 0
        doc_enc_labels[(arr_offset[:, 0] == 0) & (arr_offset[:, 1] != 0)] = doc_labels
        encoded_labels.append(doc_enc_labels.tolist())

    return encoded_labels

def get_sents(tokens):
    res = []
    # concatenate a list of tokens to sentences
    for sent_t in tokens:
        sent = ''
        for t in sent_t:
            sent += t
            sent += ' '
        res.append(sent.strip())
    return res

def evaluate(val_dataset, device, model, id2tag, categories=None):
    # evaluation
    model.eval()
    print('evaluating...')
    all_labels = []
    all_preds = []

    val_loader = DataLoader(val_dataset, batch_size=1, shuffle=False)
    for batch in val_loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        all_labels.append(labels.squeeze(0).tolist())
        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        logits = outputs.logits
        pred = torch.argmax(logits, dim=-1)
        all_preds.append(pred.squeeze(0).tolist())
    # model.train()
    labels_truncated = []
    preds_truncated = []
    # dealing with the ignored preds
    for labels, preds in zip(all_labels, all_preds):
        if len(labels) != len(preds):
            print('bad example')
            continue
        arr_labels = np.array(labels)
        preds_truncated.append(np.array(preds)[arr_labels != -100].tolist())
        labels_truncated.append(arr_labels[arr_labels != -100].tolist())

    val_labels = [[id2tag[tid] for tid in sent] for sent in labels_truncated]
    all_preds = [[id2tag[tid] for tid in sent] for sent in preds_truncated]

    result_str = classification_report(val_labels, all_preds)
    if categories:
        for key in categories.keys():
            categories[key].append(get_f1(result_str, key))
        return get_f1(result_str), result_str, categories
    else:
        return get_f1(result_str), result_str

def get_data_labels(data_folder, p_names):
    p_data = []
    p_labels = []
    # load data
    for name in p_names:
        with open(data_folder + '{}_dish.json'.format(name), 'r') as f:
            output = json.load(f)
        data = get_sents(output['data'])
        # get labels
        with open(data_folder + '{}_labels_mapped_dish.json'.format(name), 'r') as f:
            labels = json.load(f)
        p_data.append(data)
        p_labels.append(labels)
    return p_data, p_labels

def load_data(data_folder, name):
    with open(data_folder + '{}_dish_train.json'.format(name), 'r') as f:
        train_data = json.load(f)
    with open(data_folder + '{}_dish_test.json'.format(name), 'r') as f:
        test_data = json.load(f)
    with open(data_folder + '{}_dish_train_labels.json'.format(name), 'r') as f:
        train_labels = json.load(f)
    with open(data_folder + '{}_dish_test_labels.json'.format(name), 'r') as f:
        test_labels = json.load(f)
    return train_data, train_labels, test_data, test_labels

def get_tag2id(labels):
    unique_tags = set(tag for doc in labels for tag in doc)
    tag2id = {tag: id for id, tag in enumerate(unique_tags)}
    id2tag = {id: tag for tag, id in tag2id.items()}
    return tag2id, id2tag

def trainer(train_data, train_labels, test_data, test_labels, weighted_f1s_categories,
            tag_id_maps, num_epoch=10, batch_size=32, model_name="dslim/bert-base-NER", evaluate_only=False):
    batch_size = batch_size
    num_epoch = num_epoch
    tag2id, id2tag = tag_id_maps
    # we fine-tune the model on train data and evaluate on test set
    tokenizer = AutoTokenizer.from_pretrained(model_name)

    model = AutoModelForTokenClassification.from_pretrained(model_name, num_labels=len(tag2id),
                                                            ignore_mismatched_sizes=True)
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    model.to(device)
    tag2id_model = model.config.label2id
    for tag, id in tag2id.items():
        if tag not in tag2id_model.keys():
            tag2id_model[tag] = len(tag2id_model)
    tag2id = tag2id_model
    id2tag = {id: tag for tag, id in tag2id.items()}

    train_encodings = tokenizer(train_data, is_split_into_words=True, return_offsets_mapping=True, padding=True,
                                truncation=True)
    test_encodings = tokenizer(test_data, is_split_into_words=True, return_offsets_mapping=True, padding=True,
                              truncation=True)

    train_labels = encode_tags(train_labels, train_encodings, tag2id)
    test_labels = encode_tags(test_labels, test_encodings, tag2id)

    train_encodings.pop("offset_mapping")  # we don't want to pass this to the model
    test_encodings.pop("offset_mapping")
    train_dataset = CustomizedDataset(train_encodings, train_labels)
    test_dataset = CustomizedDataset(test_encodings, test_labels)
    print("test size:", len(test_dataset))
    logging.info('test size: ' + str(len(test_dataset)))

    if evaluate_only:
        model.eval()
        # # for debugging:
        # res = []
        # while True:
        #     outputs = evaluate(test_dataset, device, model, id2tag, weighted_f1s_categories)
        #     res.append(outputs)
        weighted_f1, result_str, weighted_f1s_categories = evaluate(test_dataset, device, model, id2tag, weighted_f1s_categories)
        print(result_str)
        logging.info(result_str)
        return weighted_f1, result_str, weighted_f1s_categories

    else:
        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False)
        model.train()

        optim = AdamW(model.parameters(), lr=5e-5)

        for epoch in range(num_epoch):
            print('Epoch ', epoch)
            epoch_loss = 0
            for batch in train_loader:
                optim.zero_grad()
                input_ids = batch['input_ids'].to(device)
                attention_mask = batch['attention_mask'].to(device)
                labels = batch['labels'].to(device)
                outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
                loss = outputs[0]
                # print('Loss: ', loss.item())
                epoch_loss += loss
                loss.backward()
                optim.step()
            print('Epoch loss: ', epoch_loss)

        weighted_f1, result_str, weighted_f1s_categories = evaluate(test_dataset, device, model, id2tag, weighted_f1s_categories)
        # TODO: dump all_reports to log file
        print(result_str)
        logging.info(result_str)
        return weighted_f1, result_str, weighted_f1s_categories

if __name__ == '__main__':
    DEBUG = False
    # This script is used for measure the performance and
    # calculate the performance difference for any pair of datasets
    logging.basicConfig(filename='performance_on_all_no_ft',
                        filemode='a',
                        format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
                        datefmt='%H:%M:%S',
                        level=logging.DEBUG)

    # Initialize parser
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input_folder', help='Path to input folder')
    parser.add_argument('-o', '--output_folder', help='Path to output folder for storing evaluation reports')
    args = parser.parse_args()

    seed = 2022
    set_seed(seed)

    logging.info("Testing Bert for all dataset without fine-tuning")

    # Set useful parameters
    data_folder = args.input_folder
    output_folder = args.output_folder

    datasets = ["conll_dish.json", "cerec_dish.json", "ontonotes_dish.json", "i2b2-06_dish.json",
                "GUM_dish.json", "AnEM_dish.json", "BTC_dish.json", "WNUT17_dish.json", "Wikigold_dish.json",
                "re3d_dish.json", "SEC_dish.json", "sciERC_dish.json"]
    # names following the same order
    names = ["conll", "cerec", "ontonotes", "i2b2", "GUM", "AnEM", "BTC", "WNUT17", "wikigold", "re3d", "sec", "sciERC"]

    # datasets = ["cerec_dish.json"]
    # # names following the same order
    # names = ["cerec"]
    print("Number of datasets: ", len(datasets))

    results = {}
    for t_name in names:
        print('testing on dataset: {}'.format(t_name))
        logging.info('testing on dataset: {}'.format(t_name))
        s_train_data, s_train_labels, t_test_data, t_test_labels = load_data(data_folder, t_name)
        tag2id, id2tag = get_tag2id(s_train_labels + t_test_labels)
        weighted_f1_categories = {'ORG': [], 'PER': [], 'DIG': [], 'LOC': []}
        if DEBUG:
            sample_size = 10
            s_train_data, s_train_labels, t_test_data, t_test_labels = s_train_data[10:20], \
                                                                       s_train_labels[10:20], t_test_data[10:20], t_test_labels[10:20]
        epoch = 1 if DEBUG else 10
        weighted_f1, result_str, weighted_f1s_categories = trainer(s_train_data, s_train_labels, t_test_data,
                                                                   t_test_labels, weighted_f1_categories,
                                                                   (tag2id, id2tag),
                                                                   num_epoch=epoch, evaluate_only=True)
        name_key = t_name
        results[name_key] = (weighted_f1, result_str, weighted_f1s_categories)

    with open(output_folder + 'perf_measure_conll', 'a+') as f:
        json.dump(results, f)


import numpy as np
import matplotlib.pyplot as plt
import scipy.integrate
import scipy.signal
import scipy.optimize
from mpl_toolkits.mplot3d import Axes3D

#IMPORTANT
#the following units are used:
#Angström, fs, eV, Ampere, kg, etc...
a = 6.24150913 # my personal constant
h = 0.6582119569 #hbar in units of eV*fs
e = 1/1000/a #charge of electron in units of fs*eV
c_0 = 2997.92458 #speed of light in units of Ang/fs

def gaussian(t, width):
    # a basic gaussian function
    return np.exp(-t**2/(2*width**2))

def createTransient(E_0, omega, alpha, pol, width, phase=0, N=1, omega_chirp=0):
    """Returns a tuple of three functions with parameter time: (E_x(t), E_y(t), B_z(t));
    E_0: Amplitude of the Transient;
    omega: angular frequency
    pol: Polarisation (0: E perpendicular to surface, pi/2 parallel to surface)
    width: Width of the Transient
    phase: CEP
    N: refractive index
    omega_chirp: chirp freq."""
    @np.vectorize
    def inner_Ex(t):
        if pol == np.pi/2: # fixing numeric zero issues
            return 0
        return E_0*gaussian(t, width)*np.cos(omega*t + omega_chirp*t**2 + phase)*np.sin(alpha)*np.cos(pol)
    def inner_Ey(t):
        return -E_0*gaussian(t, width)*np.cos(omega*t + omega_chirp*t**2 + phase)*np.sin(pol)
    def inner_Bz(t):
        return -E_0*N/c_0*gaussian(t, width)*np.cos(omega*t + omega_chirp*t**2 + phase)*np.cos(alpha)*np.sin(pol)
    return inner_Ex, inner_Ey, inner_Bz

def createBandTightBinding(amplitude, lattice_const):
    """Returns function e(kx, ky):
    amplitude: Amplitude of the band
    lattice_const: REAL SPACE lattice const."""
    def inner(kx, ky):
        return - amplitude*(np.cos(lattice_const*kx) + np.cos(lattice_const*ky) - 2)
    return inner

def createVelocityTightBinding(amplitude, lattice_const):
    """Returns functions vx(kx, ky), vy(kx, ky):
    amplitude: Amplitude of the band
    lattice_const: REAL SPACE lattice const."""
    factor = amplitude*lattice_const/h
    def vx(kx, ky):
        return factor*np.sin(lattice_const*kx)
    def vy(kx, ky):
        return factor*np.sin(lattice_const*ky)
    return vx, vy

def createBandGraphene(amplitude, lattice_const):
    def inner(kx,ky):
        return amplitude*np.sqrt(  1 + 4* np.cos(np.sqrt(3)*kx*lattice_const/2)* np.cos(ky*lattice_const/2) + 4* np.cos(ky*lattice_const/2)**2 )
    return inner

def createVelocityGraphene(amplitude, lattice_const):
    e = createBandGraphene(amplitude, lattice_const)
    def inner_vx(kx, ky):
        return - lattice_const*amplitude**2/h/e(kx,ky)*( 2* np.cos(lattice_const*ky/2) + np.cos(np.sqrt(3)*lattice_const*kx/2) )*np.sin(lattice_const*ky/2)
    def inner_vy(kx, ky):
        return - lattice_const*amplitude**2/h/e(kx,ky)*np.sqrt(3)*np.cos(lattice_const*ky/2)*np.sin(np.sqrt(3)*lattice_const*kx/2)
    return inner_vx, inner_vy

def createBandDiracCone(amplitude):
    def inner(kx, ky):
        return np.sqrt(kx**2 + ky**2)
    return inner

def createVelocityDiracCone(amplitude):
    def inner_vx(kx, ky):
        return kx/np.sqrt(kx**2 + ky**2)/h
    def inner_vy(kx, ky):
        return ky/np.sqrt(kx**2 + ky**2)/h
    return inner_vx, inner_vy

def fourierTransform(tArray, realValues):
    """Returns: fftFreqs, fftVals"""
    y = realValues
    n = len(tArray)
    freqs = np.fft.fftfreq(n)
    mask = freqs > 0
    fft_theo = 2*np.abs(np.fft.fft(y)/n)
    return freqs[mask], fft_theo[mask]

def cutoff_log_fit(freq, alpha1, alpha2, beta, cutoff):
    """log of a basic cutoff function to fit onto a spectrum"""
    if freq <= cutoff:
        return alpha1*freq + beta
    else:
        return alpha2*freq + cutoff*(alpha1-alpha2) + beta

cutoff_log_fit = np.vectorize(cutoff_log_fit, otypes=[float]) # vectorize it

def cutoff_fit(freq, alpha1, alpha2, beta, cutoff):
    """basic cutoff function to fit onto a spectrum"""
    if freq <= cutoff:
        return np.exp(alpha1*freq + beta)
    else:
        return np.exp(alpha2*freq + cutoff*(alpha1-alpha2) + beta)

cutoff_fit = np.vectorize(cutoff_fit, otypes=[float]) # vectorize this one as well

class Transient:
    def __init__(self, E_0 = 25*a, omega = 0.05*np.pi, alpha = 86/180*np.pi, pol = np.pi/2, width = 100/2.35482, phase = 0, refrac = 10, omega_chirp=0):
        self.E_0 = E_0
        self.omega = omega
        self.alpha = alpha
        self.pol = pol
        self.width = width
        self.phase = phase
        self.refrac = refrac
        self.omega_chirp = omega_chirp
        self.E_x, self.E_y, self.B_z = createTransient(self.E_0, self.omega, self.alpha, self.pol, self.width, self.phase, self.refrac, self.omega_chirp)

    def changeParam(self, **kwargs):
        for elem in kwargs:
            self.__dict__[elem] = kwargs[elem]
        self.E_x, self.E_y, self.B_z = createTransient(self.E_0, self.omega, self.alpha, self.pol, self.width, self.phase, self.refrac)

    def plotTrans(self, t_init=-4*10**2, t_final=4*10**2, t_delta=.01):
        t_eval = np.arange(t_init, t_final, t_delta)
        fig_1 = plt.figure(1, figsize = (10, 10))
        ex_chart = fig_1.add_subplot(311)
        ex_chart.set_title("E_x(t)")
        ey_chart = fig_1.add_subplot(312)
        ey_chart.set_title("E_y(t)")
        bz_chart = fig_1.add_subplot(313)
        bz_chart.set_title("B_z(t)")
        ex_chart.plot(t_eval, self.E_x(t_eval))
        ex_chart.set_ylim(self.E_x(t_eval).min()-0.01, self.E_x(t_eval).max()+0.01)
        ey_chart.plot(t_eval, self.E_y(t_eval))
        ey_chart.set_ylim(self.E_y(t_eval).min()-0.01, self.E_y(t_eval).max()+0.01)
        bz_chart.plot(t_eval, self.B_z(t_eval))
        bz_chart.set_ylim(self.B_z(t_eval).min()-0.01, self.B_z(t_eval).max()+0.01)
        plt.show()

    def plotFFT(self, t_init=-4*10**2, t_final=4*10**2, t_delta=.01, scale="log"):
        t_eval = np.arange(t_init, t_final, t_delta)
        fig_1 = plt.figure(1, figsize = (10, 10))
        ex_chart = fig_1.add_subplot(311)
        ex_chart.set_yscale(scale)
        ex_chart.set_title("FFT of E_x(t)")
        ey_chart = fig_1.add_subplot(312)
        ey_chart.set_yscale(scale)
        ey_chart.set_title("FFT of E_y(t)")
        bz_chart = fig_1.add_subplot(313)
        bz_chart.set_yscale(scale)
        bz_chart.set_title("FFT of B_z(t)")
        ex_chart.plot(*fourierTransform(t_eval, self.E_x(t_eval)))
        ey_chart.plot(*fourierTransform(t_eval, self.E_y(t_eval)))
        bz_chart.plot(*fourierTransform(t_eval, self.B_z(t_eval)))
        plt.show()

class BandTightBinding:
    def __init__(self, lattice_const, amplitude):
        self.lattice_const = lattice_const
        self.amplitude = amplitude
        self.vx, self.vy = createVelocityTightBinding(self.amplitude, self.lattice_const)
        self.band = createBandTightBinding(amplitude, lattice_const)

    def plotVel(self, size, numStep):
        X, Y = np.linspace(-size, size, numStep), np.linspace(-size, size, numStep)
        X, Y = np.meshgrid(X, Y)
        U, V = self.vx(X, Y), self.vy(X, Y)
        plt.quiver(X,Y,U,V)
        plt.show()

class BandGraphene:
    def __init__(self, amplitude, lattice_const):
        self.amplitude = amplitude
        self.lattice_const = lattice_const
        self.band = createBandGraphene(amplitude, lattice_const)
        self.vx, self.vy = createVelocityGraphene(amplitude, lattice_const)

    def plotBand(self, xrange=(-1,1), yrange=(-1,1), stepsize=0.01, contourmap=True, wireframe=False):
        x = np.arange(*xrange, stepsize)
        y = np.arange(*yrange, stepsize)
        xx, yy = np.meshgrid(x, y, sparse=True)
        if contourmap:
            z = self.band(xx, yy)
            plt.contourf(x,y,z)
            plt.show()
        if wireframe:
            z2 = self.band(xx, yy)
            fig = plt.figure()
            ax = fig.add_subplot(111, projection='3d', title = "BandStruc")
            ax.plot_wireframe(xx, yy, z2)
            plt.show()

class BandDiracCone:
    def __init__(self, amplitude):
        self.amplitude = amplitude
        self.band = createBandDiracCone(amplitude)
        self.vx, self.vy = createVelocityDiracCone(amplitude)

    def plotBand(self, xrange=(-1,1), yrange=(-1,1), stepsize=0.01, contourmap=True, wireframe=False):
        x = np.arange(*xrange, stepsize)
        y = np.arange(*yrange, stepsize)
        xx, yy = np.meshgrid(x, y, sparse=True)
        if contourmap:
            z = self.band(xx, yy)
            plt.contourf(x,y,z)
            plt.show()
        if wireframe:
            z2 = self.band(xx, yy)
            fig = plt.figure()
            ax = fig.add_subplot(111, projection='3d', title = "BandStruc")
            ax.plot_wireframe(xx, yy, z2)
            plt.show()

class Solution:
    def __init__(self, bandstructure, transient, t_init=-4*10**2, t_final=4*10**2, t_delta=.01, kx_0 = 0, ky_0 = 0): #the bandstructure and the transient are objects
        self.bandstructure = bandstructure
        self.transient = transient
        self.t_init = t_init
        self.t_final = t_final
        self.t_delta = t_delta
        self.y0 = [0, 0, kx_0, ky_0]

    def solve(self):
        Vlc_x = self.bandstructure.vx
        Vlc_y = self.bandstructure.vy
        E_x, E_y, B_z = self.transient.E_x, self.transient.E_y, self.transient.B_z
        def f(t, y): #y is array: y = [r_x, r_y, k_x, k_y].
            dr_x = Vlc_x(y[2], y[3])
            dr_y = Vlc_y(y[2], y[3])
            dk_x = (-e * E_x(t) - e * Vlc_y(y[2], y[3]) * B_z(t)) / h
            dk_y = (-e * E_y(t) + e * Vlc_x(y[2], y[3]) * B_z(t)) / h
            return np.array([dr_x, dr_y, dk_x, dk_y])

        t_span = (self.t_init, self.t_final)
        t_eval = np.arange(self.t_init, self.t_final, self.t_delta)

        self.raw_sol = scipy.integrate.solve_ivp(fun = f, t_span = t_span, y0 = self.y0, t_eval = t_eval)
        self.raw_sol_mirror = scipy.integrate.solve_ivp(fun = f, t_span = t_span, y0 = [self.y0[0], self.y0[1], -self.y0[2], -self.y0[3]], t_eval = t_eval)
        return self.raw_sol, self.raw_sol_mirror

    def plotReziTraj(self):
        plt.plot(self.raw_sol.y[2], self.raw_sol.y[3])
        plt.plot(self.raw_sol_mirror.y[2], self.raw_sol_mirror.y[3])
        plt.show()

    def plotRealTraj(self):
        plt.plot(self.raw_sol.y[2], self.raw_sol.y[3])
        plt.plot(self.raw_sol_mirror.y[2], self.raw_sol_mirror.y[3])
        plt.show()

    def process(self):
        self.vx = self.bandstructure.vx(self.raw_sol.y[2], self.raw_sol.y[3])
        self.vy = self.bandstructure.vy(self.raw_sol.y[2], self.raw_sol.y[3])
        self.vx_mirror = self.bandstructure.vx(self.raw_sol_mirror.y[2], self.raw_sol_mirror.y[3])
        self.vy_mirror  = self.bandstructure.vy(self.raw_sol_mirror.y[2], self.raw_sol_mirror.y[3])

        self.ax = np.gradient(self.vx)/self.t_delta
        self.ay = np.gradient(self.vy)/self.t_delta
        self.ax_mirror = np.gradient(self.vx_mirror)/self.t_delta
        self.ay_mirror = np.gradient(self.vy_mirror)/self.t_delta

        self.fftx = fourierTransform(self.raw_sol.t, self.ax + self.ax_mirror)
        self.ffty = fourierTransform(self.raw_sol.t, self.ay + self.ay_mirror)

    def plotVel(self): #WARNING: DOES NOT INCLUDE MIRROR PARTICLE
        plt.plot(self.raw_sol.t, self.vx)
        plt.plot(self.raw_sol.t, self.vy)
        plt.show()

    def plotAcc(self): #WARNING: DOES NOT INCLUDE MIRROR PARTICLE
        plt.plot(self.raw_sol.t, self.ax)
        plt.plot(self.raw_sol.t, self.ay)
        plt.show()

    def plotFFt(self, scale="log"):
        figg = plt.figure(1, figsize=(22,7))
        chrt_x = figg.add_subplot(211)
        chrt_y = figg.add_subplot(212)
        chrt_x.plot(*self.fftx)
        chrt_y.plot(*self.ffty)
        chrt_x.set_title("fftx")
        chrt_y.set_title("ffty")
        chrt_x.set_yscale(scale)
        chrt_y.set_yscale(scale)
        plt.show()

    def processRadPower(self, delta): #NOT FINISHED
        gauss = gaussian(np.arange(-4*delta, 4*delta, self.t_delta),delta)
        self.radX = np.convolve(self.ax + self.ax_mirror,gauss, mode="same")**2
        self.radY = np.convolve(self.ay + self.ay_mirror,gauss, mode="same")**2

    def processFFTMaxima(self, num_delta=20, shorten=0, plotX=False, plotY=False):
        self.ind_max_X = scipy.signal.find_peaks(self.fftx[1][:-shorten], distance=num_delta)[0]
        self.log_fftX_max = np.log(self.fftx[1][self.ind_max_X])
        if plotX:
            plt.scatter(self.fftx[0][self.ind_max_X], self.log_fftX_max) # scatter plot to estimate start parameters
            plt.show()

        self.ind_max_Y = scipy.signal.find_peaks(self.ffty[1][:-shorten], distance=num_delta)[0]
        self.log_fftY_max = np.log(self.ffty[1][self.ind_max_Y])
        if plotY:
            plt.scatter(self.ffty[0][self.ind_max_Y], self.log_fftY_max) # scatter plot to estimate start parameters
            plt.show()

    def processCutoffFit(self, p0_X=None, p0_Y = None, plotX=False, plotY=False):
        self.poptX, _ = scipy.optimize.curve_fit(
            f = cutoff_log_fit,
            xdata = self.fftx[0][self.ind_max_X],
            ydata = self.log_fftX_max,
            p0 = p0_X)
        if plotX:
            plt.scatter(self.fftx[0][self.ind_max_X], self.log_fftX_max) # scatter plot to estimate start parameters
            plt.plot(self.fftx[0][self.ind_max_X], cutoff_log_fit(self.fftx[0][self.ind_max_X], *self.poptX), color="red")
            plt.show()

        self.poptY, _ = scipy.optimize.curve_fit(
            f = cutoff_log_fit,
            xdata = self.ffty[0][self.ind_max_Y],
            ydata = self.log_fftY_max,
            p0 = p0_Y)
        if plotY:
            plt.scatter(self.ffty[0][self.ind_max_Y], self.log_fftY_max) # scatter plot to estimate start parameters
            plt.plot(self.ffty[0][self.ind_max_Y], cutoff_log_fit(self.ffty[0][self.ind_max_Y], *self.poptY), color="red")
            plt.show()

        return self.poptX, self.poptY


"""added user, role, document

Revision ID: 1a69d59e87d6
Revises:
Create Date: 2021-04-24 16:47:06.569269

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1a69d59e87d6'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('document_type',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=128), nullable=False),
    sa.Column('description', sa.String(length=256), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_document_type_description'), 'document_type', ['description'], unique=False)
    op.create_index(op.f('ix_document_type_name'), 'document_type', ['name'], unique=False)
    op.create_table('revoked_tokens',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('jti', sa.String(length=120), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('role',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.Enum('registered_user', name='rolename'), nullable=False),
    sa.Column('documents_number', sa.Integer(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('user',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('username', sa.String(length=64), nullable=True),
    sa.Column('email', sa.String(length=120), nullable=True),
    sa.Column('password_hash', sa.String(length=128), nullable=True),
    sa.Column('created_at', sa.DateTime(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_user_email'), 'user', ['email'], unique=True)
    op.create_index(op.f('ix_user_username'), 'user', ['username'], unique=True)
    op.create_table('document',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=128), nullable=False),
    sa.Column('path', sa.String(), nullable=False),
    sa.Column('description', sa.String(), nullable=True),
    sa.Column('hash_sha256', sa.String(length=65), nullable=True),
    sa.Column('simhash', sa.String(), nullable=True),
    sa.Column('user_id', sa.Integer(), nullable=True),
    sa.Column('doc_type', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['doc_type'], ['document_type.id'], ),
    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_document_name'), 'document', ['name'], unique=False)
    op.create_index(op.f('ix_document_path'), 'document', ['path'], unique=False)
    op.create_table('users_roles',
    sa.Column('user_id', sa.Integer(), nullable=False),
    sa.Column('role_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['role_id'], ['role.id'], ),
    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
    sa.PrimaryKeyConstraint('user_id', 'role_id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('users_roles')
    op.drop_index(op.f('ix_document_path'), table_name='document')
    op.drop_index(op.f('ix_document_name'), table_name='document')
    op.drop_table('document')
    op.drop_index(op.f('ix_user_username'), table_name='user')
    op.drop_index(op.f('ix_user_email'), table_name='user')
    op.drop_table('user')
    op.drop_table('role')
    op.drop_table('revoked_tokens')
    op.drop_index(op.f('ix_document_type_name'), table_name='document_type')
    op.drop_index(op.f('ix_document_type_description'), table_name='document_type')
    op.drop_table('document_type')
    # ### end Alembic commands ###


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='NewsPost',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('title', models.CharField(max_length=200, verbose_name='Titel')),
                ('posted', models.DateTimeField(verbose_name='Dato')),
                ('body', models.TextField(verbose_name='Indhold')),
                ('group_handle', models.CharField(max_length=20)),
                ('year', models.IntegerField(default=0)),
                ('author', models.ForeignKey(to=settings.AUTH_USER_MODEL, verbose_name='Forfatter', on_delete=models.CASCADE)),
            ],
            options={
                'verbose_name_plural': 'nyheder',
                'verbose_name': 'nyhed',
            },
        ),
    ]


from django import forms
from .models import Categories
from videos.models import Video

class CategoriesForm(forms.ModelForm):
    class Meta:
        model = Categories
        fields = [
            'title',
            'slug',
            'description'
        ]

class CategoriesAdminForm(forms.ModelForm):
    class Meta:
        model = Categories
        fields = [
            'title',
            'slug',
            'video',
            'description'
        ]

    def __init__(self, *args, **kwargs):
        super(CategoriesAdminForm, self).__init__(*args, **kwargs)
        obj = kwargs.get('instance')
        qs = Video.objects.all().unused()
        if obj:
            if obj.video:
                this_ = Video.objects.filter(pk=obj.video.pk)
                qs = (qs | this_)
            self.fields['video'].queryset = qs
        else:
            self.fields['video'].queryset = qs


# Convert COG metadata into possible (aligned) base sequence that would produce these mutations and ambiguities.
# It will choose some plausible triple of bases for each AA name, which will often not be the actual triple used.
# Output in GISAID-style format with hCoV-19/ prefix and |collection date.

from stuff import *
import sys,argparse

parser=argparse.ArgumentParser()
parser.add_argument('-f', '--mindate',       default="2019-01-01",  help="Minimum date used from input")
parser.add_argument('-s', '--sorted',     action="store_true",      help="Whether input is sorted into reverse date order")
args=parser.parse_args()

stats={}
for geneloc in genes.values():
  for p in range(geneloc[0],geneloc[1],3):
    c=refgenome[p:p+3]
    x=codontable.get(c,'?')
    if x not in stats: stats[x]={}
    stats[x][c]=stats[x].get(c,0)+1

# Find most likely triple of bases for each AA
# AA2bases[ new AA name, refgenome triple ] = best guess for new triple
AA2bases={}
w0,w1,w2=0.4,0.7,0.75# Weights that work reasonably well
for x in stats:
  y=stats[x]
  s=sum(stats[x].values())
  for old3 in codontable:
    best=(-1e30,)
    for new3 in y:
      # Mutation from old3 -> new3
      score=y[new3]/s-w0*(old3[0]!=new3[0])-w1*(old3[1]!=new3[1])-w2*(old3[2]!=new3[2])
      if score>best[0]: best=(score,new3)
    AA2bases[x,old3]=best[1]

for (name,date,ml,am) in csvrows_it(sys.stdin,["sequence_name","sample_date","mutations","ambiguities"]):
  if date<args.mindate:
    if args.sorted: break
    continue

  ml=ml.split('|')
  am=am.split('|')

  output=list(refgenome)

  for mut in ml:
    (gene,m)=mut.split(':')
    x=m[0]
    p1=int(m[1:-1])
    y=m[-1]
    if gene=='synSNP':
      p=p1-1
      output[p]=y
    else:
      if gene=='orf1ab':
        if p1<=4401: gene='ORF1a'
        else: gene='ORF1b';p1-=4401
      p0=genes[gene]
      p=p0[0]+(p1-1)*3-1
      output[p:p+3]=list(AA2bases[y,refgenome[p:p+3]])

  for ar in am:
    x=[int(p)-1 for p in ar.split('-')]
    if len(x)==1: output[x[0]]='N'
    else: output[x[0]:x[1]+1]=['N']*(x[1]+1-x[0])

  print(">hCoV-19/"+name+"|"+date)
  print("".join(output))


"""
This is an image viewer application
"""

import sys
import signal
# pylint: disable=no-name-in-module
from PyQt5.QtGui import QPixmap, QPainter
from PyQt5.QtWidgets import QWidget, QApplication, QTableWidget

signal.signal(signal.SIGINT, signal.SIG_DFL)

class ImageWidget(QWidget):

    def __init__(self, image_path, parent):
        super().__init__(parent)
        self.picture = QPixmap(image_path)

    def paintEvent(self, _event):
        painter = QPainter(self)
        painter.drawPixmap(0, 0, self.picture)

class TableWidget(QTableWidget):

    def setImage(self, row, col, image_path):
        image = ImageWidget(image_path, self)
        self.setCellWidget(row, col, image)

def main():
    app = QApplication([])
    table_widget = TableWidget(10, 2)
    table_widget.setImage(0, 1, "data/jpg/image0000.jpg")
    table_widget.show()
    sys.exit(app.exec_())

main()


import tkinter
import customtkinter
from pytube import YouTube

def startDownloadHigh():
    try:
        youtubeLink = link.get()
        youtubeObject = YouTube(youtubeLink, on_progress_callback=onProgress)
        title.configure(text=youtubeObject.title, text_color="white")
        author.configure(text="Uploaded by " + youtubeObject.author, text_color="white")
        length.configure(text=str(youtubeObject.length) + " seconds", text_color="white")
        finishLabel.configure(text="")
        video = youtubeObject.streams.get_highest_resolution()
        video.download()
        finishLabel.configure(text="Downloaded", text_color="green")
    except:
        finishLabel.configure(text="Download Error!", text_color="red")

def startDownloadAudio():
    try:
        youtubeLink = link.get()
        youtubeObject = YouTube(youtubeLink, on_progress_callback=onProgress)
        title.configure(text=youtubeObject.title, text_color="white")
        author.configure(text="Uploaded by " + youtubeObject.author, text_color="white")
        length.configure(text=str(youtubeObject.length) + " seconds", text_color="white")
        finishLabel.configure(text="")
        audio = youtubeObject.streams.get_audio_only()
        audio.download()
        finishLabel.configure(text="Downloaded", text_color="green")
    except:
        finishLabel.configure(text="Download Error!", text_color="red")

def onProgress(stream, chunk, bytes_remaining):
    total_size = stream.filesize
    bytes_downloaded = total_size - bytes_remaining
    percentage_of_completion = bytes_downloaded / total_size * 100
    per = str(int(percentage_of_completion))
    pPercentage.configure(text=per+"%")
    pPercentage.update()

    # Update progress bar
    progressBar.set(float(percentage_of_completion / 100))

# System settings
customtkinter.set_appearance_mode("dark")
customtkinter.set_default_color_theme("blue")

# App frame
app = customtkinter.CTk()
app.geometry("720x480")
app.title("Youtube Downloader")

# UI elements
title = customtkinter.CTkLabel(app, text="Insert Youtube link")
title.pack(padx = 10, pady = 10)
author = customtkinter.CTkLabel(app, text="")
author.pack(padx = 10, pady = 10)
length = customtkinter.CTkLabel(app, text="")
length.pack(padx = 10, pady = 10)

# Link input
url = tkinter.StringVar()
link = customtkinter.CTkEntry(app, width=350, height=40, textvariable=url)
link.pack()

# Finished Downloading
finishLabel = customtkinter.CTkLabel(app, text="")
finishLabel.pack()

# Progress percentage
pPercentage = customtkinter.CTkLabel(app, text="0%")
pPercentage.pack()

progressBar = customtkinter.CTkProgressBar(app, width=400)
progressBar.set(0)
progressBar.pack()

# Download button

## Button for high resolution download
download_high_resolution = customtkinter.CTkButton(app, text="Download Highest Resolution", command=startDownloadHigh, height=40, width=150)
download_high_resolution.pack(padx=10, pady=15)

## Button for mp4 download
download_audio = customtkinter.CTkButton(app, text=" Download Audio Only (MP4) ", command=startDownloadAudio, height=40, width=150)
download_audio.pack(padx=10, pady=15)

# Footer
footer = customtkinter.CTkLabel(app, text="RegusAl ©2023")
footer.pack(pady=30)

# Run app
app.mainloop()


from flask import Flask, render_template
import sqlite3
app = Flask(__name__)

@app.route('/')
def index():
    return render_template("index.html")

@app.route('/index')
def home():
    return index()

@app.route('/rank')
def rank():
    datalist = []
    con = sqlite3.connect("movie.db")
    cur = con.cursor()
    sql = "select * from movie250"
    data = cur.execute(sql)
    for item in data:
        datalist.append(item)
    cur.close()
    con.close()
    return render_template("rank.html", movies=datalist)

@app.route('/score')
def score():
    score = []
    num = []
    con = sqlite3.connect("movie.db")
    cur = con.cursor()
    sql = "select score,count(score) from movie250 group by score"
    data = cur.execute(sql)
    for item in data:
        score.append(item[0])
        num.append(item[1])
    cur.close()
    con.close()
    return render_template("score.html", score=score, num=num,)

# def score_2():
#     name = []
#     score2 = []
#     people = []
#     con2 = sqlite3.connect("movie.db")
#     cur2 = con2.cursor()
#     sql2 = "select cname,score,rated from movie250"
#     data2 = cur2.execute(sql2)
#     for item2 in data2:
#         name.append(item2[0])
#         score2.append(item2[1])
#         people.append(item2[2])
#     cur2.close()
#     con2.close()
#     return render_template("score.html", name=name, score2=score2, people=people)

@app.route('/wordcloud')
def wordcloud():
    return render_template("wordcloud.html")

@app.route('/other')
def other():
    return render_template("other.html")

if __name__ == '__main__':
    app.run()


import os
import re
import time
import httpretty
import unittest
import socket

from app_fixture import app

test_iri = "/us/judgment/us/1986-06-30/05-179/eng@published#test/2014-01-01/main.xml"
test_transform = "mayer-to-akx"
test_xml_file = "./test/data/0b3e349463dcdd20.xml"
test_file = ""
with open(test_xml_file, 'r') as f:
    test_file = f.read()
full_iri = test_iri + "?transform=%s" % test_transform

test_forwarding_headers = {
    "Content-Type": "text/xml",
    "X-Forward-Id": "550e8400-e29b-41d4-a716-446655440000",
    "X-Forward-To": "http://example.com,http://2example.com",
    "X-Forward-Query-Params": "foo=bar&baz=quux,testing=true",
    "X-Forward-Method": "POST,PUT",
    "X-Forward-Errors-To": "errors.com:7070",
    "X-Forward-Referer": "ORIGIN"
}

test_unforwardable_headers = {
    "Content-Type": "application/xml",
    "X-Forward-Id": "550e8400-e29b-41d4-a716-446655440000",
    "X-Forward-Errors-To": "errors.com:7070",
    "X-Forward-Referer": "ORIGIN"
}

fail_forwarding_headers = {
    "Content-Type": "text/xml",
    "X-Forward-Id": "550e8400-e29b-41d4-a716-446655440000",
    "X-Forward-To": "http://err-gen.com",
    "X-Forward-Query-Params": "foo=bar&baz=quux",
    "X-Forward-Method": "POST",
    "X-Forward-Errors-To": "asherah.int.casetext.com:7099",
    "X-Forward-Referer": "ORIGIN"
}

timeout_forwarding_headers = {
    "Content-Type": "text/xml",
    "X-Forward-Id": "550e8400-e29b-41d4-a716-446655440000",
    "X-Forward-To": "http://timeout.com",
    "X-Forward-Query-Params": "foo=bar&baz=quux",
    "X-Forward-Method": "POST",
    "X-Forward-Errors-To": "errors.com:7070",
    "X-Forward-Referer": "ORIGIN"
}

expected_forwarding_headers = {
    "Content-Type": "text/xml",
    "X-Forward-Id": "550e8400-e29b-41d4-a716-446655440000",
    "X-Forward-To": "http://2example.com",
    "X-Forward-Query-Params": "testing=true",
    "X-Forward-Method": "PUT",
    "X-Forward-Errors-To": "errors.com:7070",
    "X-Forward-Referer": "http://localhost" + full_iri
}

class TestForwarding(unittest.TestCase):
    def setUp(self):
        self.flask_app_obj = app
        self.app = self.flask_app_obj.test_client()
        httpretty.enable()
        self.response_count = 0
        self.error_count = 0
        def callback(request, uri, headers):
            self.assertEqual(request.body.decode('utf8'), test_file)
            for expected_header, value in expected_forwarding_headers.items():
                self.assertIn(expected_header, request.headers)
                self.assertEqual(request.headers[expected_header], value)

            self.response_count += 1
            return (202, headers, "")
        def err_callback(request, uri, headers):
            self.error_count += 1
            return (200, headers, "")

        httpretty.register_uri(httpretty.POST,
                               re.compile("example.com/(.*)$"),
                               body=callback)
        httpretty.register_uri(httpretty.POST,
                               re.compile("errors.com/(.*)$"),
                               body=callback)

    def tearDown(self):
        httpretty.disable()
        httpretty.reset()

    def test_without_forwarding(self):
        """ Not setting forwarding headers should trigger a 200. """
        full_iri = test_iri + "?transform=%s" % test_transform
        with open(test_xml_file, 'r') as f:
            r = self.app.post(full_iri, data=f.read(), headers=test_unforwardable_headers)
            self.assertEqual(r.status_code, 200)

    def test_with_bad_headers(self):
        """ Failing to set any one of the forwarding headers should trigger a 400. """
        for i in range(0, len(test_forwarding_headers)):
            items = list(test_forwarding_headers.items())
            missing_header = items.pop(i)

            # we can't not set the Content-Type, so skip that one
            if missing_header[0] == "Content-Type":
                continue

            shortened_headers = dict(items)
            for header, value in shortened_headers.items():
                with open(test_xml_file, 'r') as f:
                    r = self.app.post(full_iri, data=f.read(), headers=shortened_headers)
                    self.assertEqual(r.status, "400 Missing header %s" % missing_header[0])

    def test_with_valid_forwarding_scheme(self):
        """ With a valid forwarding scheme in place, we should get a 202.
            We should also expect the dummy HTTP server to get hit correctly. """
        with open(test_xml_file, 'r') as f:
            r = self.app.post(full_iri, data=f.read(), headers=test_forwarding_headers)
            self.assertEqual(r.status_code, 202, r.status)
            time.sleep(0.5)
            self.assertEqual(self.error_count, 0)
            self.assertEqual(self.response_count, 1)

class TestForwardingErrors(unittest.TestCase):
    def setUp(self):
        self.flask_app_obj = app
        self.app = self.flask_app_obj.test_client()
        httpretty.enable()
        self.error_provocation_count = 0
        self.error_response_count = 0

        def provoke_timeout_error(request, uri, headers):
            raise socket.timeout('timeout')
            return (200, headers, "OK")

        def provoke_error_callback(request, uri, headers):
            self.error_provocation_count += 1
            return (500, headers, "ERROR!")

        def handle_error_callback(request, uri, headers):
            self.error_response_count += 1
            return (200, headers, "OK")

        httpretty.register_uri(httpretty.POST,
                               re.compile("err-gen.com/(.*)$"),
                               body=provoke_error_callback)
        httpretty.register_uri(httpretty.POST,
                               re.compile("errors.com/(.*)$"),
                               body=handle_error_callback)
        httpretty.register_uri(httpretty.POST,
                               re.compile("timeout.com/(.*)$"),
                               body=provoke_timeout_error)

    # We should set up a udp listener to verify the changed error handling here, but
    # probably only if we actually continue using this library.
    '''def test_error_handling(self):
        """ If a forward fails, we should see a message to the error-handling server. """
        with open(test_xml_file, 'r') as f:
            r = self.app.post(full_iri, data=f.read(), headers=fail_forwarding_headers)
            self.assertEqual(r.status_code, 202, r.status)
            time.sleep(0.5)
            self.assertEqual(self.error_provocation_count, 1)
            self.assertEqual(self.error_response_count, 1)
    def test_timeout_handling(self):
        """ If a forwarding times out, we should see a message to the error-handling server. """
        with open(test_xml_file, 'r') as f:
            r = self.app.post(full_iri, data=f.read(), headers=timeout_forwarding_headers)
            self.assertEqual(r.status_code, 202, r.status)
            time.sleep(0.5)
            self.assertEqual(self.error_response_count, 1)
    '''


#!/usr/bin/env python3

import zmq

if __name__ == "__main__":

    context = zmq.Context()
    socket = context.socket(zmq.REQ)

    print("Connecting to hello world server…")
    socket.connect("tcp://localhost:5555")

    #  Socket to talk to server
    while True:
        msg = input("> ")
        print("Sent message")
        socket.send_string(msg)
        message = socket.recv_string()
        print("Message received: ", message)


import numpy as np
import pandas as pd
from keras import models
from keras import layers
from collections import Counter
import matplotlib.pyplot as plt
import re
from keras.utils import to_categorical
import tensorflow as tf

#Data Read
data = pd.read_csv("C:\\Users\\Wojtek\\Desktop\\alldata.csv", sep=',', encoding='latin-1')
data2 = pd.DataFrame(data)

#Shuffle
data3 = data2.sample(frac=1, random_state=42).reset_index(drop=True)

#Division of data
text = data3.iloc[:, 1]
labels = data3.iloc[:, 0]
for i in enumerate(labels):
    if i[1] == 'positive':
        labels[i[0]] = 2
    elif i[1] == 'negative':
        labels[i[0]] = 0
    else:
        labels[i[0]] = 1

labels = to_categorical(labels)

#Strings transformations
text = [*map(lambda x: re.sub('[\W_]+', ' ', x), text)]
text = [*map(lambda x: x.lower(), text)]
split_text = map(lambda x: x.split(), text)
split_text2 = pd.DataFrame(split_text)
split_text3 = list(filter(None, split_text2.values.reshape(split_text2.size)))
split_text4 = list(filter(str.isalpha, split_text3))

#Creating dictionary consisting of 5000 most popular words
Counter = Counter(split_text4)
nowords = 5000
most_popular = Counter.most_common(nowords - 1)
most_popular_word = [tup[0] for tup in most_popular]
myDict = {s: i + 1 for i, s in enumerate(most_popular_word)}

#Translating data into dictionary keys
occurences = split_text2.applymap(myDict.get).fillna(0).astype(int)
occurences = np.asarray(occurences)

#Reforging data into categorical classification if the word exists
results = np.zeros((len(occurences), nowords))
for i, sequence in enumerate(occurences):
    results[i, list(map(int, sequence))] = 1.

#Division into training and test sets
tr_results = results[1:3500]
tst_results = results[3500:]
tr_labels = labels[1:3500]
tst_labels = labels[3500:]

train = np.asarray(tr_results).astype('float32')
test = np.asarray(tst_results).astype('float32')

#Division training set into training and validation set
x_val = train[:1000]
x_train = train[1000:]
y_val = tr_labels[:1000]
y_train = tr_labels[1000:]

# 64-32-16-8-3 -> 0.741
# 256-3 -> 0.748
# 256-8 -> 0.748
# 128-16-8-3 -> 0.75/0.63 -> epoch 7
model = models.Sequential()
model.add(layers.Dense(128, activation='relu', input_shape=(5000,)))
model.add(layers.Dense(16, activation='relu'))
model.add(layers.Dense(8, activation='relu'))
model.add(layers.Dense(3, activation='softmax'))
model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['accuracy'])

history = model.fit(x_train,
                    y_train,
                    epochs=30,
                    batch_size=256,
                    validation_data=(x_val, y_val))

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)

plt.plot(epochs, loss, 'bo', label='Strata trenowania')
plt.plot(epochs, val_loss, 'b', label='Strata walidacji')
plt.title('Strata trenowania i walidacji')
plt.xlabel('Epoki')
plt.ylabel('Strata')
plt.legend()
plt.show()

plt.plot(epochs, acc, 'bo', label='Dokladnosc trenowania')
plt.plot(epochs, val_acc, 'b', label='Dokladnosc walidacji')
plt.title('Dokladnosc trenowania i walidacji')
plt.xlabel('Epoki')
plt.ylabel('Strata')
plt.legend()
plt.show()

# highest accuracy index and value
print(val_acc.index(max(val_acc)))
print(max(val_acc))
# lowest loss index and value
print(val_loss.index(min(val_loss)))
print(min(val_loss))

#Creating a model with efficient hyperparameters
model2 = models.Sequential()
model2.add(layers.Dense(128, activation='relu', input_shape=(5000,)))
model2.add(layers.Dense(16, activation='relu'))
model2.add(layers.Dense(8, activation='relu'))
model2.add(layers.Dense(3, activation='softmax'))
model2.compile(optimizer='rmsprop',
               loss='categorical_crossentropy',
               metrics=['accuracy'])

history = model2.fit(train,
                     tr_labels,
                     epochs=7,
                     batch_size=256)

tst_labels_model = model2.predict(tst_results)
tst_labels_model2 = np.round(tst_labels_model)
test_labels_model3 = tf.math.argmax(tst_labels_model2, axis=1).numpy()
tst_labels = tf.math.argmax(tst_labels, axis=1).numpy()

# noinspection PyTypeChecker - Printing test accuracy
print(sum(test_labels_model3 == tst_labels) / len(tst_labels))

#Saving result
# pd.DataFrame({'Neural' : test_labels_model3, 'Real':tst_labels}).to_csv(
# "C:\\Users\\Wojtek\\Desktop\\resultsCategory.csv")

#Checking single results

# change sequence to a sequence of numbers corresponding to most popular words
own_sentence = "We have a new and innovative strategy. Our company reported a raise in the amount of assets?"
own_sentence = re.sub('[\W_]+', ' ', own_sentence)
own_sentence2 = pd.DataFrame(own_sentence.lower().split())
own = own_sentence2.applymap(myDict.get).fillna(0).astype(int)

# change the representation of numbers to a vector of length=len(most_popular_words), and put ones if a word occur
own2 = np.zeros(nowords)
own2[own.values] = 1
own2 = np.asarray(own2).astype('float32')
own2 = own2.reshape(1, 5000)
lab = model2.predict(own2)[0]
print(lab)
if lab.argmax() == 0:
    print('Negative')
elif lab.argmax() == 1:
    print('Neutral')
else:
    print('Negative')