from pathlib import Path
import numpy as np
from collections import Counter
from itertools import product

DIRECTIONS = ("up", "down", "left", "right")
ROTATIONS = (0, 90, 180, 270)

def get_tile_borders(tile, dir="all"):
    borders = []
    border = None
    for d in DIRECTIONS:
        if dir in [d, "all"]:
            id = tile_to_id(tile, d, False)
            id_flip = tile_to_id(tile, d, True)
            border = min(id, id_flip)
            borders.append(border)
    if dir != "all":
        return border
    return borders

def tile_to_id(tile, dir, reverse=False):

    step = 1
    if reverse:
        step = -1
    if dir == "up":
        return border_to_id(tile[0, ::step])
    if dir == "down":
        return border_to_id(tile[-1, ::step])
    if dir == "left":
        return border_to_id(tile[::step, 0])
    if dir == "right":
        return border_to_id(tile[::step, -1])

def border_to_id(border):
    return int("".join([str(el) for el in border.astype(int)]), 2)

def rotate(tile, deg, flip_hor=False):
    if deg not in [0, 90, 180, 270]:
        raise RuntimeError("Invalid degrees")
    arr = np.copy(tile)
    if flip_hor:
        arr = arr[:, ::-1]

    if deg == 90:
        return arr[::-1, :].T
    elif deg == 180:
        return arr[::-1, ::-1]
    elif deg == 270:
        return arr[:, ::-1].T
    return arr

class Image:
    def __init__(self, data, pattern):

        self.tiles = dict()
        self.borders = dict()
        self.n_tiles = 0
        self.tile_dim = None
        self.corners = set()
        self.edges = set()
        self.interiors = set()
        self.border_counter = None
        self.corner_prod = None

        i = 0
        lines = data.split("\n")
        while i < len(lines):
            if lines[i].startswith("Tile"):
                id = int(lines[i][5:-1])
                i += 1
            tile = []
            while (i < len(lines)) and lines[i]:
                tile.append([int(l == "#") for l in lines[i]])
                i += 1
            if self.tile_dim is None:
                self.tile_dim = len(tile)
            tile = np.array(tile, dtype=np.uint8)
            self.tiles[id] = tile
            self.borders[id] = get_tile_borders(tile)
            self.n_tiles += 1
            i += 1
        for id in self.tiles:
            assert self.tiles[id].shape[0] == self.tile_dim
            assert self.tiles[id].shape[1] == self.tile_dim
        self.puzzle_dim = int(np.round(np.sqrt(self.n_tiles)))
        assert self.puzzle_dim ** 2 == self.n_tiles

        self.solution = np.zeros(
            (self.puzzle_dim, self.puzzle_dim), dtype=int
        )
        self._solve()

        lines = pattern.split("\n")
        self.pattern = []
        for line in lines:
            self.pattern.append([l == "#" for l in line])
        self.pattern = np.array(self.pattern, dtype=bool)

        im_size = self.puzzle_dim * (self.tile_dim - 2)
        self.image = np.zeros((im_size, im_size), dtype=np.uint8)
        self._assemble_image()
        self.n_monsters = self._mark_sea_monsters()

    def _check_direction_unique(self, tile, direction):
        id = get_tile_borders(tile, direction)
        return self.border_counter[id] == 1

    def _solve(self):
        borders = []
        for id in self.borders:
            borders.extend(self.borders[id])
        self.border_counter = Counter(borders)

        max_edge_count = 0
        for id in self.borders:
            n_unique = 0
            for border in self.borders[id]:
                c = self.border_counter[border]
                n_unique += c == 1
                if c > max_edge_count:
                    max_edge_count = c
            if n_unique == 2:
                self.corners.add(id)
            elif n_unique == 1:
                self.edges.add(id)
            elif n_unique == 0:
                self.interiors.add(id)
        assert max_edge_count == 2

        self.corner_prod = self.find_corner_product()

        dim_range = range(self.puzzle_dim)
        for col, row in product(dim_range, dim_range):
            self._solve_coord(row, col)

    def _solve_coord(self, row, col):
        unique_edges = (
            (row == 0)
            + (col == 0)
            + (row == self.puzzle_dim - 1)
            + (col == self.puzzle_dim - 1)
        )
        pieces = None
        if unique_edges == 0:
            pieces = self.interiors
        elif unique_edges == 1:
            pieces = self.edges
        if unique_edges == 2:
            pieces = self.corners

        orientations = list(product(ROTATIONS, [True, False]))
        cand = None
        tile = None
        for cand in pieces:
            valid = False
            for rot, flip in orientations:
                tile = rotate(self.tiles[cand], rot, flip)
                valid = True
                if row == 0:
                    valid &= self._check_direction_unique(tile, "up")
                else:
                    up = self.tiles[self.solution[row - 1, col]]
                    valid &= tile_to_id(tile, "up") == tile_to_id(
                        up, "down"
                    )

                if col == 0:
                    valid &= self._check_direction_unique(tile, "left")
                else:
                    left = self.tiles[self.solution[row, col - 1]]
                    valid &= tile_to_id(tile, "left") == tile_to_id(
                        left, "right"
                    )

                if row == self.puzzle_dim - 1:
                    valid &= self._check_direction_unique(tile, "down")
                if col == self.puzzle_dim - 1:
                    valid &= self._check_direction_unique(tile, "right")

                if valid:
                    break

            if valid:
                break
        self.solution[row, col] = cand
        pieces.remove(cand)
        self.tiles[cand] = tile

    def _assemble_image(self):
        im_col = 0
        for col in range(self.puzzle_dim):
            im_row = 0
            for row in range(self.puzzle_dim):
                tile = np.copy(self.tiles[self.solution[row, col]])[
                    1:-1, 1:-1
                ]
                self.image[
                    im_row : im_row + tile.shape[0],
                    im_col : im_col + tile.shape[1],
                ] = tile
                im_row += tile.shape[0]
            im_col += tile.shape[1]

    def _mark_sea_monsters(self):
        orientations = list(product(ROTATIONS, [True, False]))
        n_monsters = 0
        image = np.copy(self.image)
        for rot, flip in orientations:
            found_pattern = False
            image = rotate(self.image, rot, flip)
            n_monsters = 0
            for row in range(image.shape[0] - self.pattern.shape[0] + 1):
                for col in range(
                    image.shape[1] - self.pattern.shape[1] + 1
                ):
                    part = image[
                        row : row + self.pattern.shape[0],
                        col : col + self.pattern.shape[1],
                    ]
                    if part[self.pattern].all():
                        part[self.pattern] = 2
                        n_monsters += 1
                        found_pattern = True
            if found_pattern:
                break
        self.image = image
        return n_monsters

    def print_image(self, image):
        line = []
        char = {1: "#", 0: ".", 2: "O"}
        for row in range(image.shape[0]):
            s = ""
            for col in range(image.shape[1]):
                s += char[image[row, col]]
            line.append(s)
        line.append("")
        output = "\n".join(line)
        print(output)

    def find_corner_product(self):
        corner_prod = 1
        for corner in self.corners:
            corner_prod *= corner
        return corner_prod

    def find_habitat_roughness(self):
        return np.sum(self.image == 1)

def main():
    data_folder = Path(__file__).parent.resolve()
    data = data_folder.joinpath("input.txt").read_text()
    pattern = data_folder.joinpath("seamonster.txt").read_text()
    t = Image(data, pattern)
    print("Part 1")
    print("The product of the IDs of the four corner ")
    print(f"tiles is {t.corner_prod}")
    print()

    print("Part 2")
    print(f"The habitat's roughness is {t.find_habitat_roughness()}")
    print()

    print(f"Final image with {t.n_monsters} marked sea monsters:")
    t.print_image(t.image)

if __name__ == "__main__":
    main()


import re
import datetime
from .dean import Dean
from bs4 import BeautifulSoup
from freeclass.models import Classroom
from course.models import Course
from main import get_now_week

class FreeClassroom:
    session = Dean().dean_session
    week = get_now_week()
    building_dict = {
        "1": "思源楼",
        "2": "思源西楼",
        "3": "思源东楼",
        "4": "第九教学楼",
        "5": "第八教学楼",
        "6": "第五教学楼",
        "7": "第二教学楼",
        "8": "东区一教",
        "9": "东区二教",
        "10": "东教三楼",
        "11": "逸夫教学楼",
        "12": "机械楼",
        "13": "第十七号教学楼",
        "90": "科技大厦",
        "91": "天佑会堂",
        "92": "工程素质",
        "93": "综合实验楼",
        "94": "机械实验馆",
        "100": "学生活动服务中心"

    }
    status_dict = {
        '#fff': '空闲',
        '#394ed6': '考试占用',
        '#e46868': '排课占用',
        '#77bf6d': '预约',
        '#d8cc56': '个人占用'
    }

    def _get_building_room_info(self, building_no, page=1):
        url = 'https://dean.bjtu.edu.cn/classroom/timeholdresult/room_stat/'
        queries = {
            'zxjxjhh': '2019-2020-1-2',
            'zc': self.week,
            'jxlh': building_no,
            'jash': '',
            'submit': '查询+',
            'has_advance_query': '',
            'page': page,
            'perpage': 200
        }
        building = self.building_dict[str(building_no)]
        content = self.session.get(url, params=queries).content.decode()
        table = BeautifulSoup(content, 'html5lib').find('table', {'class': 'table table-bordered'})
        weekday = datetime.datetime.now().isoweekday()
        classroom_list = table.find_all('tr')[2:]
        for classroom in classroom_list:
            course_list = classroom.find_all('td', {'title': re.compile(r'星期{}'.format(weekday))})
            name = classroom.td.text.split()[0]
            room = Classroom.objects.get_or_create(name=name, building=building)[0]
            for course in course_list:
                no = int(course.get('title')[-2])
                color = course.get('style').replace('background-color: ', '')
                status = self.status_dict[color]
                if status != '空闲':
                    print(self.week, building, name, no, weekday)
                    course = Course.objects.filter(week__regex=r',{week},|^{week},|,{week}$'.format(week=self.week),
                                                   building=building, classroom=name, day_no=no,
                                                   day=weekday)
                    if course:
                        try:
                            teacher = course[0].teacher.name
                        except:
                            teacher = ''
                        status = course[0].name + '-' + teacher
                room.__setattr__('class{}'.format(no), status)
                print(name, no, status)
                room.save()

    def update(self):
        for building_no in self.building_dict.keys():
            self._get_building_room_info(building_no=building_no)

def run():
    FreeClassroom().update()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import pickle

# In[2]:

train=pd.read_csv('sales_train.csv')
items=pd.read_csv('items.csv')
shops=pd.read_csv('shops.csv')
cats=pd.read_csv('item_categories.csv')

# In[3]:

sns.boxplot(train.item_cnt_day)

# In[4]:

train = train[train.item_cnt_day < 1000]

# In[5]:

sns.boxplot(train.item_price)

# In[6]:

train = train[train.item_price < 300000]

# In[7]:

train = train[train.item_price > 0].reset_index(drop = True)
train.loc[train.item_cnt_day < 1, "item_cnt_day"] = 0

# In[8]:

train.loc[train.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
train.loc[train.shop_id == 40, 'shop_id'] = 39

# In[9]:

shops.loc[ shops.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shops["city"] = shops.shop_name.str.split(" ").map( lambda x: x[0] )
shops["category"] = shops.shop_name.str.split(" ").map( lambda x: x[1] )
shops.loc[shops.city == "!Якутск", "city"] = "Якутск"

# In[10]:

category = []
for cat in shops.category.unique():
    if len(shops[shops.category == cat]) >= 5:
        category.append(cat)
shops.category = shops.category.apply( lambda x: x if (x in category) else "other" )

# In[11]:

from sklearn.preprocessing import LabelEncoder
shops["shop_category"] = LabelEncoder().fit_transform( shops.category )
shops["shop_city"] = LabelEncoder().fit_transform( shops.city )
shops = shops[["shop_id", "shop_category", "shop_city"]]
shops=shops.drop([0,1,10,40]).reset_index(drop=True)

# In[12]:

cats["type_code"] = cats.item_category_name.apply( lambda x: x.split(" ")[0] ).astype(str)
cats.loc[ (cats.type_code == "Игровые")| (cats.type_code == "Аксессуары"), "category" ] = "Игры"

# In[13]:

category = []
for cat in cats.type_code.unique():
    if len(cats[cats.type_code == cat]) >= 5:
        category.append( cat )
cats.type_code = cats.type_code.apply(lambda x: x if (x in category) else "etc")

# In[14]:

cats.type_code = LabelEncoder().fit_transform(cats.type_code)
cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x) > 1 else x[0].strip())
cats["subtype_code"] = LabelEncoder().fit_transform( cats["subtype"] )
cats = cats[["item_category_id", "subtype_code", "type_code"]]

# In[15]:

import re
def name_correction(x):
    x = x.lower() # all letters lower case
    x = x.partition('[')[0] # partition by square brackets
    x = x.partition('(')[0] # partition by curly brackets
    x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x) # remove special characters
    x = x.replace('  ', ' ') # replace double spaces with single spaces
    x = x.strip() # remove leading and trailing white space
    return x

# In[16]:

items["name1"], items["name2"] = items.item_name.str.split("[", 1).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1).str

items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " ").str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " ").str.lower()

items = items.fillna('0')

items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))

items.name2 = items.name2.apply( lambda x: x[:-1] if x !="0" else "0")

# In[17]:

items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0] )
items.loc[(items.type == "x360") | (items.type == "xbox360") | (items.type == "xbox 360") ,"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply( lambda x: x.replace(" ", "") )
items.loc[ (items.type == 'pc' )| (items.type == 'pс') | (items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"

# In[18]:

group_sum = items.groupby(["type"]).agg({"item_id": "count"})
group_sum = group_sum.reset_index()
drop_cols = []
for cat in group_sum.type.unique():
    if group_sum.loc[(group_sum.type == cat), "item_id"].values[0] <40:
        drop_cols.append(cat)
items.name2 = items.name2.apply( lambda x: "other" if (x in drop_cols) else x )
items = items.drop(["type"], axis = 1)

# In[19]:

items.name2 = LabelEncoder().fit_transform(items.name2)
items.name3 = LabelEncoder().fit_transform(items.name3)

items.drop(["item_name", "name1"],axis = 1, inplace= True)

# In[20]:

price=train[['item_id','item_price']]
price=price.groupby(['item_id']).mean()
price=price.reset_index()

items=pd.merge(items,price,on=['item_id'],how='inner')
items=pd.merge(items,cats,on=['item_category_id'],how='inner')

# In[21]:

train=train.drop('date',axis=1)
train=train.groupby(['date_block_num', 'shop_id', 'item_id']).sum()
train.drop('item_price',axis=1,inplace=True)
train=train.reset_index()
train=train.rename(columns = {'item_cnt_day' : 'item_cnt_month'})

# In[22]:

from itertools import product
matrix = []
cols  = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
    sales = train[train.date_block_num == i]
    matrix.append( np.array(list( product( [i], sales.shop_id.unique(), sales.item_id.unique() ) ), dtype = np.int16) )

matrix = pd.DataFrame( np.vstack(matrix), columns = cols )
matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
matrix["item_id"] = matrix["item_id"].astype(np.int16)
matrix.sort_values( cols, inplace = True )

train=pd.merge(matrix,train,on=["date_block_num", "shop_id", "item_id"],how='left')
train=train.fillna(0)
train=pd.merge(train,items,on=['item_id'],how='inner')
train=pd.merge(train,shops,on=['shop_id'],how='inner')
train.sort_values(["date_block_num", "shop_id", "item_id"], inplace = True )
train=train.reset_index(drop=True)

# In[23]:

train["month"] = train["date_block_num"] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
train["days"] = train["month"].map(days).astype(np.int8)
train["year"] = 2013+(train["date_block_num"]/12).astype(np.int16)
train['month']=train['month']+1

# In[24]:

train=train[['date_block_num', 'shop_id', 'item_id', 'shop_category', 'shop_city',
       'item_category_id', 'name2', 'name3', 'subtype_code', 'type_code', 'days', 'month', 'year',
       'item_price', 'item_cnt_month']]

items.to_csv('items_data.csv')
shops.to_csv('shops_data.csv')

# In[25]:

X_train = train[train.date_block_num < 33].drop(['item_cnt_month'], axis=1)
Y_train = train[train.date_block_num < 33]['item_cnt_month']
X_valid = train[train.date_block_num == 33].drop(['item_cnt_month'], axis=1)
Y_valid = train[train.date_block_num == 33]['item_cnt_month']

# In[26]:

Y_train = Y_train.clip(0, 20)
Y_valid = Y_valid.clip(0, 20)

# In[27]:

from xgboost import XGBRegressor
from matplotlib.pylab import rcParams
rcParams['figure.figsize'] = 12, 4

# In[28]:

model = XGBRegressor(
    max_depth=10,
    n_estimators=1000,
    min_child_weight=0.5,
    colsample_bytree=0.8,
    subsample=0.8,
    eta=0.1,
#     tree_method='gpu_hist',
    seed=42)

model.fit(
    X_train,
    Y_train,
    eval_metric="rmse",
    eval_set=[(X_train, Y_train), (X_valid, Y_valid)],
    verbose=True,
    early_stopping_rounds = 20)

pickle.dump(model,open('model.pkl','wb'))

model=pickle.load(open('model.pkl','rb'))


from PythonPrograms.Oblig2_A_star.Vertex import Vertex
from PythonPrograms.Oblig2_A_star.Edge import Edge
from PythonPrograms.Oblig2_A_star.Graph import Graph

import pygame as pg

'''
The AStar class inherits the Graph class
'''

class AStar(Graph):
    """
    #
    #  delay: seconds between each iteration when visualizing is turned on
    # visual: Turns pygame visualization on/off
    #
    """

    def __init__(self, delay=0.001, visual=True):
        super().__init__()
        self.pygame = visual
        self.background = None
        self.LIGHTGREY = (180, 180, 180)
        self.DARKGREEN = (0, 255, 0)
        self.PINK = (255, 200, 200)
        self.GREEN = (200, 255, 200)
        self.WHITE = (245, 245, 245)
        self.BLACK = (0, 0, 0)
        self.RED = (255, 0, 0)
        self.BLUE = (0, 0, 255)
        self.delay = delay
        # walls define obstacles in grid, e.g. walls, boxes etc, by defining each position in grid that is part of an
        # obstacle
        self.walls = []

    '''
    #
    # Defines obstacles provided in removevertecies from the graph by removing all edges
    # to vertecies that is defined as an obstacle
    #
    '''

    def removeVertecies(self, removevertecies=[]):
        err_v = []
        err_e = []
        # Removing the vertecies
        for r in removevertecies:
            self.walls.append(self.vertecies[r])
            self.vertecies.pop(r)
        # Checking the edges ...
        for v in self.vertecies:
            vtex = self.vertecies[v]
            for edge in vtex.adjecent:
                vertex = edge.vertex
                if vertex.name not in self.vertecies:
                    err_v.append(vtex)
                    err_e.append(edge)
        for i in range(0, len(err_v)):
            err_v[i].adjecent.remove(err_e[i])
        return removevertecies

    '''
    #
    # Read in the list of obstacles (defined by tag "remove"), the startnode from which traversal is defined,
    # and the targetnode
    #
    '''

    def readLimitations(self, filename):
        import pandas as pd
        from collections import namedtuple

        columnnames = ['item', 'valuelist']
        df = pd.read_csv(filename, error_bad_lines=False,
                         encoding='latin-1', warn_bad_lines=False,
                         names=columnnames, header=None, sep=':')
        for i in range(0, 3):
            if df.iat[i, 0] == 'startvertex':
                startVertex = df.iat[i, 1]
            elif df.iat[i, 0] == 'targetvertex':
                targetVertex = df.iat[i, 1]
            elif df.iat[i, 0] == 'remove':
                removed = self.removeVertecies(df.iat[i, 1].split(';'))
        return startVertex, targetVertex, removed

    '''
    # Initialize pygame visualization, if visualization has been defined
    # Creates a dynamic visual grid according to the number of columns and rows
    # read/defined during initialization. The visual limitations are defined by 1000x1000 pixels
    '''

    def initPygame(self):
        if self.pygame:
            xmin = ymin = xmax = ymax = 0
            for v in self.vertecies:
                vertex = self.vertecies[v]
                x, y = vertex.position()
                if x < xmin:
                    xmin = x
                elif x > xmax:
                    xmax = x
                if y < ymin:
                    ymin = y
                elif y > ymax:
                    ymax = y

            pg.init()
            w, h = 1000, 1000
            self.xboxsize = int(w / ((xmax + 1) - xmin))
            self.yboxsize = int(w / ((ymax + 1) - ymin))

            w = self.xboxsize * ((int(w / self.xboxsize)))
            h = self.yboxsize * ((int(h / self.yboxsize)))
            self.background = pg.display.set_mode((w, h))
            background = self.background

            self.clock = pg.time.Clock()

            self.clock.tick()

            for c in range(0, (int(w / self.xboxsize)) + 1):
                for l in range(0, (int(h / self.yboxsize)) + 1):
                    pg.draw.rect(background, self.WHITE,
                                 (c * self.xboxsize, l * self.yboxsize, self.xboxsize, self.yboxsize))
                    pg.draw.line(background, self.BLACK, (c * self.xboxsize, l * self.yboxsize),
                                 (c * self.xboxsize + self.xboxsize, l * self.yboxsize))
                    pg.draw.line(background, self.BLACK, (c * self.xboxsize, l * self.yboxsize),
                                 (c * self.xboxsize, l * self.yboxsize + self.yboxsize))
                    pg.draw.line(background, self.BLACK, (c * self.xboxsize + self.xboxsize, l * self.yboxsize),
                                 (c * self.xboxsize + self.xboxsize, l * self.yboxsize + self.yboxsize))
                    pg.draw.line(background, self.BLACK, (c * self.xboxsize, l * self.yboxsize + self.yboxsize),
                                 (c * self.xboxsize + self.xboxsize, l * self.yboxsize + self.yboxsize))
            for wall in self.walls:
                self.pygameState(wall, self.BLACK)
            pg.display.flip()

    '''
    # Draw a box, representing the current vertex in the position defined by the name of the vertex.
    # The color-parameter defines the (R,G,B)-value of the box
    # If no visualization is used (self.pygame == False), no box is visualized
    '''

    def pygameState(self, current, color):
        import time
        if self.pygame:
            background = self.background
            x, y = current.position()
            pg.draw.rect(background, color, (x * self.xboxsize, y * self.yboxsize, self.xboxsize, self.yboxsize))
            pg.draw.line(background, self.BLACK, (x * self.xboxsize, y * self.yboxsize),
                         (x * self.xboxsize + self.xboxsize, y * self.yboxsize))
            pg.draw.line(background, self.BLACK, (x * self.xboxsize, y * self.yboxsize),
                         (x * self.xboxsize, y * self.yboxsize + self.yboxsize))
            pg.draw.line(background, self.BLACK, (x * self.xboxsize + self.xboxsize, y * self.yboxsize),
                         (x * self.xboxsize + self.xboxsize, y * self.yboxsize + self.yboxsize))
            pg.draw.line(background, self.BLACK, (x * self.xboxsize, y * self.yboxsize + self.yboxsize),
                         (x * self.xboxsize + self.xboxsize, y * self.yboxsize + self.yboxsize))
            if color not in [self.BLUE, self.RED, self.BLACK]:
                time.sleep(self.delay)
                pass
            pg.display.flip()

    '''
    #
    # Defining the heuristics used to calculate the estimated distance between the node being handled (startVertexName),
    # and the targetnode (targetVertexName)
    # Please note that the name of the vertecies are passed, not the vertex itself. The name is used for lookup
    # in the list of vertecies in the graph.
    # Further, the name of the vertex has the syntax: xNNyMM, where NN and MM are numerical and indicates column and row.
    # E.g. x4y15 means column 4 of row 15.
    # By identifying the column and row of a vertex, the estimated shorted distance between two vertecies may be
    # calculated using the Manhatten distance
    #
    '''

    def heuristics(self, startVertexName=None, targetVertexName=None):
        if not startVertexName or not targetVertexName:
            raise KeyError("VertexLookup need the names of the Vertecies addressed.")
        if startVertexName not in self.vertecies:
            raise KeyError("Node/Vertex defined as FROM-vertex is not present in graph")
        if targetVertexName not in self.vertecies:
            raise KeyError("Node/Vertex defined as TO-vertex is not present in graph")

        xstart, ystart = self.vertecies[startVertexName].position()
        xend, yend = self.vertecies[targetVertexName].position()
        #
        # Manhatten heuristics
        #
        dx = abs(xstart - xend)
        dy = abs(ystart - yend)
        D = 1
        return D * (dx + dy)

    '''
    #
    # The Dijkstra's algorithm has been adapted to support visualization as defined by the graph
    # It has further been adopted to present a targetVetrx even though Dijkstra's has no use of
    # it during execution. The tragetVertex is used only for visualization purposes.
    '''

    def Dijkstra(self, startVertexName=None, targetVertexName=None):
        self.initPygame()

        # Check to see that startvertex is in Graph
        if startVertexName not in self.vertecies:
            raise KeyError("Start node not present in graph")

        # Reset visited and previous pointer before running algorithm
        vertex = self.vertecies[startVertexName]
        vertex.distance = distance = weight = 0
        previous_node = None
        startNode = self.vertecies[startVertexName]
        toNode = self.vertecies[targetVertexName]

        #
        # Create priority queue, priority = current weight on edge ...
        # No duplicate edges in queue allowed
        #

        edge = Edge(0, vertex)
        from queue import PriorityQueue
        priqueue = PriorityQueue()

        # Defines enqueue/dequeue methods on priqueue
        def enqueue(data):
            priqueue.put(data)

        def dequeue():
            return priqueue.get()

        enqueue(edge)

        while not priqueue.empty():
            # Get the element with lowest priority (i.e. weight on edge)
            edge = dequeue()
            eyeball = edge.vertex
            self.pygameState(eyeball, self.GREEN)
            self.pygameState(startNode, self.BLUE)
            self.pygameState(toNode, self.RED)

            # If not visited previously, we need to define the distance
            if not eyeball.known:
                eyeball.distance = distance
                eyeball.previous = previous_node
            eyeball.known = True

            # If the vertex pointed to by the edge has an adjecency list, we need to iterate on it
            for adjecentedge in eyeball.adjecent:
                if not adjecentedge.vertex.known:
                    adjecentedge.vertex.distance = eyeball.distance + adjecentedge.weight
                    adjecentedge.vertex.previous = eyeball
                    adjecentedge.vertex.known = True
                    enqueue(adjecentedge)
                    self.pygameState(adjecentedge.vertex, self.PINK)
                else:
                    if adjecentedge.vertex.distance > eyeball.distance + adjecentedge.weight:
                        adjecentedge.vertex.distance = eyeball.distance + adjecentedge.weight
                        adjecentedge.vertex.previous = eyeball
                        enqueue(adjecentedge)

            self.pygameState(eyeball, self.LIGHTGREY)
        for n in self.getPath(startVertexName, targetVertexName):
            self.pygameState(n, self.DARKGREEN)
        return self.getPath(startVertexName, targetVertexName)

    '''
    ###############################################################################
    #
    # def AStarSearch(self, startVertexName = None, targetVertexName = None)
    #
    # Implement your code below.
    # Please note that no other parts of this code or provided code should be altered
    #
    ###############################################################################
    '''

    """
    #################################################
    #
    # Karaktersatt Obligatorisk Oppgave #2, A* Pathfinding
    #
    # Halil Ibrahim Keser
    #
    #
    # DOKUMENTASJON:
    # Kode dokumentasjon er skrevet i selve koden som kommentarer.
    #
    # Denne koden bruker mindre noder enn i fasiten. Og finner en kortere vei.
    #
    # Implementasjons ide og pseudo kode er hentet fra to nettsider + forelesning:
    #
    # http://theory.stanford.edu/~amitp/GameProgramming/AStarComparison.html
    # https://medium.com/@nicholas.w.swift/easy-a-star-pathfinding-7e6689c7f7b2
    #
    ##################################################
    """
    def AStarSearch(self, startVertexName=None, targetVertexName=None):
        """Returns vertex objects as a path from the given start to the given end in the given graph"""

        """
        The Initialization part
        """
        self.initPygame()  # initialize the game

        # Check to see that startvertex is in Grap.....................................................
        if startVertexName not in self.vertecies:
            raise KeyError("Start node not present in graph")

        # Reset visited and previous pointer before running algorithm..................................
        # Create start and end node as on Dijkstras funskjon
        vertex = self.vertecies[startVertexName]
        vertex.distance = 0
        start_node = self.vertecies[startVertexName]
        toNode = self.vertecies[targetVertexName]

        # Initialize open list, no need for closed list..................................................
        # Open_list as PriorityQueue for faster iteration of Big data
        from queue import PriorityQueue
        edge = Edge(0, vertex)  # Initialize the Edge with 0 weight
        open_list = PriorityQueue()
        closed_list = []

        def enqueue_open_list(data):
            open_list.put(data)

        def dequeue_open_list():
            return open_list.get()

        # Add the start node
        open_list.put((edge.vertex.g, edge.vertex))

        """
        The looping part, main part
        """
        # Loop until you find the end....................................................................
        while not open_list.empty():

            # Get the current node, current node here is the eyeball..................
            # Removing current from open list
            edge = dequeue_open_list()

            # Get the current index from edge, it is the first index of edge
            # current node that getting treated now
            # it contains edges of the current node.
            eyeball = edge[1]

            # Update the colours of eyeball, startnode and endnode...................
            self.pygameState(eyeball, self.GREEN)
            self.pygameState(start_node, self.BLUE)
            self.pygameState(toNode, self.RED)

            # Found the goal!.........................................................
            # if currentNode (eyeball) is the goal,
            # continue and make the eyeball.known as True
            # The path will be created later at the button. We have a function for it.
            if eyeball == toNode:
                break
            eyeball.known = True
            closed_list.append(eyeball)

            # Generate children. The children is the adjacents of the node ...........
            # If the vertex pointed to by the edge has an adjacent list,
            # we need to iterate on it
            # Loop through children
            for adjecentedge in eyeball.adjecent:
                # Loop through those we have not been through, and not known
                if not adjecentedge.vertex.known:  # not known, not in open list

                    if adjecentedge.vertex not in closed_list:  # not in closed_list
                        # Initialize the previous children
                        adjecentedge.vertex.previous = eyeball

                        # Create the f, g, and h values
                        # A - algorithm: F(n) = G(n) + H(n)
                        # We have a function for heuristics. We will use it to update vertex.h
                        adjecentedge.vertex.g = eyeball.g + adjecentedge.weight
                        adjecentedge.vertex.h = self.heuristics(adjecentedge.vertex.name, toNode.name)
                        adjecentedge.vertex.f = adjecentedge.vertex.g + adjecentedge.vertex.h

                        # Add the edge.node to open list as tuple
                        open_list.put((adjecentedge.vertex.f, adjecentedge.vertex))
                        adjecentedge.vertex.known = True

                        # Update the colours of known children.............................
                        self.pygameState(adjecentedge.vertex, self.PINK)

                    else:
                        # Update
                        # If edge.node weight greater than current.node weight + edge.node weight
                        if adjecentedge.vertex.g > (eyeball.g + adjecentedge.weight):
                            adjecentedge.vertex.previous = eyeball
                            adjecentedge.vertex.g = eyeball.g + adjecentedge.weight
                            adjecentedge.vertex.f = adjecentedge.vertex.g + adjecentedge.vertex.h
                        # Back-propagation
                        if not adjecentedge.vertex.known:
                            if adjecentedge.vertex in closed_list:
                                closed_list.remove(adjecentedge.vertex)
                                open_list.put((adjecentedge.vertex.f, adjecentedge.vertex))

            # Update the colours of known vertexes.....................................
            self.pygameState(eyeball, self.LIGHTGREY)

        # Get the path and, update the colours for path.....................................................
        # Return reversed path.
        # We will use the getPath function from the Graph class
        for n in self.getPath(startVertexName, targetVertexName):
            self.pygameState(n, self.DARKGREEN)
        return self.getPath(startVertexName, targetVertexName)

astar = AStar(delay=0, visual=True)

# astar.readFile('minigraf.txt')
# startVertexName, targetVertexName, removed = astar.readLimitations('minigraf_xtras.txt')
# astar.readFile('astjernegraf.txt')
# startVertexName, targetVertexName, removed = astar.readLimitations('xtras.txt')
astar.readFile('biggraph.txt')
startVertexName, targetVertexName, removed = astar.readLimitations('biggraph_xtras.txt')
# astar.readFile('AStarObligGraf.txt')
# startVertexName, targetVertexName, removed = astar.readLimitations('AStarObligGraf_xtras.txt')

# astar.Dijkstra(startVertexName, targetVertexName)
astar.AStarSearch(startVertexName, targetVertexName)

if astar.pygame:
    from pygame.locals import *

    while astar.pygame:
        for events in pg.event.get():
            if events.type == QUIT:
                exit(0)
    pg.quit()
else:
    print(astar.getPathAsString(startVertexName, targetVertexName))

{'pd': 'pandas', 'namedtuple': 'collections.namedtuple', 'time': 'time', 'PriorityQueue': 'queue.PriorityQueue'}


from flask import Response, Flask
from time import sleep
import random
import prometheus_client
from prometheus_client import Counter, Histogram

app = Flask('prometheus-app')

REQUESTS = Counter(
    'requests', 'Application Request Count',
    ['endpoint']
)

TIMER = Histogram(
    'slow', 'Slow Requests',
    ['endpoint']
)

@app.route('/metrics/')
def metrics():
    return Response(
        prometheus_client.generate_latest(),
        mimetype='text/plain; version=0.0.4; charset=utf-8'
    )

@app.route('/')
def index():
    REQUESTS.labels('/').inc()
    return '
 Przykładowa aplikacja Flask z obsługą systemu Prometheus
'

@app.route('/database/')
def database():
    with TIMER.labels('/database').time():
        sleep(random.uniform(1, 3))
    return '
Zakończono obsługę kosztownej operacji w bazie danych
'


import citations as cit
import json
from flask import Flask, request
from flask_cors import CORS

app = Flask(__name__)
cors = CORS(app, resources={r"/api/*": {"origins": "*"}})

years = [ 2015, 2016, 2017, 2018, 2019 ]

class_names = [
    "-",
    "A",
    "B",
    "C",
    "D",
    "E",
]

sectors = [
  "MAT01",
  "MAT02",
  "MAT03",
  "MAT05",
  "MAT06",
  "MAT07",
  "MAT08"
]

classifications = {
   "mcq": {
     "db": "scopus",
     "folder": "MCQ-SCOPUS",
     "sheet": "article_MCQ"
   },

   "sjr": {
     "db": "scopus",
     "folder": "SJR-SNIP",
     "sheet": "article_SJR"
   },

   "snip": {
     "db": "scopus",
     "folder": "SJR-SNIP",
     "sheet": "article_SNIP"
   },

   "mcq-wos": {
     "db": "wos",
     "folder": "MCQ-WOS",
     "sheet": "article_MCQ"
   }
}

sectors_db = {}

for sector in sectors:
  sectors_db[sector] = {}

  for (cl, classification) in classifications.items():
    print("Loading %s / %s ..." % (sector, cl), end = '')
    sectors_db[sector][cl] = { year: db for (year, db) in zip(
        years,
        map(lambda x : cit.get_journals_classification(x, classification["db"], sector, classification["folder"], classification["sheet"]), years)
    ) }
    print(" done.")

# We also build a set with all Journal names
print("Building index of Journal names ... ", end = '')
journal_names = set()
for sector in sectors:
  for classification in classifications:
    for year in years:
      for name in sectors_db[sector][classification][year].keys():
        journal_names.add(name)
print(" done.")

@app.route("/")
def index():
  return "Hello"

@app.route("/api/v1/search-journal", methods = [ "POST" ])
def search_journal():
  d = json.loads(request.data)
  name = d["journal"].lower()

  print("Searching %s" % name)
  print(journal_names)

  matching_j = list(filter(lambda x : d["journal"].lower() in x, journal_names))

  return json.dumps(matching_j)

@app.route("/api/v1/journals", methods = [ "GET" ])
def journals():
  return json.dumps(list(journal_names))

@app.route("/api/v1/classify", methods = [ "POST" ])
def classify():
  d = json.loads(request.data)

  journal = d["journal"].lower()
  scopus_citations = d["scopus-citations"]
  wos_citations = d["wos-citations"]
  year = int(d["year"])
  sector = d["sector"].replace("/", "")

  cl_mcq = cit.get_classification(journal, scopus_citations, sectors_db[sector]["mcq"][year])
  cl_sjr = cit.get_classification(journal, scopus_citations, sectors_db[sector]["sjr"][year])
  cl_snip = cit.get_classification(journal, scopus_citations, sectors_db[sector]["snip"][year])
  cl_wos = cit.get_classification(journal, wos_citations, sectors_db[sector]["mcq-wos"][year])

  return json.dumps({
    "mcq": class_names[cl_mcq+1],
    "sjr": class_names[cl_sjr+1],
    "snip": class_names[cl_snip+1],
    "mcq-wos": class_names[cl_wos+1]
  })


import time
import random
import zmq

context = zmq.Context()
zmq_socket = context.socket(zmq.PUSH)
zmq_socket.connect("tcp://127.0.0.1:5557")
# Start your result manager and workers before you start your producers

consumer_id = random.randrange(1,10005)
print("I am consumer #%s" % (consumer_id))
for num in range(20000):
    work_message = { 'num' : num }
    zmq_socket.send_json({'id': consumer_id})
    time.sleep(1)


# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# Author:  chenhao
# Date:  2020-09-13
# Description:
#-------------------------------------------------------------------------------
import torch
import torch.nn as nn
from torch.nn import functional as F

class textcnn(nn.Module):
    def __init__(self, param):
        # super(textcnn, self).__init__()
        super().__init__()
        vocab_size = param.vocab_size
        kernel_num = param.kernel_num
        kernel_size = param.kernel_size
        embedding_dim = param.embedding_dim
        class_num = param.class_num
        dropout = param.dropout

        self.embed = nn.Embedding(vocab_size, embedding_dim, padding_idx=1)
        self.conv0 = nn.Conv2d(1, kernel_num, (kernel_size[0], embedding_dim))
        self.conv1 = nn.Conv2d(1, kernel_num, (kernel_size[1], embedding_dim))
        self.conv2 = nn.Conv2d(1, kernel_num, (kernel_size[2], embedding_dim))

        self.dropout = nn.Dropout(dropout)

        self.line = nn.Linear(len(kernel_size)*kernel_num, class_num)

    @classmethod
    def from_dict(cls, obj_dict, **kwargs):
        return cls(**obj_dict, **kwargs)

    @staticmethod
    def conv_and_pool(x, conv):
        # x: (batch, 1, sentence_length,  )
        x = conv(x)
        # x: (batch, kernel_num, H_out, 1)
        x = F.relu(x.squeeze(3))
        # x: (batch, kernel_num, H_out)
        x = F.max_pool1d(x, x.size(2)).squeeze(2)
        #  (batch, kernel_num)
        return x

    def forward(self, x):
        x = self.embed(x)
        x = x.unsqueeze(1)
        out1 = self.conv_and_pool(x, self.conv0)
        out2 = self.conv_and_pool(x, self.conv1)
        out3 = self.conv_and_pool(x, self.conv2)
        x = torch.cat([out1, out2, out3], 1)
        x = self.dropout(x)
        x= self.line(x)
        # logit = F.softmax(x, dim=1)
        logit = F.log_softmax(x, dim=1)#这里由于最后用NLLLoss所以这里直接取log
        return logit


import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
import pdb

from src import config
from src.domain import model
from src.services_layer import unit_of_work
from src.adapters import repository

@pytest.fixture
def tear_down():
    session_factory = postgres_db_session()
    yield session_factory
    clean_table(session_factory)

def clean_table(session_factory):
    session = session_factory()
    session.query(model.Title).delete()
    session.commit()

def postgres_create_engine():
    db_uri = config.get_postgres_uri()
    return create_engine(db_uri)

def postgres_db_session():
    session_factory = sessionmaker(postgres_create_engine())
    return session_factory

def add_rows_to_db():
    titles = [
        model.Title(title="Marian"),
        model.Title(title="W pustyni i w puszczy"),
        model.Title(title="Armagedon"),
        model.Title(title="Kanibal"),
    ]
    engine = postgres_create_engine()
    with Session(engine) as session:
        for title in titles:
            session.add(title)
            session.commit()
            session.close()

def test_uow_can_retrive_title_from_source(tear_down):
    add_rows_to_db()
    session_factory = tear_down
    uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory)
    title_to_find = "Kanibal"
    with uow:
        finded_title = uow.repo.get(title_to_find)
        assert finded_title == model.Title(title=title_to_find)

def test_uow_add_row_to_source(tear_down):
    session_factory = tear_down
    add_rows_to_db()
    title_to_add = "Kosmos"
    uow = unit_of_work.SqlAlchemyUnitOfWork(session_factory)
    with uow:
        uow.repo.add(title_to_add)
        uow.commit()
    session = session_factory()
    repo = repository.SQLReopsitory(session)
    rows = repo.get_all_rows()
    assert rows == [
        model.Title(title="Marian"),
        model.Title(title="W pustyni i w puszczy"),
        model.Title(title="Armagedon"),
        model.Title(title="Kanibal"),
        model.Title(title=title_to_add),
    ]


#!/usr/bin/env python3

import argparse
import requests

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--url", type=str, default="http://0.0.0.0:8000/model")
    parser.add_argument("--host", type=str)
    parser.add_argument("--port", type=str)
    parser.add_argument("-n", "--name", type=str, required=True)
    parser.add_argument("-p", "--pre_trained_model_type", type=str, required=True)
    args = parser.parse_args()
    return args

def main():
    args = parse_args()
    if args.host is not None and args.port is not None:
        url = "http://" + args.host + ":" + args.port + "/model"
    else:
        url = args.url
    headers = {"accept": "application/json", "Content-Type": "application/json"}
    params = {"name": args.name, "pre_trained_model_type": args.pre_trained_model_type}
    response = requests.post(url=url, headers=headers, params=params)

    print(response.status_code, response.reason)
    print(response)
    print(response.json())

if __name__ == "__main__":
    main()


import torch
import torch.nn as nn
import torch.nn.functional as F

class CNN(nn.Module):

	def __init__(self, args, data, vectors):
		super(CNN, self).__init__()

		self.args = args

		self.word_emb = nn.Embedding(args.embed_num, args.embed_dim, padding_idx=1)
		# initialize word embedding with pretrained word2vec
		if args.mode != 'rand':
			self.word_emb.weight.data.copy_(torch.from_numpy(vectors))
		if args.mode in ('static', 'multichannel'):
			self.word_emb.weight.requires_grad = False
		if args.mode == 'multichannel':
			self.word_emb_multi = nn.Embedding(args.embed_num, args.embed_dim, padding_idx=1)
			self.word_emb_multi.weight.data.copy_(torch.from_numpy(vectors))
			self.in_channels = 2
		else:
			self.in_channels = 1

		# <unk> vectors is randomly initialized
		nn.init.uniform_(self.word_emb.weight.data[0], -0.05, 0.05)

		for filter_size in args.kernel_sizes:
			conv = nn.Conv1d(self.in_channels, args.kernel_num, args.embed_dim * filter_size, stride=args.embed_dim)
			setattr(self, 'conv_' + str(filter_size), conv)

		self.fc = nn.Linear(len(args.kernel_sizes) * args.kernel_num, args.class_num)

	def forward(self, batch):
		x = torch.t(batch.text)
		#x = torch.t(batch)
		batch_size, seq_len = x.size()

		conv_in = self.word_emb(x).view(batch_size, 1, -1)
		if self.args.mode == 'multichannel':
			#xembed = self.word_emb_multi(x)
			conv_in_multi = self.word_emb_multi(x).view(batch_size, 1, -1)
			conv_in = torch.cat((conv_in, conv_in_multi), 1)

		conv_result = [
			F.max_pool1d(F.relu(getattr(self, 'conv_' + str(filter_size))(conv_in)), seq_len - filter_size + 1).view(-1,
																													self.args.kernel_num)
			for filter_size in self.args.kernel_sizes]

		out = torch.cat(conv_result, 1)
		out = F.dropout(out, p=self.args.dropout, training=self.training)
		out = self.fc(out)

		return out


"""This module includes dependencies."""

from fastapi import Header, status, HTTPException

from settings import INTERNAL_CONFIGS

def verify_api_key(
    x_auth: str = Header(description="API key"),
    x_from_name: str = Header(description="Name of services from where request is came"),
):
    """Check if user is logged in."""
    api_config = INTERNAL_CONFIGS.get(x_from_name)
    if not api_config:
        raise HTTPException(
            status_code=status.HTTP_401_UNAUTHORIZED, detail="X-From-Name header is invalid"
        )

    if api_config["api_key"] != x_auth:
        raise HTTPException(
            status_code=status.HTTP_401_UNAUTHORIZED, detail="X-Auth header is invalid"
        )


import time
from logging import getLogger
from typing import Any, Dict, Optional, Tuple

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import confusion_matrix, f1_score
from torch.utils.data import DataLoader

from .meter import AverageMeter, ProgressMeter
from .metric import calc_accuracy

__all__ = ["train", "evaluate"]

logger = getLogger(__name__)

def do_one_iteration(
    sample: Dict[str, Any],
    model: nn.Module,
    criterion: Any,
    device: str,
    iter_type: str,
    optimizer: Optional[optim.Optimizer] = None,
) -> Tuple[int, float, float, np.ndarray, np.ndarray]:

    if iter_type not in ["train", "evaluate"]:
        message = "iter_type must be either 'train' or 'evaluate'."
        logger.error(message)
        raise ValueError(message)

    if iter_type == "train" and optimizer is None:
        message = "optimizer must be set during training."
        logger.error(message)
        raise ValueError(message)

    x = sample["img"].to(device)
    t = sample["class_id"].to(device)

    batch_size = x.shape[0]

    # compute output and loss
    output = model(x)
    loss = criterion(output, t)

    # measure accuracy and record loss
    accs = calc_accuracy(output, t, topk=(1,))
    acc1 = accs[0]

    # keep predicted results and gts for calculate F1 Score
    _, pred = output.max(dim=1)
    gt = t.to("cpu").numpy()
    pred = pred.to("cpu").numpy()

    if iter_type == "train" and optimizer is not None:
        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    return batch_size, loss.item(), acc1, gt, pred

def train(
    loader: DataLoader,
    model: nn.Module,
    criterion: Any,
    optimizer: optim.Optimizer,
    epoch: int,
    device: str,
    interval_of_progress: int = 50,
) -> Tuple[float, float, float]:

    batch_time = AverageMeter("Time", ":6.3f")
    data_time = AverageMeter("Data", ":6.3f")
    losses = AverageMeter("Loss", ":.4e")
    top1 = AverageMeter("Acc@1", ":6.2f")

    progress = ProgressMeter(
        len(loader),
        [batch_time, data_time, losses, top1],
        prefix="Epoch: [{}]".format(epoch),
    )

    # keep predicted results and gts for calculate F1 Score
    gts = []
    preds = []

    # switch to train mode
    model.train()

    end = time.time()
    for i, sample in enumerate(loader):
        # measure data loading time
        data_time.update(time.time() - end)

        batch_size, loss, acc1, gt, pred = do_one_iteration(
            sample, model, criterion, device, "train", optimizer
        )

        losses.update(loss, batch_size)
        top1.update(acc1, batch_size)

        # save the ground truths and predictions in lists
        gts += list(gt)
        preds += list(pred)

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # show progress bar per 50 iteration
        if i != 0 and i % interval_of_progress == 0:
            progress.display(i)

    # calculate F1 Score
    f1s = f1_score(gts, preds, average="macro")

    return losses.get_average(), top1.get_average(), f1s

def evaluate(
    loader: DataLoader, model: nn.Module, criterion: Any, device: str
) -> Tuple[float, float, float, np.ndarray]:
    losses = AverageMeter("Loss", ":.4e")
    top1 = AverageMeter("Acc@1", ":6.2f")

    # keep predicted results and gts for calculate F1 Score
    gts = []
    preds = []

    # calculate confusion matrix
    n_classes = loader.dataset.get_n_classes()
    c_matrix = np.zeros((n_classes, n_classes), dtype=np.int32)

    # switch to evaluate mode
    model.eval()

    with torch.no_grad():
        for sample in loader:
            batch_size, loss, acc1, gt, pred = do_one_iteration(
                sample, model, criterion, device, "evaluate"
            )

            losses.update(loss, batch_size)
            top1.update(acc1, batch_size)

            # keep predicted results and gts for calculate F1 Score
            gts += list(gt)
            preds += list(pred)

            c_matrix += confusion_matrix(
                gt,
                pred,
                labels=[i for i in range(n_classes)],
            )

    f1s = f1_score(gts, preds, average="macro")

    return losses.get_average(), top1.get_average(), f1s, c_matrix


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import pandas as pd

# In[3]:

df=pd.read_csv(r'C:\Users\msid4\OneDrive\Desktop\cc\music.tsv',sep='\t',encoding='utf-8',error_bad_lines=False)

# In[3]:

df.head()

# In[90]:

df.shape

# In[4]:

import findspark

# In[5]:

findspark.init('C:\spark')

# In[6]:

from pyspark.sql import SparkSession

# In[7]:

from pyspark import SparkConf,SparkContext

# In[8]:

from pyspark.sql import SQLContext

# In[9]:

conf=SparkConf().setMaster("local").setAppName("Assignment2")
sc=SparkContext(conf=conf)

# In[10]:

sqlContext = SQLContext(sc)

# In[11]:

data1 = sqlContext.read.format("com.databricks.spark.csv").options(header='true').option("delimiter", "\t").load(r"C:\Users\msid4\OneDrive\Desktop\cc\music.tsv")

# In[12]:

v2=data1.select('review_body','customer_id')

# In[13]:

v2.show()

# In[176]:

v3 = v2.groupBy("customer_id").count()

# In[177]:

v3.printSchema

# In[ ]:

v3.registerTempTable("v3")

# In[ ]:

sqlContext.sql("SELECT percentile_approx(count, 0.5) as median FROM v3").show()

# In[15]:

v4=data1.select('review_body','product_id')
v4.registerTempTable("v4")

# In[16]:

v6=v4.select('review_body','product_id').groupBy("product_id").count()
v6.show()

# In[17]:

v6.registerTempTable("v6")

# In[124]:

median_product=sqlContext.sql("SELECT percentile_approx(count, 0.5) as median FROM v6")

# In[107]:

from pyspark.sql import functions as F

# In[22]:

v6=v6.withColumnRenamed("product_id", "productid")

# In[23]:

data1 = data1.join(v6, data1.product_id == v6.productid)
data1.show()

# In[29]:

data1_output=data1.where('count>=2')

# In[30]:

v10=data1_output.select('product_id','review_body','count')
v10.registerTempTable("v10")

# In[33]:

top_10_products=sqlContext.sql("SELECT review_body,product_id FROM v10  order by count desc limit 10 ").show()

# In[ ]:

# In[78]:

regex_Tokenizer=RegexTokenizer(inputCol='review_body', outputCol='sentences',pattern='\\.')

# In[79]:

count_tokens=udf(lambda sentences:len(sentences),IntegerType())

# In[81]:

reg_tokenized=regex_Tokenizer.transform(data1_output)

# In[ ]:

reg_tokenized.show()

# In[ ]:

data2=reg_tokenized.withColumn('tokens',count_tokens(col('sentences'))).show()

# In[ ]:

filtered_data=data2.where('tokens>=2')


import re
from googletrans import Translator

# Dòng chứa thông tin sinh viên
student_info = "Full name: BUI HUY HOANG\nDoB\n11/12/2003\nGender: Male\nIntake\n2021-2024\nCourse: Bachelor\nMajor\nInformation and Communication Technology\nBI12-170\nValidity: 30/10/2024"

# Tìm và ghép các phần của đối tượng lại với nhau
full_name = re.search(r"Full name: (.+)", student_info).group(1)
dob = re.search(r"DoB\n(.+)", student_info).group(1)
gender = re.search(r"Gender: (.+)", student_info).group(1)
intake = re.search(r"Intake\n(.+)", student_info).group(1)
course = re.search(r"Course: (.+)", student_info).group(1)
major = re.search(r"Major\n(.+)\n", student_info).group(1)

# Tìm kiếm student ID trong chuỗi thông tin sinh viên
match = re.search(r"B[I][0-9]{2}-[0-9]{3}", student_info)
if match:
    student_id = match.group(0)
else:
    student_id = ""

validity = re.search(r"Validity: (.+)", student_info).group(1)

# Lưu thông tin vào biến student_info_oneline
student_info_oneline = f"Full name: {full_name}\nGender: {gender}\nDoB: {dob}\nCourse: {course}\nIntake: {intake}\nMajor: {major}\nStudent ID: {student_id}\nValidity: {validity}"

translator = Translator()

# Dịch sang tiếng Việt
result_vi = translator.translate(
    student_info_oneline, src='en', dest='vi').text

# Dịch sang tiếng Pháp
result_fr = translator.translate(
    student_info_oneline, src='en', dest='fr').text

# Hiển thị kết quả dịch

# Hiển thị biến student_info_oneline
print(student_info_oneline, end='\n\n\n')
print('Thông tin sinh viên (Tiếng Việt):')
print(result_vi, end='\n\n\n')
print('Thông tin sinh viên (Tiếng Pháp):')
print(result_fr)


import os,glob
from Bio import SeqIO
import statistics
import numpy as np
from Bio.Seq import Seq
import re

vcf_folder = '/scratch/users/anniz44/genomes/donor_species/vcf_round2/merge/details/'
output_folder = '/scratch/users/anniz44/genomes/donor_species/vcf_round2/BS/'
target_TF = '%s/target.TF.faa'%(output_folder)
ref_BS = '%s/ccpA_BS.fa'%(output_folder)
input_bs_file = '%s/binding_results_ccpA.txt'%(output_folder)
BS_folder = ''
distance = 5000 # extract 5kb neighbourhood
distance_TF = 5000 # TF with BS nearby

def find_strains(vcf_file,genomewithSNP):
    mut_strains = []
    for linesvcf in open(vcf_file, 'r'):
        if linesvcf.startswith('CHR'):
            linesvcf_set = linesvcf.split('\n')[0].split('\t')
            allgenome = linesvcf_set[9:]
            i = 1
            # find mutated strains
            for genome in allgenome:
                if str(i) in genomewithSNP:
                    mut_strains.append(genome)
                i += 1
            break
    return [mut_strains,allgenome]

def load_genes(input_faa):
    Mapping_loci_all = dict()
    for record in SeqIO.parse(input_faa, 'fasta'):
        record_id = str(record.id)
        contig = '_'.join(record_id.split('_')[0:-1])
        description = str(record.description).replace(' ', '').split('#')
        Mapping_loci_all.setdefault(contig, [])
        Mapping_loci_all[contig].append([int(description[1]) - 1,
                                         int(description[2]) - 1, record_id])
    return Mapping_loci_all

def contig_length(CHR):
    try:
        total_length = CHR.split('size')[1]
    except IndexError:
        try:
            total_length = CHR.split('length_')[1].split('_cov')[0]
        except IndexError:
            total_length = 10000
    return int(total_length)

def load_BS_old(BS_file,Mapping_loci_all):
    allBS = []
    allBS.append('BS\tpvalue\tlocus\tcontig\tstrand\ttargetgane\tlocusgene\n')
    target_gene_list = dict()
    BS_loci = dict()
    i = 0
    for lines in open(BS_file, 'r'):
        if not lines.startswith('#') and not lines.startswith('motif_id') and lines != '\n':
            lines_set = lines.split('\n')[0].split('\t')
            pvalue = lines_set[7]
            contig, locus1, locus2, strand = lines_set[2:6]
            if contig_length(contig) >= 5000:
                i += 1
                BS_loci.setdefault(contig, [])
                locus1 = int(locus1)
                locus2 = int(locus2)
                targetgene = ''
                locus_target = 0
                if contig in Mapping_loci_all:
                    for locus in Mapping_loci_all[contig]:
                        locusre1, locusref2, genename = locus
                        if locus2 <= locusref2 and targetgene == '':
                            targetgene = genename
                            locus_target = locusre1
                    seq = lines_set[9]
                    if targetgene != '':
                        if strand == '-':
                            # the gene before
                            gene_locus = int(targetgene.split('_')[-1])
                            if gene_locus > 1:
                                targetgene = '_'.join(targetgene.split('_')[0:-1]) + '_%s' % (
                                        int(targetgene.split('_')[-1]) - 1)
                    else:
                        targetgene = '%s_1' % (contig)
                    allBS.append('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (
                        seq, pvalue, locus1, contig, strand, targetgene, locus_target))
                    target_gene_list.setdefault(targetgene, set())
                    target_gene_list[targetgene].add(seq)
                    BS_loci[contig].append([locus1, locus2, seq, targetgene,pvalue])
    f1 = open('%s.BS.txt' % (output_file), 'w')
    f1.write(''.join(allBS))
    f1.close()
    aa_output = []
    for record in SeqIO.parse(input_faa, 'fasta'):
        record_id = str(record.id)
        if record_id in target_gene_list:
            for seq in target_gene_list[record_id]:
                aa_output.append('>%s_%s_C_%s_G_%s\n%s\n' % (
                    seq, lineage, record_id.split('_')[1], record_id.split('_')[-1], str(record.seq)))
                select_seq_faa.setdefault(seq,'>%s_%s_C_%s_G_%s\n%s\n' % (
                        seq, lineage, record_id.split('_')[1], record_id.split('_')[-1], str(record.seq)))
    f1 = open('%s.BS.faa' % (output_file), 'w')
    f1.write(''.join(aa_output))
    f1.close()
    return BS_loci

def load_BS(BS_file,Mapping_loci_all,BS_loci,TF_locus):
    allBS = []
    allBS.append('BS\tpvalue\tlocus\tcontig\tstrand\ttargetgane\tlocusgene\n')
    target_gene_list = dict()
    i = 0
    genomename = os.path.split(BS_file)[-1].split('.fimo.tsv')[0]
    check_TF_BS = False
    TF_locus_genome = TF_locus.get(genomename,[])
    TF_contig = [i[0] for i in TF_locus_genome]
    if TF_locus_genome != []:
        # check TF
        for lines in open(BS_file, 'r'):
            if not check_TF_BS and not lines.startswith('#') and not lines.startswith('motif_id') and lines != '\n':
                lines_set = lines.split('\n')[0].split('\t')
                contig, locus1, locus2, strand = lines_set[2:6]
                if contig in TF_contig:
                    locus1 = int(locus1)
                    locus2 = int(locus2)
                    for contig_TF,locus1_TF,locus2_TF,gene in TF_locus_genome:
                        if contig_TF == contig and locus1_TF - distance_TF <= locus2 and locus2_TF + distance_TF >= locus1:
                            #print(locus1_TF - distance_TF,locus2_TF + distance_TF,locus1,locus2,gene)
                            check_TF_BS = True
                            break
        if not check_TF_BS:
            print('not BS near TF for genome %s'%(genomename))
        else:
            # BS near TF
            print('found BS near TF for genome %s' % (genomename))
            pass_genome.append(genomename)
            # load BS
            for lines in open(BS_file, 'r'):
                if not lines.startswith('#') and not lines.startswith('motif_id') and lines != '\n':
                    lines_set = lines.split('\n')[0].split('\t')
                    pvalue = lines_set[7]
                    contig, locus1, locus2, strand = lines_set[2:6]
                    i += 1
                    locus1 = int(locus1)
                    locus2 = int(locus2)
                    targetgene = ''
                    locus_target = 0
                    if contig in Mapping_loci_all:
                        for locus in Mapping_loci_all[contig]:
                            locusre1, locusref2, genename = locus
                            if locus2 <= locusref2 and targetgene == '':
                                targetgene = genename
                                locus_target = locusre1
                        seq = lines_set[9]
                        if targetgene != '':
                            if strand == '-':
                                # the gene before
                                gene_locus = int(targetgene.split('_')[-1])
                                if gene_locus > 1:
                                    targetgene = '_'.join(targetgene.split('_')[0:-1]) + '_%s' % (
                                            int(targetgene.split('_')[-1]) - 1)
                        else:
                            targetgene = '%s_1' % (contig)
                        allBS.append('%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (
                            seq, pvalue, locus1, contig, strand, targetgene, locus_target))
                        target_gene_list.setdefault(targetgene, set())
                        target_gene_list[targetgene].add(seq)
                        BS_loci.setdefault(seq, [])
                        BS_loci[seq].append([genomename, locus1, locus2, '%s_C_%s_G_%s' % (genomename,
                                                                                targetgene.split('_')[1],
                                                                                targetgene.split('_')[-1])])
                # output BS and gene target
            # output BS and gene target
            f1 = open('%s.BS.txt' % (output_file), 'w')
            f1.write(''.join(allBS))
            f1.close()
            aa_output = []
            for record in SeqIO.parse(input_faa, 'fasta'):
                record_id = str(record.id)
                if record_id in target_gene_list:
                    for seq in target_gene_list[record_id]:
                        record_seq = str(record.seq)
                        aa_output.append('>%s_%s_C_%s_G_%s\n%s\n' % (
                            seq, genomename, record_id.split('_')[1], record_id.split('_')[-1], record_seq))
                        select_seq_faa.setdefault(seq,['',set()])
                        if record_seq not in select_seq_faa[seq][-1]:
                            select_seq_faa[seq][-1].add(record_seq)
                            select_seq_faa[seq][0]+=('>%s_%s_C_%s_G_%s\n%s\n' % (
                                    seq, genomename, record_id.split('_')[1], record_id.split('_')[-1], record_seq))
            f1 = open('%s.BS.faa' % (output_file), 'w')
            f1.write(''.join(aa_output))
            f1.close()
    return BS_loci

def annotate(fasta_output):
    cutoff = 0.7
    cmd_cluster = ('%s -sort length -cluster_fast %s -id %s -centroids %s.cluster.aa -uc %s.uc -threads %s\n'
                           % ('usearch', fasta_output, cutoff, fasta_output,
                              fasta_output, 40))
    os.system(cmd_cluster)
    fasta_output = fasta_output + '.cluster.aa'
    cutoff = 0.01
    database = '/scratch/users/mit_alm/database/eggnog/xaa.hmm'
    cmds = ('hmmsearch --tblout %s.eggnog.1.txt --cpu 40 -E %s %s %s\n') % (
    fasta_output, cutoff, database, fasta_output)
    database = '/scratch/users/mit_alm/database/eggnog/xab.hmm'
    cmds += ('hmmsearch --tblout %s.eggnog.2.txt --cpu 40 -E %s %s %s\n') % (
        fasta_output, cutoff, database, fasta_output)
    database = '/scratch/users/mit_alm/database/eggnog/xac.hmm'
    cmds += ('hmmsearch --tblout %s.eggnog.3.txt --cpu 40 -E %s %s %s\n') % (
        fasta_output, cutoff, database, fasta_output)
    f1 = open(output_file_BS + '.eggnog.sh', 'w')
    f1.write(
        '#!/bin/bash\nsource ~/.bashrc\nexport LD_LIBRARY_PATH=/scratch/users/anniz44/bin/pro/lib/gsl-2.6:/scratch/users/anniz44/bin/pro/lib/glibc-2.14-build:/scratch/users/anniz44/bin/pro/lib/:/scratch/users/anniz44/bin/miniconda3/lib:$LD_LIBRARY_PATH\n%s' % (
            cmds))
    f1.close()

def allele_freq_to_allele(genotype, ALT_set):
    genotype = [int(i) for i in genotype.split(':')[-1].split(',')]
    Major_ALT = '-'
    if sum(genotype) > 0:
        ALT_set_sample = dict()
        ALT_frq_set = set()
        for i in range(0,len(ALT_set)):
            ALT_frq = int(genotype[i])
            ALT_set_sample.setdefault(ALT_frq, set())
            ALT_set_sample[ALT_frq].add(ALT_set[i])
            ALT_frq_set.add(ALT_frq)
        ALT_frq_set = sorted(ALT_frq_set, reverse=True)
        for ALT_frq in ALT_frq_set:
            for alleles in ALT_set_sample[ALT_frq]:
                if Major_ALT == '-':
                    Major_ALT = alleles
    return Major_ALT

def find_mutations_on_BS_old(vcf_file2,mut_strains,BS_loci):
    BS_SNP_output = []
    mut_strains_set = []
    wild_type_set = []
    allseqout = set()
    genomewithSNPvcf = []
    genomewithnoSNPvcf = []
    samplefile = vcf_file2.replace('.all.raw.vcf.all','.all.raw.vcf.filtered.samplename.txt')
    for lines in open(samplefile, 'r'):
        # set sample name
        sample_set = lines.split('\n')[0].split('\t')
        i = 9
        for samples in sample_set:
            genome = samples
            if genome in mut_strains:
                genomewithSNPvcf.append(i)
                mut_strains_set.append('M.%s' % (genome))
            elif genome in allgenome:
                genomewithnoSNPvcf.append(i)
                wild_type_set.append('W.%s' % (genome))
            i += 1
    BS_SNP_output.append('BS\tBS_in_ref\tCHR\tPOS\tPOS_on_BS\tTarget\tpvalue\tMutated_alleles\tWildtype_alleles\t%s\t%s\n' % (
        '\t'.join(mut_strains_set),
        '\t'.join(
            wild_type_set)
    ))
    #print(mut_strains,mut_strains_set,wild_type_set)
    for linesvcf in open(vcf_file2, 'r'):
        linesvcf_set = linesvcf.split('\n')[0].split('\t')
        if not linesvcf.startswith('#'):
            contig, POS = linesvcf_set[0:2]
            if contig in BS_loci:
                POS = int(POS)
                for BSpos in BS_loci[contig]:
                    BSpos1, BSpos2, seq, targetgene,pvalue = BSpos
                    if POS >= BSpos1 - distance and POS <= BSpos2 + distance :
                        REF = linesvcf_set[3]
                        ALT_set = [REF]
                        ALT = linesvcf_set[4]
                        for a_ALT in ALT.split(','):
                            if a_ALT != '.':
                                ALT_set.append(a_ALT)
                        # BS site
                        Mut_allele = []
                        Wild_allele = []
                        # Major_ALT in mutated strains
                        for i in genomewithSNPvcf:
                            genotype = linesvcf_set[i]
                            Major_ALT = allele_freq_to_allele(genotype, ALT_set)
                            Mut_allele.append(Major_ALT)
                        # Major_ALT in wild type strains
                        for i in genomewithnoSNPvcf:
                            genotype = linesvcf_set[i]
                            Major_ALT = allele_freq_to_allele(genotype, ALT_set)
                            Wild_allele.append(Major_ALT)
                        Mut_allele_set = sorted(set(Mut_allele))
                        Wild_allele_set = sorted(set(Wild_allele))
                        if any(i not in Wild_allele_set for i in Mut_allele_set):
                            # different alleles
                            inref = False
                            if seq in Ref:
                                inref = True
                            BS_SNP_output.append('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (
                                seq, inref, contig, POS, POS - BSpos1 + 1, targetgene,pvalue,
                                ';'.join(list(Mut_allele_set)),
                                ';'.join(list(Wild_allele_set)),
                                '\t'.join(Mut_allele),
                                '\t'.join(Wild_allele)
                            ))
                            allseqout.add(select_seq_faa.get(seq, ''))
    f1 = open('%s.BSsum.txt' % (output_file_BS),'w')
    f1.write(''.join(BS_SNP_output))
    f1.close()
    if len(allseqout)>0:
        f1 = open('%s.BS.faa' % (output_file_BS), 'w')
        f1.write(''.join(list(allseqout)))
        f1.close()
        annotate('%s.BS.faa' % (output_file_BS))

def find_mutations_on_BS(vcf_file2,mut_strains,BS_loci_co_assembly):
    BS_SNP_output = []
    mut_strains_set = []
    wild_type_set = []
    allseqout = set()
    genomewithSNPvcf = []
    genomewithnoSNPvcf = []
    samplefile = vcf_file2.replace('.all.raw.vcf.all','.all.raw.vcf.filtered.samplename.txt')
    for lines in open(samplefile, 'r'):
        # set sample name
        sample_set = lines.split('\n')[0].split('\t')
        i = 9
        for samples in sample_set:
            genome = samples
            if genome in mut_strains:
                genomewithSNPvcf.append(i)
                mut_strains_set.append('M.%s' % (genome))
            elif genome in allgenome:
                genomewithnoSNPvcf.append(i)
                wild_type_set.append('W.%s' % (genome))
            i += 1
    BS_SNP_output.append('BS\tBS_in_ref\tCHR\tPOS\tPOS_on_BS\tMutated_alleles\tWildtype_alleles\t%s\t%s\n' % (
        '\t'.join(mut_strains_set),
        '\t'.join(
            wild_type_set)
    ))
    #print(mut_strains,mut_strains_set,wild_type_set)
    finished_set = []
    for linesvcf in open(vcf_file2, 'r'):
        linesvcf_set = linesvcf.split('\n')[0].split('\t')
        if not linesvcf.startswith('#'):
            contig, POS = linesvcf_set[0:2]
            if contig in BS_loci_co_assembly:
                POS = int(POS)
                for seq_loci_sub,seq in BS_loci_co_assembly[contig]:
                    if [seq_loci_sub,seq] not in finished_set:
                        finished_set.append([seq_loci_sub,seq])
                        BSpos1, BSpos2 = seq_loci_sub
                        if POS >= BSpos1 - distance and POS <= BSpos2 + distance:
                            REF = linesvcf_set[3]
                            ALT_set = [REF]
                            ALT = linesvcf_set[4]
                            for a_ALT in ALT.split(','):
                                if a_ALT != '.':
                                    ALT_set.append(a_ALT)
                            # BS site
                            Mut_allele = []
                            Wild_allele = []
                            # Major_ALT in mutated strains
                            for i in genomewithSNPvcf:
                                genotype = linesvcf_set[i]
                                Major_ALT = allele_freq_to_allele(genotype, ALT_set)
                                Mut_allele.append(Major_ALT)
                            # Major_ALT in wild type strains
                            for i in genomewithnoSNPvcf:
                                genotype = linesvcf_set[i]
                                Major_ALT = allele_freq_to_allele(genotype, ALT_set)
                                Wild_allele.append(Major_ALT)
                            Mut_allele_set = sorted(set(Mut_allele))
                            Wild_allele_set = sorted(set(Wild_allele))
                            if any(i not in Wild_allele_set for i in Mut_allele_set):
                                # different alleles
                                inref = False
                                if seq in Ref:
                                    inref = True
                                #targetgene = [i[-1] for i in BS_loci[seq]]
                                BS_SNP_output.append('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (
                                    seq, inref, contig, POS, POS - BSpos1 + 1,
                                    ';'.join(list(Mut_allele_set)),
                                    ';'.join(list(Wild_allele_set)),
                                    '\t'.join(Mut_allele),
                                    '\t'.join(Wild_allele)
                                ))
                                allseqout.add(select_seq_faa.get(seq, '')[0])
    f1 = open('%s.BSsum.txt' % (output_file_BS), 'w')
    f1.write(''.join(BS_SNP_output))
    f1.close()
    if len(allseqout)>0:
        f1 = open('%s.BS.faa' % (output_file_BS), 'w')
        f1.write(''.join(list(allseqout)))
        f1.close()
        annotate('%s.BS.faa' % (output_file_BS))

def find_TF(input_faa,blast_out,alloutput):
    withTF = False
    genomename = os.path.split(input_faa)[-1].split('.faa')[0]
    #donor = genomename.split('_')[0]
    TF_set = dict()
    for lines in open(blast_out):
        lines_set = lines.split('\t')
        query,reference = lines_set[0:2]
        TF_set.setdefault(query,[])
        TF_set[query].append(reference)
    for record in SeqIO.parse(input_faa, 'fasta'):
        record_id = str(record.id)
        if record_id in TF_set:
            for reference in TF_set[record_id]:
                #if donor in reference:
                    description = str(record.description).replace(' ', '').split('#')
                    alloutput.append('%s\t%s\t%s\t%s\t%s\n'%(genomename,record_id,description[1],description[2],reference))
                    withTF = True
    if not withTF:
        print('TF not found for %s'%(input_faa))
    return alloutput

def load_TF(TF_locus_file):
    TF_locus = dict()
    for lines in open(TF_locus_file,'r'):
        lines_set = lines.split('\t')
        genome, gene,locus1,locus2 = lines_set[0:4]
        genome = genome.split('.all')[0]
        contig = '_'.join(gene.split('_')[0:-1])
        TF_locus.setdefault(genome,[])
        TF_locus[genome].append([contig,int(locus1),int(locus2),gene])
    return TF_locus

def find_target_BS(mut_strains, BS_loci,allgenome):
    BS_select = set()
    wild_strains = [i for i in allgenome if i not in mut_strains]
    if any(genome in mut_strains for genome in pass_genome) and any(genome in wild_strains for genome in pass_genome):
        # mutated genome and wild strains all passed
        print('found enough genomes passing the criterion of BS near TF')
        for seq in BS_loci:
            allgenomewithBS = [i[0] for i in BS_loci[seq]]
            # only in mutated
            if (any(genome in mut_strains for genome in allgenomewithBS) and not any(
                    genome in wild_strains for genome in allgenomewithBS)):
                BS_select.add(seq)
    else:
        print('not enough genomes passing the criterion of BS near TF')
        BS_select = set()
    return BS_select

def find_BS_coassembly(co_assembly,BS_select):
    BS_loci_co_assembly = dict()
    for record in SeqIO.parse(co_assembly, 'fasta'):
        record_id = str(record.id)
        record_seq = str(record.seq)
        for seq in BS_select:
            seq_loci = [[m.start()+1, m.end()+1] for m in re.finditer(seq, record_seq)]
            if seq_loci!=[]:
                BS_loci_co_assembly.setdefault(record_id,[])
                for seq_loci_sub in seq_loci:
                    BS_loci_co_assembly[record_id].append([seq_loci_sub,seq])
            seq_rev = str(Seq(seq).reverse_complement())
            seq_loci = [[m.start()+1, m.end()+1] for m in re.finditer(seq_rev, record_seq)]
            if seq_loci!=[]:
                if seq_rev in select_seq_faa:
                    seq = seq_rev
                BS_loci_co_assembly.setdefault(record_id, [])
                for seq_loci_sub in seq_loci:
                    BS_loci_co_assembly[record_id].append([seq_loci_sub, seq])
    return BS_loci_co_assembly

def TF_near_BS(TF_locus,BS_file_all):
    BS_TF = []
    for BS_file in BS_file_all:
        check_TF_BS = False
        genome = os.path.split(BS_file)[-1].split('.BS.txt')[0]
        TF_locus_genome = TF_locus.get(genome, [])
        TF_contig = [i[0] for i in TF_locus_genome]
        for lines in open(BS_file,'r'):
            if not lines.startswith('BS'):
                lines_set = lines.split('\n')[0].split('\t')
                seq,contig,locus = lines_set[0:3]
                if contig in TF_contig:
                    locus = int(locus)
                    for contig_TF, locus1_TF, locus2_TF,gene in TF_locus_genome:
                        if contig_TF == contig and locus1_TF - distance_TF <= locus and locus2_TF + distance_TF >= locus:
                            check_TF_BS = True
                            BS_TF.append('%s\t%s\t%s\t%s\t%s\n'%(genome,contig,seq,locus,gene))
        if not check_TF_BS:
            print('BS not found near TF %s'%(genome))
    f1 = open('%s/allBSnearTF.txt' % (output_folder), 'w')
    f1.write(''.join(BS_TF))
    f1.close()

# find all TF
try:
    f1 = open('%s/allTFloci.txt' % (output_folder), 'r')
except FileNotFoundError:
    cmds = 'export LD_LIBRARY_PATH=/scratch/users/anniz44/bin/pro/lib/gsl-2.6:/scratch/users/anniz44/bin/pro/lib/glibc-2.14-build:/scratch/users/anniz44/bin/pro/lib/:/scratch/users/anniz44/bin/miniconda3/lib:$LD_LIBRARY_PATH\n'
    os.system(cmds)
    alloutput = []
    allinput_faa = glob.glob('%s/*/*.faa'%(output_folder))
    for input_faa in allinput_faa:
        cmds = (
            "diamond blastp --query %s --db %s.dmnd --out %s.TF.txt --id 80 --outfmt 6 --max-target-seqs 10 --evalue 1e-1 --threads 40\n"%
            (
                input_faa,target_TF,input_faa,
            ))
        os.system(cmds)
        alloutput = find_TF(input_faa, input_faa + '.TF.txt', alloutput)
    f1 = open('%s/allTFloci.txt' % (output_folder), 'w')
    f1.write(''.join(alloutput))
    f1.close()

# load TF loci
TF_locus = load_TF('%s/allTFloci.txt' % (output_folder))

# find BS predicted in genomes near TF
try:
    f1 = open('%s/allBSnearTF.txt' % (output_folder), 'r')
except FileNotFoundError:
    BS_file_all = glob.glob('%s/*/*.BS.txt'%(output_folder))
    #BS_file_all = glob.glob('%s/*.BS.txt' % (output_folder))
    TF_near_BS(TF_locus, BS_file_all)
    print('finished processing TF near BS')

# load ref
Ref = []
if ref_BS != 'None':
    for record in SeqIO.parse(ref_BS, 'fasta'):
        Ref.append(str(record.seq))

# process each SNP
for lines in open(input_bs_file,'r'):
    if not lines.startswith('AA_POS_ref'):
        try:
            lines_set = lines.split('\t')
            lineage = lines_set[4].split('__')[0].replace('CL', 'clustercluster')
            species = lines_set[4].split('_')[0]
            donor = lines_set[5]
            SNP = lines_set[3]
            output_file_BS = '%s/%s/%s_%s_%s' % (output_folder, BS_folder,species, donor, SNP)
            print(output_file_BS)
            try:
                f1 = open('%s.BSsum.txt' % (output_file_BS), 'r')
            except FileNotFoundError:
                select_seq_faa = dict()
                # find genome names
                vcf_file = '%s/%s%s' % (
                    vcf_folder, lineage, '.all.parsi.fasta.linktrunc.sum.txt')
                genomewithSNP = lines_set[-9].split(';')
                mut_strains, allgenome = find_strains(vcf_file, genomewithSNP)
                print('process %s mutated strains %s'%(lineage,mut_strains))
                lineage = lineage.split('.donor')[0]
                # process fino results
                donor_species = '%s_%s'%(species,donor)
                BS_file_all = glob.glob('%s/%s/%s/*.fimo.tsv'%(output_folder,BS_folder,donor_species))
                BS_loci = dict()
                pass_genome = []
                for BS_file in BS_file_all:
                    output_file = BS_file.split('.fimo.tsv')[0]
                    input_faa = '%s/%s/%s.faa'%(output_folder,donor_species,os.path.split(output_file)[-1])
                    # load all gene position
                    Mapping_loci_all = load_genes(input_faa)
                    # load BS
                    BS_loci = load_BS(BS_file, Mapping_loci_all,BS_loci,TF_locus)
                # find targeted BS only in mutated or only in WT
                BS_select = find_target_BS(mut_strains, BS_loci,allgenome)
                if BS_select!= set():
                    # find genome locus for BS_select
                    co_assembly = '%s/co-assembly/%s.all.spades2.fasta'%(output_folder,lineage)
                    BS_loci_co_assembly = find_BS_coassembly(co_assembly,BS_select)
                    print(BS_loci_co_assembly)
                    # find mutations on BS in mutated strains
                    vcf_file2 = '%s/moredetails/%s.all.raw.vcf.all' % (
                        vcf_folder, lineage)
                    print(vcf_file2)
                    find_mutations_on_BS(vcf_file2, mut_strains, BS_loci_co_assembly)
            # filter BS
            BS_file = '%s.BSsum.txt' % (output_file_BS)
            # filter BS
            BS_set = set()
            for lines in open(BS_file, 'r'):
                if not lines.startswith('BS'):
                    lines_set = lines.split('\t')
                    BS = lines_set[0]
                    POS_on_BS = int(lines_set[4])
                    Mut = lines_set[5]
                    if POS_on_BS >= -20 and POS_on_BS <= 20:
                        # BS with changes on BS itself and not just loss
                        BS_set.add(BS)
            BS_file_out = []
            for lines in open(BS_file, 'r'):
                if lines.startswith('BS'):
                    BS_file_out.append(lines)
                elif lines != '':
                    seq = lines.split('\t')[0]
                    if seq in BS_set:
                        # BS with changes on BS itself and not just loss
                        BS_file_out.append(lines)
            f1 = open('%s.BSsum.filtered.txt' % (output_file_BS), 'w')
            f1.write(''.join(BS_file_out))
            f1.close()
        except FileNotFoundError:
            pass

alleggnog = glob.glob(os.path.join(output_folder, '%s/*eggnog.sh'%(BS_folder)))
if alleggnog!= []:
    f1 = open(os.path.join(output_folder, '%s/allanno.sh'%(BS_folder)), 'w')
    f1.write('#!/bin/bash\nsource ~/.bashrc\n')
    for sub_scripts in alleggnog:
        f1.write('jobmit %s %s small1\n' % (sub_scripts, os.path.split(sub_scripts)[-1]))
    f1.close()
    print('please run %s/%s/allanno.sh'%(output_folder,BS_folder))


"""\
Reference Directive for L.E.A.R.N
=================================

Author: Akshay Mestry <xa@mes3.dev>
Created on: Friday, July 28 2023
Last updated on: Monday, July 31 2023

This module provides a custom directive for L.E.A.R.N's custom theme,
that allows authors and contributors to add a dedicated references
section to their documentation. The directive identifies and collects all
the links present within the documentation page and collates them in a
dedicated section for easy reference.

L.E.A.R.N's external references are created using the ``references``
directive, which is included as part of this project. The directive is
implemented below, and is available to use by authors and contributors
when building the documentation.

.. note:: This directive is compatible with Sphinx 3.0 and higher.

.. note:: It is encouraged to mention references for credibility.

.. note::

    This module is designed specifically for the Sphinx's L.E.A.R.N
    theme, hence the directive may not be available or may be
    implemented differently for different themes. Please consult the
    documentation for more information.

The references can be styled using CSS. The class encapsulating the
reference is ``learn-references`` and the references itself are
``references``.

.. code-block:: css

    .learn-references ol {
      ...
      text-align: left;
      font-size: 1.375rem;
      ...
    }

    .learn-references>ol>li>p {
      ...
      text-align: left;
      ...
    }

    .references-border {
      ...
      font-size: 1.375rem;
      ...
    }

This will style the list of references with required colors and format.

.. versionadded:: 1.0.2
    Added support for creating a dedicated references section through
    the ``references`` directive. The directive handles all the
    external links used in the current document and puts them together
    as a list.
"""

from __future__ import annotations

import typing as t

import jinja2
from docutils import nodes
from docutils.nodes import Element
from docutils.nodes import Node
from docutils.parsers.rst import Directive
from docutils.parsers.rst import roles
from sphinx.writers.html import HTMLTranslator

__all__ = [
    "References",
    "ReferencesNode",
    "depart_references_html",
    "visit_references_html",
]

REFERENCES_TEMPLATE: t.Final[jinja2.Template] = jinja2.Template(
    """\

References

        {% for (title, link) in content %}

{{ title }}, {{ link }}

        {% endfor %}

    """
)

class ReferencesNode(Element):
    pass

class References(Directive):
    """A directive class for the ``references`` directive.

    This class allows using a custom ``references`` directive and maps
    the source reStructuredText to ``ReferencesNode`` element doctree
    node.

    By using the ``references`` directive, it allows contributors or
    authors to render a dedicated references section for the provided
    content. This class' rendering HTML behavior is further extended
    using another Python function, ``visit_references_div``.

    Example::

        .. references::

            arxiv >> Papers are published on arXiv every day.
            python >> We will be using Python all the time.

    :var has_content: A boolean flag to allow content in the directive,
                      defaults to ``True``.
    :var final_argument_whitespace: A boolean flag, may the final
                                    argument contain whitespace, set to
                                    ``True``.
    """

    has_content: bool = True
    final_argument_whitespace: bool = True

    def run(self) -> list[Node]:
        """Create node from the reStructuredText source.

        This method processes the directive's arguments, options and
        content, and return a list of Docutils/Sphinx nodes that will be
        inserted into the document tree at the point where the directive
        was encountered.

        :return: List of Docutils node for ``references`` directive.
        """
        self.options["class"] = ["learn-references"]
        roles.set_classes(self.options)
        node = ReferencesNode("\n".join(self.content), **self.options)
        self.add_name(node)
        self.state.nested_parse(self.content, self.content_offset, node)
        return [node]

def visit_references_html(self: HTMLTranslator, node: ReferencesNode) -> None:
    """Node visitor function which maps ``ReferencesNode`` element to the
    HTML output.

    This function allows the rendering of the references of a topic on the
    webpage. It relies on ``Jinja`` templating to perform the expansion
    and rendering of the HTML source code.
    """
    content: list[tuple[str, str]] = []
    pairs: dict[str, str] = {}
    corrections = node.children[0].astext().splitlines()
    node.remove(node.children[0])
    for correction in corrections:
        doc, new = correction.split(" >> ")
        pairs[doc] = new
    for target in node._document.findall(nodes.target):
        if target.hasattr("refuri") and target.indirect_reference_name:
            title = target["names"][0]
            content.append((pairs.get(title, title.title()), target["refuri"]))
    html_src = REFERENCES_TEMPLATE.render(content=content)
    self.body.append(f"{self.starttag(node, 'div')}{html_src}")

def depart_references_html(self: HTMLTranslator, node: ReferencesNode) -> None:
    self.body.append("
")


import eval7
from tqdm import tqdm
import traceback
import sys
from pprint import pprint
sys.path.insert(0, "../")
sys.path.insert(0, "../.libs")
from pokereval import PokerEval
pokereval = PokerEval()
from pprint import pprint
import numpy as np
import operator
from tqdm import tqdm
import datetime
import re

rank = ['A','K','Q','J','T','9','8','7','6','5','4','3','2']

def hand_init(init):
    init_hand = {}
    for idx1,i in enumerate(rank):
        for idx2,j in enumerate(rank):
            if idx1==idx2:
                hand0 = rank[idx1] + rank[idx2]
                init_hand[hand0]=init
            if idx1<idx2:
                hand1 = rank[idx1] + rank[idx2] + 'o'
                init_hand[hand1]=init
                hand2 = rank[idx1] + rank[idx2] + 's'
                init_hand[hand2]=init
    # print(len(init_hand))
    # pprint(init_hand)
    return init_hand

def listToStringHand(cardList):
    card1=eval7.Card(cardList[0])
    card2=eval7.Card(cardList[1])
    if card1.rank < card2.rank:
        temp=card1
        card1=card2
        card2=temp
    if card1.suit != card2.suit:
        if card1.rank != card2.rank:
            hand = str(card1)[0]+str(card2)[0]+'o'
        else:
            hand = str(card1)[0]+str(card2)[0]
    else:
        hand = str(card1)[0]+str(card2)[0]+'s'
    return hand

def good_hand(hand, benchmark):
    new_hand = [i for i in hand if hand[i]>=benchmark]
    return new_hand

def handlist(hand):
    new_hand = [i for i in hand]
    return new_hand

def hand_proceed(file):
    with open(file, "r") as ins:
        pos=[]
        for line in ins:
            pos.append(line.split(','))
    # cardStringList1 = [i[0] for i in pos ]
    # hand = [i[0] for i in pos if float(re.sub('\s','',i[1]))>0]
    # hand = percentile(cardStringList1,30)
    hand = {}
    for i in pos:
        hand[i[0]] = float(re.sub('\s','',i[1]))

    return hand

def percentile(cardStringList, n):
    hr=[]
    for i in range(len(cardStringList)):
        hr += eval7.HandRange(cardStringList[i]).hands

    i=len(hr)
    index=np.percentile(np.arange(i), n, interpolation='nearest')
    # print hr[index]
    # print( str(hr[index][0][0]) )
    if str(hr[index][0][0])[1] != str(hr[index][0][1])[1]:
        if str(hr[index][0][0])[0]!= str(hr[index][0][1])[0]:
            hand = str(hr[index][0][0])[0]+str(hr[index][0][1])[0] +'o'
        else:
            hand = str(hr[index][0][0])[0]+str(hr[index][0][1])[0]
    else:
        hand = str(hr[index][0][0])[0]+str(hr[index][0][1])[0] +'s'
    # print(hand)
    new_list=cardStringList[:(cardStringList.index(hand)+1)]
    # print(new_list)
    return new_list

########################################################
hand_preceed = 'proceed'
rounds=150
R1C=1  # Bet structure: Assume raise followed by call.
n = 1  # number of players except hero and bb.
iteration=200 # for each hand range eg. 89O
noise=np.zeros((rounds, 169))
benchmark = 3 # good hand benchmark for investigation.
##########################################################

if hand_preceed == 'proceed' :
    hand0=hand_proceed("./log3/guru_initial_palyers=3_betStruct=1_rounds=20_'2019-11-12T23:55:36'.txt")
    list_hands = good_hand(hand0, benchmark)     # update hands better than benchmark.
    # list_hands = handlist(hand_init(benchmark))  # update all hands

else:
    hand0=hand_init(benchmark)
    list_hands = handlist(hand_init(benchmark))   # update all hands

# pprint(good_hand(hand0))
# pprint(list_hands)

for round in tqdm(range(rounds)):
    report={}
    for index in range(len(list_hands)):
    # hand['32o']=-100
        hr = eval7.HandRange(list_hands[index])
        good_hand1 = good_hand(hand0, benchmark)
        number = iteration*len(hr.hands)
        sum=0
        # pprint(hand0)
        for itr in range(number):
            # good_hand1 = good_hand(hand0)
            # pprint(good_hand1)

            deck = eval7.Deck()
            deck.shuffle()
            cards = deck.cards
            # print(list_hands[index])
            # hr = eval7.HandRange(list_hands[index])
            # pprint(hr.hands)  #[((Card("Qc"), Card("7c")), 1.0),
            # pprint(hr.string) #'Q7s'

            select_one = np.random.randint(len(hr.hands))
            remove_card1 = hr.hands[select_one][0][0]
            remove_card2 = hr.hands[select_one][0][1]
            # print(select_one)
            # print(remove_card1,remove_card2)
            cards.remove(remove_card1)
            cards.remove(remove_card2)
            # print(len(cards))
            dealt_cards = cards[:21]
            # print(len(dealt_cards))
            hands = [str(i) for i in dealt_cards]

            hand1=[str(remove_card1),str(remove_card2)]
            r1=[hands[2*n+5],hands[2*n+6]]
            '''
            For simplicity: Assume BB share good hand with other player.
            '''
            # n = 8
            # bb_play = False
            player=['','','','','','','']
            for i in range(n):
                list1 = [hands[2*i],hands[1+2*i]]
                hand = listToStringHand(list1)
                if hand in good_hand1:
                    player[i] = [hands[2*i],hands[1+2*i]]
                    # if i==0:
                    #     bb_play = True

            participation = [i for i in player if i!='']
            pockets = [hand1, r1] + participation
            board = [hands[2*n],hands[2*n+1],hands[2*n+2],hands[2*n+3],hands[2*n+4]]
            # pprint(hand1)
            # pprint(hands)
            # pprint(pockets)
            # pprint(board)
            # print(len(good_hand1))

            result = pokereval.poker_eval(game='holdem', pockets=pockets, board=board)
            # pprint(result['eval'])
            m = len(participation)
            pot = R1C*(m + 2)
            # if bb_play:
            #     pot = R1C*(m + 1)
            # else:
            #     pot = R1C*(m + 1) + 1   # 1BB from bb as dead money.
            winners = [i for i in result['eval'] if int(i['ev'])>0]

            # for index, value in enumerate(result['eval']):
            #     if value['ev']>0:
            #         # print(pockets[index], "wins", pot*1.0/len(winners))
            #         stringHand=listToStringHand(pockets[index])
            #         # print(stringHand)
            #         hand0[stringHand]+=pot*1.0/len(winners)
            #     else:
            #         # print(pockets[index], "lose 1")
            #         stringHand=listToStringHand(pockets[index])
            #         hand0[stringHand]-=1
            # pprint(good_hand(hand0))
            # pprint(len(good_hand(hand0)))

            if  result['eval'][0]['ev']>0:
                sum = sum + pot*1.0/len(winners) - 1*R1C
            else:
                sum = sum - 1*R1C
            # Add diminishing noise factor.
            # print(np.random.uniform(-1,1)/(itr+1))
            # hand0[list_hands[index]]+=sum + 0.1*np.random.uniform(-1,1)/(round*itr+1)
        noise[round][index]= 0.5*np.random.uniform(-1,1)/(round+1)
        hand0[list_hands[index]]+=sum*1.0/number  #+ noise[round][index]
        report[list_hands[index]]= hand0[list_hands[index]] #,noise[round][index]]
        # print(list_hands[index],"=",hand0[list_hands[index]])
        # print(list_hands[index])
    # pprint(hand0)
    # pprint(good_hand(hand0))
    # pprint(len(good_hand(hand0)))
    pprint(report)

hand1={}
for i in list_hands:
    hand1[i] = hand0[i] #/rounds

rank_hand0 = sorted(hand1.items(), key=operator.itemgetter(1), reverse=True)
pprint(rank_hand0)
pprint(len(good_hand(hand0, benchmark)))

t1 = datetime.datetime.now()
f=open("./log3/guru_"+hand_preceed+"_palyers="+str(n+2)+"_"+"betStruct="+str(R1C)+"_"+"rounds="+str(rounds)+"_"+repr(t1.isoformat()[:19])+".txt","a+")
for i in range(len(rank_hand0)):
    f.write("%.4s, %s\n" % (str(rank_hand0[i][0]), str(rank_hand0[i][1])))
f.close()