import cv2
import numpy as np

# 검출할 영역(관심영역)의 가로,세로 크기 지정. <175 x 125 = 21,875 사각형 면적 크기>
ROI_w = 175
ROI_h = 125

# ROI 좌표 지정.
ROI_list = [
    [(1300, 5)], # 까지 첫번째 줄

    [(5, 150)],
    [(250, 150)],
    [(500, 150)],
    [(750, 150)],
    [(990, 150)],
    [(1235, 150)], # 까지 두번째줄

    [(340, 340)],
    [(620, 340)],
    [(890, 340)],
    [(1160, 340)], # 까지 세번째 줄

    [(470, 560)],
    [(1070, 560)], # 까지 네번째 줄
]

def nothing(x):
    pass

# 면적을 퍼센트로 구해줌
def areaToPercentage(area):
    global ROI_h, ROI_w
    percentage = 100 * area / (ROI_w * ROI_h)
    return percentage  # [%]단위

# Contour 내부의 구멍을 모두 채워줌.
def fullfill_inside(im_in):

    im_in = cv2.bitwise_not(im_in)

    # Threshold.
    # Set values equal to or above 220 to 0.
    # Set values below 220 to 255.
    th, im_th = cv2.threshold(im_in, 220, 255, cv2.THRESH_BINARY_INV);
    cv2.imshow('im_th', im_th)

    # Copy the thresholded image.
    im_floodfill = im_th.copy()
    # Mask used to flood filling.
    # Notice the size needs to be 2 pixels than the image.
    h, w = im_th.shape[:2]
    mask = np.zeros((h + 2, w + 2), np.uint8)

    # Floodfill from point (0, 0)
    cv2.floodFill(im_floodfill, mask, (0, 0), 255);

    # Invert floodfilled image
    im_floodfill_inv = cv2.bitwise_not(im_floodfill)

    # Combine the two images to get the foreground.
    im_out = im_th | im_floodfill_inv

    return im_out

# 트렉바 생성
cv2.namedWindow('trackbarwindow')
cv2.createTrackbar('Lower_R', 'trackbarwindow', 0, 255, nothing)
cv2.createTrackbar('Lower_G', 'trackbarwindow', 80, 255, nothing)
cv2.createTrackbar('Lower_B', 'trackbarwindow', 0, 255, nothing)
cv2.createTrackbar('Upper_R', 'trackbarwindow', 255, 255, nothing)
cv2.createTrackbar('Upper_G', 'trackbarwindow', 255, 255, nothing)
cv2.createTrackbar('Upper_B', 'trackbarwindow', 255, 255, nothing)
cv2.createTrackbar('Gaussian_Blur', 'trackbarwindow', 0, 255, nothing)

while(1):
    img = cv2.imread('Test.PNG')

    # ROI의 사각형 마스크를 씌우기 위해
    h, w = img.shape[:2]
    rec_mask = np.zeros((h, w), np.uint8)
    for i in range(len(ROI_list)):
        frame = cv2.rectangle(rec_mask, ROI_list[i][0], (ROI_list[i][0][0] + ROI_w, ROI_list[i][0][1] + ROI_h), (255, 255, 255), -1)
    #cv2.imshow('recmask', rec_mask)

    l_r = cv2.getTrackbarPos('Lower_R', 'trackbarwindow')
    l_g = cv2.getTrackbarPos('Lower_G', 'trackbarwindow')
    l_b = cv2.getTrackbarPos('Lower_B', 'trackbarwindow')
    u_r = cv2.getTrackbarPos('Upper_R', 'trackbarwindow')
    u_g = cv2.getTrackbarPos('Upper_G', 'trackbarwindow')
    u_b = cv2.getTrackbarPos('Upper_B', 'trackbarwindow')
    Gauss_blur = cv2.getTrackbarPos('Gaussian_Blur', 'trackbarwindow')

    # 가우시안 필터, 짝수일때 +1, 홀수일때 그데로
    if Gauss_blur % 2 == 1:     # 홀수
        pass
    else:                       # 짝수
        Gauss_blur = Gauss_blur + 1
    # 가우시안 필터
    img = cv2.GaussianBlur(img, (Gauss_blur, Gauss_blur), 0)

    frame = img
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

    # ROI 설정구역 Display
    for i in range(len(ROI_list)):
        frame = cv2.rectangle(frame, ROI_list[i][0], (ROI_list[i][0][0] + ROI_w, ROI_list[i][0][1] + ROI_h), (150, 50, 150), 5)
        cv2.putText(frame, 'ROI%d' % (i + 1), (ROI_list[i][0][0], ROI_list[i][0][1] - 3), cv2.FONT_HERSHEY_SIMPLEX, 1,
                    (150, 50, 150), 2, cv2.LINE_AA)

    # define range of blue color in HSV (B,G,R)
    # lower_blue = np.array([110,50,50])
    lower = np.array([l_b, l_g, l_r])
    upper = np.array([u_r, u_g, u_b])

    # Threshold the HSV image to get all range
    mask = cv2.inRange(hsv, lower, upper)

    # Bitwise-AND mask and original image
    res = cv2.bitwise_and(frame, frame, mask= mask)
    # 사각마스크로 관심영역 범위만 추출함.
    res_rec = cv2.bitwise_and(mask, mask, mask= rec_mask)

    # # 사각 마스크 위에 원본 씌우기
    # frame_rec_res = cv2.bitwise_and(frame, frame, mask=res_rec)
    # cv2.imshow('rec_mask+result', frame_rec_res)

    # Closed shape에 채우기.
    im_floodfill = res_rec.copy()                         # 스레솔드 마스크 복사.
    im_floodfill = fullfill_inside(im_floodfill)          # 내부 Hole이 채워진 이미지 리턴.

    # 컨투어 찾기, 중심점 찾기
    # cnt = 0
    try:
        _, contours, _ = cv2.findContours(im_floodfill, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)  # 컨투어 찾기
    except:
        contours, _ = cv2.findContours(im_floodfill, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)  # 버전에 따른 오류 예방

    if len(contours) != 0:
        for contour in contours:
            if (cv2.contourArea(contour) > 0) and (cv2.contourArea(contour) < 21875):  # **필요한 면적을 찾아 중심점 표시
                ball_area = cv2.contourArea(contour)
                print(ball_area)
                mom = contour
                cv2.drawContours(frame, [mom], 0, (0, 255, 0), 2)
                M = cv2.moments(mom)
                cx_origin = int(M['m10'] / M['m00'])
                cy_origin = int(M['m01'] / M['m00'])
                #cv2.circle(frame, (cx_origin, cy_origin), 10, (0, 0, 255), -1) # Center of puck

                # cv2.putText(frame, 'area : %d' % (ball_area), (ROI_list[cnt][0][0], ROI_list[cnt][0][1]+30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255),
                #                2, cv2.LINE_AA)
                ball_area = areaToPercentage(ball_area)

                if ball_area > 50:
                    cv2.putText(frame, '%d%%' % (ball_area), (cx_origin - 30, cy_origin), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 3, cv2.LINE_AA)
                else:
                    cv2.putText(frame, 'NG', (cx_origin - 30, cy_origin), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 3, cv2.LINE_AA)
                # cnt = cnt + 1
                # print(cx_origin, cy_origin) # 중심점값읽기

    print("****" * 30)
    # cv2.putText(frame, 'Number of Gasket : %d' % (cnt), (300, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv2.LINE_AA)

    #cv2.imshow('blur', blur)
    cv2.imshow('inside fullfill', im_floodfill)
    cv2.imshow('frame', frame)
    cv2.imshow('mask', mask)
    cv2.imshow('res', res)
    k = cv2.waitKey(5) & 0xFF
    if k == 27:
        break

cv2.destroyAllWindows()


import torch

class RankSort(torch.autograd.Function):
    @staticmethod
    def forward(ctx, logits, targets, delta_RS=0.50, eps=1e-10):

        classification_grads=torch.zeros(logits.shape).cuda()

        #Filter fg logits
        fg_labels = (targets > 0.)
        fg_logits = logits[fg_labels]
        fg_targets = targets[fg_labels]
        fg_num = len(fg_logits)

        #Do not use bg with scores less than minimum fg logit
        #since changing its score does not have an effect on precision
        threshold_logit = torch.min(fg_logits)-delta_RS
        relevant_bg_labels=((targets==0) & (logits>=threshold_logit))

        relevant_bg_logits = logits[relevant_bg_labels]
        relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda()
        sorting_error=torch.zeros(fg_num).cuda()
        ranking_error=torch.zeros(fg_num).cuda()
        fg_grad=torch.zeros(fg_num).cuda()

        #sort the fg logits
        order=torch.argsort(fg_logits)
        #Loops over each positive following the order
        for ii in order:
            # Difference Transforms (x_ij)
            fg_relations=fg_logits-fg_logits[ii]
            bg_relations=relevant_bg_logits-fg_logits[ii]

            if delta_RS > 0:
                fg_relations=torch.clamp(fg_relations/(2*delta_RS)+0.5,min=0,max=1)
                bg_relations=torch.clamp(bg_relations/(2*delta_RS)+0.5,min=0,max=1)
            else:
                fg_relations = (fg_relations >= 0).float()
                bg_relations = (bg_relations >= 0).float()

            # Rank of ii among pos and false positive number (bg with larger scores)
            rank_pos=torch.sum(fg_relations)
            FP_num=torch.sum(bg_relations)

            # Rank of ii among all examples
            rank=rank_pos+FP_num

            # Ranking error of example ii. target_ranking_error is always 0. (Eq. 7)
            ranking_error[ii]=FP_num/rank

            # Current sorting error of example ii. (Eq. 7)
            current_sorting_error = torch.sum(fg_relations*(1-fg_targets))/rank_pos

            #Find examples in the target sorted order for example ii
            iou_relations = (fg_targets >= fg_targets[ii])
            target_sorted_order = iou_relations * fg_relations

            #The rank of ii among positives in sorted order
            rank_pos_target = torch.sum(target_sorted_order)

            #Compute target sorting error. (Eq. 8)
            #Since target ranking error is 0, this is also total target error
            target_sorting_error= torch.sum(target_sorted_order*(1-fg_targets))/rank_pos_target

            #Compute sorting error on example ii
            sorting_error[ii] = current_sorting_error - target_sorting_error

            #Identity Update for Ranking Error
            if FP_num > eps:
                #For ii the update is the ranking error
                fg_grad[ii] -= ranking_error[ii]
                #For negatives, distribute error via ranking pmf (i.e. bg_relations/FP_num)
                relevant_bg_grad += (bg_relations*(ranking_error[ii]/FP_num))

            #Find the positives that are misranked (the cause of the error)
            #These are the ones with smaller IoU but larger logits
            missorted_examples = (~ iou_relations) * fg_relations

            #Denominotor of sorting pmf
            sorting_pmf_denom = torch.sum(missorted_examples)

            #Identity Update for Sorting Error
            if sorting_pmf_denom > eps:
                #For ii the update is the sorting error
                fg_grad[ii] -= sorting_error[ii]
                #For positives, distribute error via sorting pmf (i.e. missorted_examples/sorting_pmf_denom)
                fg_grad += (missorted_examples*(sorting_error[ii]/sorting_pmf_denom))

        #Normalize gradients by number of positives
        classification_grads[fg_labels]= (fg_grad/fg_num)
        classification_grads[relevant_bg_labels]= (relevant_bg_grad/fg_num)

        ctx.save_for_backward(classification_grads)

        return ranking_error.mean(), sorting_error.mean()

    @staticmethod
    def backward(ctx, out_grad1, out_grad2):
        g1, =ctx.saved_tensors
        return g1*out_grad1, None, None, None

class aLRPLoss(torch.autograd.Function):
    @staticmethod
    def forward(ctx, logits, targets, regression_losses, delta=1., eps=1e-5):
        classification_grads=torch.zeros(logits.shape).cuda()

        #Filter fg logits
        fg_labels = (targets == 1)
        fg_logits = logits[fg_labels]
        fg_num = len(fg_logits)

        #Do not use bg with scores less than minimum fg logit
        #since changing its score does not have an effect on precision
        threshold_logit = torch.min(fg_logits)-delta

        #Get valid bg logits
        relevant_bg_labels=((targets==0)&(logits>=threshold_logit))
        relevant_bg_logits=logits[relevant_bg_labels]
        relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda()
        rank=torch.zeros(fg_num).cuda()
        prec=torch.zeros(fg_num).cuda()
        fg_grad=torch.zeros(fg_num).cuda()

        max_prec=0
        #sort the fg logits
        order=torch.argsort(fg_logits)
        #Loops over each positive following the order
        for ii in order:
            #x_ij s as score differences with fgs
            fg_relations=fg_logits-fg_logits[ii]
            #Apply piecewise linear function and determine relations with fgs
            fg_relations=torch.clamp(fg_relations/(2*delta)+0.5,min=0,max=1)
            #Discard i=j in the summation in rank_pos
            fg_relations[ii]=0

            #x_ij s as score differences with bgs
            bg_relations=relevant_bg_logits-fg_logits[ii]
            #Apply piecewise linear function and determine relations with bgs
            bg_relations=torch.clamp(bg_relations/(2*delta)+0.5,min=0,max=1)

            #Compute the rank of the example within fgs and number of bgs with larger scores
            rank_pos=1+torch.sum(fg_relations)
            FP_num=torch.sum(bg_relations)
            #Store the total since it is normalizer also for aLRP Regression error
            rank[ii]=rank_pos+FP_num

            #Compute precision for this example to compute classification loss
            prec[ii]=rank_pos/rank[ii]
            #For stability, set eps to a infinitesmall value (e.g. 1e-6), then compute grads
            if FP_num > eps:
                fg_grad[ii] = -(torch.sum(fg_relations*regression_losses)+FP_num)/rank[ii]
                relevant_bg_grad += (bg_relations*(-fg_grad[ii]/FP_num))

        #aLRP with grad formulation fg gradient
        classification_grads[fg_labels]= fg_grad
        #aLRP with grad formulation bg gradient
        classification_grads[relevant_bg_labels]= relevant_bg_grad

        classification_grads /= (fg_num)

        cls_loss=1-prec.mean()
        ctx.save_for_backward(classification_grads)

        return cls_loss, rank, order

    @staticmethod
    def backward(ctx, out_grad1, out_grad2, out_grad3):
        g1, =ctx.saved_tensors
        return g1*out_grad1, None, None, None, None

class APLoss(torch.autograd.Function):
    @staticmethod
    def forward(ctx, logits, targets, delta=1.):
        classification_grads=torch.zeros(logits.shape).cuda()

        #Filter fg logits
        fg_labels = (targets == 1)
        fg_logits = logits[fg_labels]
        fg_num = len(fg_logits)

        #Do not use bg with scores less than minimum fg logit
        #since changing its score does not have an effect on precision
        threshold_logit = torch.min(fg_logits)-delta

        #Get valid bg logits
        relevant_bg_labels=((targets==0)&(logits>=threshold_logit))
        relevant_bg_logits=logits[relevant_bg_labels]
        relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda()
        rank=torch.zeros(fg_num).cuda()
        prec=torch.zeros(fg_num).cuda()
        fg_grad=torch.zeros(fg_num).cuda()

        max_prec=0
        #sort the fg logits
        order=torch.argsort(fg_logits)
        #Loops over each positive following the order
        for ii in order:
            #x_ij s as score differences with fgs
            fg_relations=fg_logits-fg_logits[ii]
            #Apply piecewise linear function and determine relations with fgs
            fg_relations=torch.clamp(fg_relations/(2*delta)+0.5,min=0,max=1)
            #Discard i=j in the summation in rank_pos
            fg_relations[ii]=0

            #x_ij s as score differences with bgs
            bg_relations=relevant_bg_logits-fg_logits[ii]
            #Apply piecewise linear function and determine relations with bgs
            bg_relations=torch.clamp(bg_relations/(2*delta)+0.5,min=0,max=1)

            #Compute the rank of the example within fgs and number of bgs with larger scores
            rank_pos=1+torch.sum(fg_relations)
            FP_num=torch.sum(bg_relations)
            #Store the total since it is normalizer also for aLRP Regression error
            rank[ii]=rank_pos+FP_num

            #Compute precision for this example
            current_prec=rank_pos/rank[ii]

            #Compute interpolated AP and store gradients for relevant bg examples
            if (max_prec<=current_prec):
                max_prec=current_prec
                relevant_bg_grad += (bg_relations/rank[ii])
            else:
                relevant_bg_grad += (bg_relations/rank[ii])*(((1-max_prec)/(1-current_prec)))

            #Store fg gradients
            fg_grad[ii]=-(1-max_prec)
            prec[ii]=max_prec

        #aLRP with grad formulation fg gradient
        classification_grads[fg_labels]= fg_grad
        #aLRP with grad formulation bg gradient
        classification_grads[relevant_bg_labels]= relevant_bg_grad

        classification_grads /= fg_num

        cls_loss=1-prec.mean()
        ctx.save_for_backward(classification_grads)

        return cls_loss

    @staticmethod
    def backward(ctx, out_grad1):
        g1, =ctx.saved_tensors
        return g1*out_grad1, None, None


import os,json,csv,re
import pandas as pd
import xml.etree.ElementTree as et

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKCYAN = '\033[96m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

class DISCO_Reader:

    def __init__(self,data_path):
        self.__data_path =data_path
        self.__paths_dict = None
        self.__xml_dict = None
        self.__current_file = None
        self.__tree = None
        self.__meta = { "start_date":None,
                        "end_time":None,
                        "channel_name":None,
                        "team_domain":None,
                    }
        self.__data = {"data":[], "meta":{}}
        self.__output_path=None
        self.__so_posts = {"data":[], "meta":{}}
        self.__current_convo = None
        self.__categories = [
                "Debugging Solution",
                "Alternative Solution",
                "Question Repost",
            "Direct Solution Provided by Asker",
            "Related Information (Not Solution)",
            "Direct Solution Provided by Non-Asker",
            "Validating Solution",
            "Outsourcing SO Post"
        ]

    def set_data_path(self, path):
        self.__data_path = os.path.abspath(path)
        self.set_xml_paths()
        return

    def set_output_path(self,path):
        self.__output_path = os.path.abspath(path)
        return

    def get_file_paths(self):
        if self.__data_path is None:
            return "Error: data path not set"
        paths = []
        data_path = self.__data_path
        try:
            project_dirs = os.listdir(self.__data_path)
            for d in project_dirs:
                timerange_dirs = os.listdir(f"{data_path}/{d}/")
                for t in timerange_dirs:
                    files = os.listdir(f"{data_path}/{d}/{t}/")
                    for f in files:
                        if ".out" in f:
                            paths.append(os.path.abspath(f"{data_path}/{d}/{t}/{f}"))
        except Exception as e:
            print(e)
        self.__paths_dict = paths
        return paths

    def load_xml(self,path):
        try:
            abs_path = os.path.abspath(path)
            tree = et.parse(abs_path)
            self.__current_file = abs_path
            self.__tree = tree
            self.read_meta()
            self.read_messages()
            return tree
        except Exception as e:
            print(e)
        return None

    def read_meta(self):
        root = self.get_root()
        for k in self.__meta.keys():
            if root.find(k) is not None:
                self.__meta[k] = root.find(k).text
        return

    def get_root(self):
        return self.__tree.getroot()

    def read_message(self,e):
        msg = { "ts":"", # timestamp
                "user":"",
                "text":"",
                }
        for k in msg.keys():
            msg[k] = e.find(k).text
        msg['conversation_id'] = e.attrib['conversation_id'] #thread id
        return msg

    def read_messages(self):
        root = self.get_root()
        count = 0
        convos = {}
        for child in iter(root.findall("message")):
            count+=1
            msg = self.read_message(child)
            convo_id = msg['conversation_id']

            if convo_id not in convos.keys():
                msg['index']=0
                convos[convo_id] = [msg]
            else:
                msg['index'] = len(convos[convo_id])
                convos[convo_id].append(msg)

        for k in convos.keys(): # for each convo_id
            thread = {"conversation_id" : k, "messages":convos[k]}
            self.__data['data'].append(thread)

        self.__meta['count'] = count
        return

    def get_data_dict(self):
        return self.__data['data']

    def get_so_data_dict(self):
        return self.__so_posts['data']
    def get_meta_data_dict(self):
        return self.__meta

    def export_to_json(self,path=""):
        try:
            if self.__output_path is None:
                self.set_output_path(path)
            data = self.__data
            data['meta'] = self.__meta
            data['so_posts'] = self.__so_posts['data']
            with open(self.__output_path, "w+") as f:
                json.dump(data, f)
        except Exception as e:
            print(e)
        return

    def get_convo_ids(self):
        data = self.get_data_dict()
        return [x['conversation_id'] for x in data]

    def get_convo(self,convo_id):
        if str(convo_id) in self.get_convo_ids():
            data =self.get_data_dict()
            return data[int(convo_id)-1]
        else:
            return None

    def get_so_convo_ids(self):
        data = self.get_so_data_dict()
        return [x['conversation_id'] for x in data]

    def get_so_convo(self,convo_id):
        if str(convo_id) in self.get_so_convo_ids():
            data =self.get_so_data_dict()
            for convo in data:
                if str(convo_id) == convo['conversation_id']:
                    return convo
        else:
            return None

    def find_so_posts(self,regex):
        so_convos = []
        pattern = re.compile(regex)
        data = self.get_data_dict()
        for convo in data:
            convo_id = convo['conversation_id']
            convo_thread = {"conversation_id":convo_id,
                            "messages":[],
                            "so_post_count":0
                            }
            msgs = convo['messages']
            for msg in msgs:
                if re.search(pattern,msg['text']):
                    convo_thread['so_post_count']+=1
                    convo_thread['messages'].append(msg)
            if convo_thread['so_post_count'] > 0:
                so_convos.append(convo_thread)
        self.__so_posts['data'] = so_convos
        return

    def get_so_posts(self):
        return self.__so_posts['data']

    def get_convos(self,convo_id):
        so_convo = self.get_so_convo(convo_id)
        total_convo = self.get_convo(convo_id)
        return so_convo, total_convo

    """
    def get_convo_df(self,so_msg, total_convo):
        keys = so_msg.keys()
        all_msgs = total_convo['messages']
        df = pd.DataFrame()
        for msg in all_msgs:
            msg = {'user':msg['user'] ,'text':msg['text']}
            df = df.append(msg,ignore_index=True)
        return df
    """

    def label_matches(self,output_dir_path,start_conversation_id=None):
        matches = self.get_so_data_dict()

        if start_conversation_id is None:
            start_conversation_id = matches[0]['conversation_id']

        # jump to starting conversation id and msg idx
        i = 0
        while i < len(matches) and matches[i]['conversation_id'] != start_conversation_id:
            i+=1
        if i >= len(matches):
            raise Exception(f"start_conversation_id {start_conversation_id} is invalid")
            return

        categories_str = ""
        for x in range(len(self.__categories)):
            categories_str = categories_str + f"{x}:{self.__categories[x]}\n"

        while i < len(matches):
            match_conversation = matches[i]
            match_conversation_id = match_conversation["conversation_id"]
            match_msgs = [x for x in match_conversation['messages']]
            full_conversation = self.get_convo(match_conversation_id)
            full_msgs = full_conversation['messages']
            for match_msg in match_msgs:
                match_msg_text = match_msg['text']
                match_msg_user = match_msg['user']
                match_msg_index = match_msg['index']
                target_str = f"{match_msg_index}:{match_msg_user}|{match_msg_text}"

                os.system('clear')
                k = 0
                for k in range(len(full_msgs)): # for each message in conversation
                    msg = full_msgs[k]
                    msg_text = msg['text']
                    msg_user = msg['user']
                    msg_index = msg['index']
                    output_str = f"{msg_index}:{msg_user}|{msg_text}"

                    if msg_index ==match_msg_index:
                        print(bcolors.WARNING + output_str + bcolors.ENDC)
                    elif msg_user == match_msg_user:
                        print(bcolors.OKCYAN + output_str + bcolors.ENDC)
                    elif '?' in msg_text:
                        print(bcolors.HEADER + output_str + bcolors.ENDC)
                    else:
                        print(bcolors.OKGREEN + output_str + bcolors.ENDC)

                progress = round((i/len(matches))*100, 2)

                print(bcolors.FAIL + f"{progress}%|CONVERSATION_ID: {match_conversation_id} MSG_IDX: {match_msg_index}\n{target_str}" + bcolors.ENDC)
                q_idx= input("Enter question msg index, t for target as Q&A, ENTER to skip, or x to quit: ")
                if q_idx =="x" or q_idx== "X":
                    print(f"QUITTING AT CONVERSATION_ID {match_conversation_id} and MSG IDX: {match_msg_index}")
                    return
                elif q_idx == "t" or q_idx=="T":
                    q_msg = match_msg
                elif len(q_idx) == 0:
                   continue #skip if
                else:
                    q_msg = full_msgs[int(q_idx)]
                category  = self.__categories[int(input(f"Pick from the following categories:\n{categories_str}"))]
                comments = input("Enter comments:")
                dump = {'related_post': q_msg ,
                        'so_post': match_msg,
                        'category' : category,
                        'comments': comments,
                        'meta': self.get_meta_data_dict()
                        }
                filename = f"{match_conversation_id}-{match_msg_index}.json"
                with open(f"{output_dir_path}/{filename}", "w+") as f:
                    json.dump(dump, f, indent=4)
            i=i+1
        return

if __name__ == "__main__":
    """
    d = done
    e = empty
    /home/colinm/Documents/serene/disco/data/pythongeneral/Dec2019/pythongeneralDec2019.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Apr2020/pythongeneralApr2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Jan2020/pythongeneralJan2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Aug2020/pythongeneralAug2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Mar2020/pythongeneralMar2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Nov2019/pythongeneralNov2019.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Feb2020/pythongeneralFeb2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Oct2020/pythongeneralOct2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/May2020/pythongeneralMay2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Jul2020/pythongeneralJul2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Jun2020/pythongeneralJun2020.xml.out
    /home/colinm/Documents/serene/disco/data/pythongeneral/Sep2020/pythongeneralSep2020.xml.out
d   /home/colinm/Documents/serene/disco/data/racketgeneral/May2020-July2020/Racket_general_May2020-Jul2020.xml.out
e   /home/colinm/Documents/serene/disco/data/racketgeneral/Feb2020-Apr2020/Racket_general_Feb2020-Apr2020.xml.out
e   /home/colinm/Documents/serene/disco/data/racketgeneral/Aug2020-Oct2020/Racket_general_Aug2020-Oct2020.xml.out
d   /home/colinm/Documents/serene/disco/data/racketgeneral/Nov2019-Jan2020/racketgeneral_Nov2019-Jan2020.xml.out
    /home/colinm/Documents/serene/disco/data/golang/May2020-July2020/golang_May2020-Jul2020.xml.out
    /home/colinm/Documents/serene/disco/data/golang/Feb2020-Apr2020/golang_Feb2020-Apr2020.xml.out
    /home/colinm/Documents/serene/disco/data/golang/Aug2020-Sep2020/golang_Aug2020-Sep2020.xml.out
    /home/colinm/Documents/serene/disco/data/golang/Nov2019-Jan2020/golang_Nov2019-Jan2020.xml.out
    /home/colinm/Documents/serene/disco/data/clojurians/May2020-July2020/clojure_May2020-Jul2020.xml.out
    /home/colinm/Documents/serene/disco/data/clojurians/Feb2020-Apr2020/clojure_Feb2020-Apr2020.xml.out
    /home/colinm/Documents/serene/disco/data/clojurians/Aug2020-Oct2020/clojure_Aug2020-Oct2020.xml.out
    /home/colinm/Documents/serene/disco/data/clojurians/Nov2019-Jan2020/clojure_Nov2019-Jan2020.xml.out
    """
    reader = DISCO_Reader("./data")
    """to easily get paths if needed
    for path in reader.get_file_paths():
        print(path)
    """
    reader.load_xml("/home/colinm/Documents/serene/disco/data/golang/May2020-July2020/golang_May2020-Jul2020.xml.out")
    reader.find_so_posts("stackoverflow.com") # can use to search w/ regexes
    reader.label_matches(output_dir_path="./output/golang", start_conversation_id="129") # actual labeling method


from django.db import models
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

class BlackLists(models.Model):
    user = models.ForeignKey(User, related_name='blacklist', verbose_name=_('user'))
    shielding = models.ForeignKey(User, related_name='shielding', verbose_name=_('shielding'))
    created_at = models.DateTimeField(auto_now_add=True)

    class Meta:
        db_table = 'blacklists'
        verbose_name = _('BlackLists')


from sklearn import linear_model
from sklearn.preprocessing import StandardScaler
import numpy as np

data=np.loadtxt("ex1data1.txt",delimiter=",",dtype=np.float64)
X = data[:, 0:-1]  # X表示0到倒数第二列
y=data[:,-1]
#归一化操作
scaler=StandardScaler()
scaler.fit(X)
x_train=scaler.transform(X)
x_test = scaler.transform(np.array([6.1101]).reshape(1,-1))
#线性模型拟合
model=linear_model.LinearRegression()
model.fit(x_train,y)

#预测
result=model.predict(x_test)
print(result)


from PIL import Image
from io import BytesIO

class ImageProcess(object):
    """
    图片处理类
    将Requests中获取到的二进制图片转换为Pillow可以读取的JPEG图片格式
    并对图像进行二值化处理
    可使用的成员函数
    byte2jpeg(byte)——将二进制格式图片byte转化为Pillow可读取格式
    image_binarize(image)——将Pillow格式图片image进行二值化
    process(source)——将request返回的二进制图片格式进行二值化
    """""

    def byte2jpeg(self, byte_content) -> Image:
        """
        将request包获取的二进制图像格式转换成Pillow可以读取的格式
        :param byte_content: Byte类型的图片数据
        :return: Pillow可以读取的jpeg图片格式
        """""
        return Image.open(BytesIO(byte_content))

    def image_binarize(self, image) -> Image:
        """
        将验证码图像进行二值化处理
        :param image: 原始图像
        :return: 二值化处理后的图像
        """""

        # 将图片转化为灰度图像
        image = image.convert("L")
        # 按像素点将图像转换为矩阵
        image_pixel_matrix = image.load()
        # 获取图像的宽度与高度，并进行遍历
        # 图片的宽度与高度默认都为66和22
        width, height = image.size
        for h in range(height):
            for w in range(width):
                # 消除首尾行列的黑框
                if h == 0 or h == 21 or w == 0 or w == 61:
                    image_pixel_matrix[w, h] = 255
                else:
                    # 通过反复尝试，阈值为110-140时对于目前的验证码效果最好
                    # 105±5的范围内为最优情况，过小会丢失图像，过大会保留干扰图像
                    if image_pixel_matrix[w, h] < 103:
                        image_pixel_matrix[w, h] = 0
                    else:
                        image_pixel_matrix[w, h] = 255
        return image

    def process(self, source) -> Image:
        """
        直接对Request返回的二进制图片格式进行二值化处理
        :param source: Requests返回的二进制格式图片
        :return: 二值化处理后的JPEG格式图片
        """""
        image = self.byte2jpeg(source)
        result = self.image_binarize(image)
        return result


from __future__ import annotations

from cranet import Tensor

from ..parameter import Parameter

from typing import (
    Iterator,
    Iterable,
    Optional,
    List,
    Dict,
    Set,
    Union,
    Tuple,
    Callable,
)
from collections import OrderedDict

class Module:
    _version = 1
    training: bool

    def __init__(self):
        self.training = True
        self._parameters: Dict[str, Optional[Parameter]] = OrderedDict()
        self._buffers: Dict[str, Optional[Tensor]] = OrderedDict()
        self._non_persistent_buffers_set: Set[str] = set()
        self._modules: Dict[str, Optional[Module]] = OrderedDict()

    def register_buffer(self, name: str, tensor: Optional[Tensor], persistent: bool = True) -> None:
        r"""
        This is typically used to register a buffer that should not to be
        considered a model parameter.

        Args:
            name (string): name of the buffer. The buffer can be accessed
                from this module using the given name
            tensor (Tensor): buffer to be registered.
            persistent (bool): whether the buffer is part of this module's
                :attr:`state_dict`.
        """
        if name == "":
            raise KeyError("key can't be empty string")
        if '.' in name:
            raise KeyError("key can't contain '.'")
        if hasattr(self, name) and name not in self._buffers:
            raise KeyError(f"attribute {name} already exist")

        self._buffers[name] = tensor
        if persistent:
            self._non_persistent_buffers_set.discard(name)
        else:
            self._non_persistent_buffers_set.add(name)

    def register_parameter(self, name: str, param: Optional[Parameter]) -> None:
        """add a parameter to the module"""
        if name == "":
            raise KeyError("key can't be empty string")
        if '.' in name:
            raise KeyError("key can't contain '.'")
        if hasattr(self, name) and name not in self._parameters:
            raise KeyError(f"attribute {name} already exist")

        if param is None:
            self._parameters[name] = None
        elif not isinstance(param, Parameter):
            raise TypeError("cranet.nn.Parameter or None required")
        else:
            self._parameters[name] = param

    def register_module(self, name: str, module: Optional[Module]) -> None:
        if name == "":
            raise KeyError("key can't be empty string")
        if '.' in name:
            raise KeyError("key can't contain '.'")
        if hasattr(self, name) and name not in self._modules:
            raise KeyError(f"attribute {name} already exist")
        self._modules[name] = module

    def get_name(self) -> str:
        return self.__class__.__name__

    def zero_grad(self):
        for p in self.parameters():
            p.zero_grad()

    def forward(self, *args, **kwargs) -> Tensor:
        raise NotImplementedError

    def __call__(self, *args, **kwargs) -> Tensor:
        return self.forward(*args, **kwargs)

    def __setstate__(self, state):
        self.__dict__.update(state)
        if '_non_persistent_buffers_set' not in self.__dict__:
            self._non_persistent_buffers_set = set()

    def __getattr__(self, name: str):
        if '_parameters' in self.__dict__:
            _parameters = self.__dict__['_parameters']
            if name in _parameters:
                return _parameters[name]
        if '_buffers' in self.__dict__:
            _buffers = self.__dict__['_buffers']
            if name in _buffers:
                return _buffers[name]
        if '_modules' in self.__dict__:
            modules = self.__dict__['_modules']
            if name in modules:
                return modules[name]
        raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")

    def __setattr__(self, name: str, value: Union[Tensor, Module]) -> None:
        def remove_from(*dicts_or_sets):
            for d in dicts_or_sets:
                if name in d:
                    if isinstance(d, dict):
                        del d[name]
                    else:
                        d.discard(name)

        params = self.__dict__.get('_parameters')
        if isinstance(value, Parameter):
            if params is None:
                raise AttributeError(
                    "cannot assign parameters before Module.__init__() call")
            remove_from(self.__dict__, self._buffers, self._modules, self._non_persistent_buffers_set)
            self.register_parameter(name, value)
        elif params is not None and name in params:
            if value is not None:
                raise TypeError(f"cannot assign '{type(value)}' as parameter '{name}' "
                                "(torch.nn.Parameter or None expected)")
            self.register_parameter(name, value)
        else:
            modules = self.__dict__.get('_modules')
            if isinstance(value, Module):
                if modules is None:
                    raise AttributeError("cannot assign module before Module.__init__() call")
                remove_from(self.__dict__, self._parameters, self._buffers, self._non_persistent_buffers_set)
                modules[name] = value
            elif modules is not None and name in modules:
                if value is not None:
                    raise TypeError(f"cannot assign '{type(value)}' as child module '{name}' "
                                    "(torch.nn.Module or None expected)")
                modules[name] = value
            else:
                buffers = self.__dict__.get('_buffers')
                if buffers is not None and name in buffers:
                    if value is not None and not isinstance(value, Tensor):
                        raise TypeError(f"cannot assign '{type(value)}' as buffer '{name}' "
                                        "(torch.Tensor or None expected)")
                    buffers[name] = value
                else:
                    object.__setattr__(self, name, value)

    def __delattr__(self, name):
        if name in self._parameters:
            del self._parameters[name]
        elif name in self._buffers:
            del self._buffers[name]
            self._non_persistent_buffers_set.discard(name)
        elif name in self._modules:
            del self._modules[name]
        else:
            object.__delattr__(self, name)

    def _named_members(self, get_members_fn: Callable[[Module], Iterable], prefix='', recurse=True) -> Iterator:
        r"""Helper method for yielding various names + members of modules."""
        memo = set()
        modules = self.named_modules(prefix=prefix) if recurse else [(prefix, self)]
        for module_prefix, module in modules:
            members = get_members_fn(module)
            for k, v in members:
                if v is None or v in memo:
                    continue
                memo.add(v)
                name = module_prefix + ('.' if module_prefix else '') + k
                yield name, v

    def named_parameters(self, prefix: str = '', recurse: bool = True) -> Iterator[Tuple[str, Parameter]]:
        memo: Set[Parameter] = set()
        modules: List[Tuple[str, Module]] = self.named_modules(prefix=prefix) if recurse else [(prefix, self)]
        for module_prefix, module in modules:
            members = module._parameters.items()
            for k, v in members:
                if v is None or v in memo:
                    continue
                memo.add(v)
                name = module_prefix + ('.' if module_prefix else '') + k
                yield name, v

    def named_modules(self, memo: Optional[Set[Module]] = None, prefix: str = '', remove_duplicate: bool = True):
        if memo is None:
            memo = set()
        if self not in memo:
            if remove_duplicate:
                memo.add(self)
            yield prefix, self
            for name, module in self._modules.items():
                if module is None:
                    continue
                submodule_prefix = prefix + ('.' if prefix else '') + name
                for m in module.named_modules(memo, submodule_prefix, remove_duplicate):
                    yield m

    def children(self) -> Iterator['Module']:
        r"""Returns an iterator over immediate children modules.

        Yields:
            Module: a child module
        """
        for name, module in self.named_children():
            yield module

    def named_children(self) -> Iterator[Tuple[str, 'Module']]:
        r"""Returns an iterator over immediate children modules, yielding both
        the name of the module as well as the module itself.

        Yields:
            (string, Module): Tuple containing a name and child module
        """
        memo = set()
        for name, module in self._modules.items():
            if module is not None and module not in memo:
                memo.add(module)
                yield name, module

    def parameters(self, recurse: bool = True) -> Iterator[Parameter]:
        for name, param in self.named_parameters(recurse=recurse):
            yield param

    def state_dict(self, destination=None, prefix='', keep_vars=False):
        if destination is None:
            destination = OrderedDict()
            destination._metadata = OrderedDict()
        destination._metadata[prefix[:-1]] = dict(version=self._version)
        self._save_to_state_dict(destination, prefix, keep_vars)
        for name, module in self._modules.items():
            if module is not None:
                module.state_dict(destination, prefix + name + '.', keep_vars=keep_vars)
        return destination

    def _save_to_state_dict(self, destination, prefix, keep_vars):
        for name, param in self._parameters.items():
            if param is not None:
                destination[prefix + name] = param if keep_vars else param.detach()
        for name, buf in self._buffers.items():
            if buf is not None and name not in self._non_persistent_buffers_set:
                destination[prefix + name] = buf if keep_vars else buf.detach()

    def train(self, mode: bool = True):
        self.training = mode
        for m in self.children():
            m.train(mode)
        return self

    def eval(self):
        return self.train(False)

    def extra_repr(self) -> str:
        return ''

    def _get_name(self):
        return self.__class__.__name__

    def __repr__(self) -> str:
        extra_lines = []
        extra_repr = self.extra_repr()
        if extra_repr:
            extra_lines = extra_repr.split('\n')
        child_lines = []
        for key, module in self._modules.items():
            mod_str = repr(module)
            mod_str = _addindent(mod_str, 2)
            child_lines.append('(' + key + '): ' + mod_str)
        lines = extra_lines + child_lines

        main_str = self._get_name() + '('
        if lines:
            if len(extra_lines) == 1 and not child_lines:
                main_str += extra_lines[0]
            else:
                main_str += '\n  ' + '\n  '.join(lines) + '\n'

        main_str += ')'
        return main_str

def _addindent(s_, num_spaces):
    s = s_.split('\n')
    # don't do anything for single-line stuff
    if len(s) == 1:
        return s_
    first = s.pop(0)
    s = [(num_spaces * ' ') + line for line in s]
    s = '\n'.join(s)
    s = first + '\n' + s
    return s


from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.http import HttpResponseRedirect, Http404
from django.core.paginator import Paginator
from django.contrib.auth.decorators import login_required, permission_required
from django.shortcuts import render_to_response, get_object_or_404
from models import *

@login_required
def meus_trajetos(request):
    trajetos = Trajeto.objects.filter(usuario=request.user)
    return render_to_response('meus_trajetos.html', locals(), context_instance = RequestContext(request),)

@login_required
def editar_trajeto(request, trajeto_id,classe):
    trajeto = Trajeto.objects.get(id=trajeto_id)
    nome = trajeto.nome
    distancia = trajeto.distancia
    coordenadas = Coordenadas.objects.filter(trajeto=trajeto.id)

    if request.method == 'POST':
        novas_coordenadas = request.POST.getlist('waypoints')
        if(len(novas_coordenadas) > 0):
            trajeto.nome = request.POST.get('nome')
            trajeto.distancia = request.POST.get('distancia')[:-2].replace(',','.')
            coordenadas.delete()

            trajeto.save()

            for coordenada in novas_coordenadas:
                tupla_coordenada = coordenada[1:-1].split(',')
                temp_latitude = tupla_coordenada[0]
                temp_longitude = tupla_coordenada[1]
                nova_coordenada = Coordenadas(latitude=temp_latitude, longitude= temp_longitude, trajeto=trajeto)
                nova_coordenada.save()

    return render_to_response('edit_trajeto.html',locals(),context_instance=RequestContext(request))

@login_required
def trajeto(request):
    if request.method == 'POST':
        nome_trajeto = request.POST.get('nome')
        '''Lembrar de depois tratar o erro para distancia 0
        Fazer funcao parser dps :B'''
        distancia_total = request.POST.get('distancia')
        distancia_total = distancia_total[:-2].replace(',','.')
        coordenadas_todas = request.POST.getlist('waypoints')

        novo_trajeto = Trajeto(nome=nome_trajeto, distancia=distancia_total,usuario= request.user)
        novo_trajeto.save()

        for coordenada in coordenadas_todas:
            tupla_coordenada = coordenada[1:-1].split(',')
            temp_latitude = tupla_coordenada[0]
            temp_longitude = tupla_coordenada[1]
            nova_coordenada = Coordenadas(latitude=temp_latitude, longitude= temp_longitude, trajeto=novo_trajeto)
            nova_coordenada.save()

    return render_to_response(
        'set_trajetos.html',
        locals(),
        context_instance=RequestContext(request),
        )


import sys
import os
from matplotlib import pyplot as plt
import numpy as np
import math
import pandas as pd

sys.path.append(os.path.dirname(os.path.abspath(__file__))
                + "/../ModelPredictiveTrajectoryGenerator/")

try:
    import model_predictive_trajectory_generator as planner
    import motion_model
except:
    raise

table_path = os.path.dirname(os.path.abspath(__file__)) + "/lookuptable.csv"

show_animation = True

def search_nearest_one_from_lookuptable(tx, ty, tyaw, lookup_table):
    mind  = float("inf")
    minid = -1

    for (i, table) in enumerate(lookup_table):

        dx   = tx - table[0]
        dy   = ty - table[1]
        dyaw = tyaw - table[2]
        d    = math.sqrt(dx ** 2 + dy ** 2 + dyaw ** 2)

        if d <= mind:
            minid = i
            mind  = d

    return lookup_table[minid]

def get_lookup_table():
    data = pd.read_csv(table_path)

    return np.array(data)

def generate_path(target_states, k0):
    # x, y, yaw, s, km, kf

    lookup_table = get_lookup_table
    result = []

    for state in target_states:
        bestp = search_nearest_one_from_lookuptable(
            state[0], state[1], state[2], lookup_table)

        target = motion_model.State(x=state[0], y=state[1], yaw=state[2])
        init_p = np.array(
            [math.sqrt(state[0] ** 2 + state[1] ** 2), bestp[4], bestp[5]]).reshape(3, 1)

        x, y, yaw, p = planner.optimize_trajectory(target, k0, init_p)

        if x is not None:
            print("find goof path")
            result.append(
                [x[-1], y[-1], yaw[-1], float(p[0]), float(p[1]), float(p[2])])
    print("finish path generation")
    return result

def calculate_uniform_polar_states(nxy, nh, d, a_min, a_max, p_min, p_max):

    '''
    calculate uniform state

    :param nxy   : number of position sampling
    :param nh    : number of heading sampling
    :param d     : distance of terminal state
    :param a_min : position sampling min angle
    :param a_max : position sampling max angle
    :param p_min : heading sampling min angle
    :param p_max : heading sampling max angle
    :return      : states list

    '''
    angle_samples = [i / (nxy - 1) for i in range(nxy)]
    states = sample_states(angle_samples, a_min, a_max, d, p_min, p_max, nh)

    return states

def sample_states(angle_samples, a_min, a_max, d, p_min, p_max, nh):
    states = []
    for i in range(angle_samples):
        a = a_min + (a_max - a_min) * i

        for j in range(nh):
            xf = d * math.cos(a)
            yf = d * math.sin(a)

            if nh == 1:
                yawf = (p_max - p_min) / 2.0 + a
            else:
                yawf = p_min + (p_max - p_min) * j / (nh - 1) + a

            states.append([xf, yf, yawf])

    return states

def uniform_terminal_state_sampling_test1():

    k0     = 0.0
    nxy    = 5
    nh     = 3
    d      = 20
    a_min  = - np.deg2rad(45.0)
    a_max  =   np.deg2rad(45.0)
    p_min  = - np.deg2rad(45.0)
    p_max  =   np.deg2rad(45.0)
    states =   calculate_uniform_polar_states (nxy, nh, d, a_min, a_max, p_min, p_max)
    result =   generate_path(states, k0)

    for table in result:
        xc, yc, yawc = motion_model.generate_trajectory(
            table[3], table[4], table[5], k0)
        if show_animation:
            plt.plot(xc, yc, "-r")
    if show_animation:
        plt.grid(True)
        plt.axis("equal")
        plt.show()
    print("Done")

def main():
    uniform_terminal_state_sampling_test1()

if __name__ == '__main__':
    main()


import os

import numpy as np
import torch

from PIL import Image
from diffusers import ControlNetModel, StableDiffusionControlNetInpaintPipeline, DDIMScheduler
from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import MultiControlNetModel
from transformers import BlipProcessor, BlipForQuestionAnswering

def postprocess(output: StableDiffusionPipelineOutput) -> list[Image.Image]:
    images = output.images
    assert isinstance(images[0], Image.Image), "images should be a list of PIL.Image.Image"
    nsfw = output.nsfw_content_detected
    images = [images[i] for i in range(len(images)) if not nsfw[i]]
    return images

def resize_tile_condition_image(input_image: Image, resolution: int, mode="long_side"):
    assert mode in ["long_side", "short_side"], "mode should be long_side or short_side"
    input_image = input_image.convert("RGB")
    W, H = input_image.size
    k = float(resolution) / (min(H, W) if mode == "short_side" else max(H, W))
    H *= k
    W *= k
    H = int(round(H / 64.0)) * 64
    W = int(round(W / 64.0)) * 64
    img = input_image.resize((W, H), resample=Image.LANCZOS)
    return img

def make_inpaint_condition(image, image_mask):
    image = np.array(image.convert("RGB")).astype(np.float32) / 255.0
    image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0

    assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size"
    image[image_mask > 0.5] = -1.0  # set as masked pixel
    image = np.expand_dims(image, 0).transpose(0, 3, 1, 2)
    image = torch.from_numpy(image)
    return image

class BLIP:
    def __init__(self, model_name="./checkpoints/blip-vqa-capfilt-large", device='cuda'):
        self.device = device
        self.processor = BlipProcessor.from_pretrained(model_name)
        self.model = BlipForQuestionAnswering.from_pretrained(model_name).to(device)

    def vqa(self, image_path: str, question: str) -> str:
        raw_image = Image.open(image_path).convert('RGB')
        inputs = self.processor(raw_image, question, return_tensors="pt").to(self.device)
        out = self.model.generate(**inputs)
        answer = self.processor.decode(out[0], skip_special_tokens=True)
        return answer

class InpaintControlNet:
    controlnet_models = {
        'softedge': 'control_v11p_sd15_softedge',
        'openpose': 'control_v11p_sd15_openpose',
        'lineart': 'control_v11p_sd15_lineart'
    }

    def __init__(self,
                 base_model="./checkpoints/realisticVisionV40VAE-inpainting",
                 controlnet_folder: str = "./checkpoints",
                 controlnet: list[str] = None,
                 device='cuda'):

        if controlnet is None:
            controlnet = ['lineart']

        assert all([k in self.controlnet_models for k in controlnet]), \
            f"controlnet should be a subset of {self.controlnet_models.keys()}"

        self.device = device
        self.controlnet = {}
        for k in controlnet:
            model_path = os.path.join(controlnet_folder, self.controlnet_models[k])
            self.controlnet[k] = ControlNetModel.from_pretrained(model_path, torch_dtype=torch.float16).to(self.device)

        self.pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained(
            base_model, controlnet=list(self.controlnet.values()), torch_dtype=torch.float16
        ).to(self.device)

        self.pipe.scheduler = DDIMScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.enable_xformers_memory_efficient_attention()
        self.pipe.enable_model_cpu_offload()

    def __call__(self, prompt: str, image_path: str, mask_image_path: str,
                 control_image: list[str] = None,
                 controlnet: list[str] = None, **kwargs) -> list[str]:
        image = Image.open(image_path).convert("RGB")
        mask_image = Image.open(mask_image_path).convert("L")
        if control_image is not None:
            control_image = [Image.open(path).convert("RGB") for path in control_image]
        if not isinstance(controlnet, list) and not isinstance(controlnet, str):
            controlnet = ['lineart']
            control_image = [mask_image]
            kwargs['controlnet_conditioning_scale'] = 0.0
        else:
            assert len(controlnet) > 0, "controlnet should be a subset of {self.controlnet.keys()}"
            if isinstance(controlnet, str):
                controlnet = [controlnet]
            if isinstance(control_image, str):
                control_image = [control_image]
            assert len(controlnet) == len(control_image), "'controlnet' and 'control_image' should have the same length"
            assert all([k in self.controlnet for k in controlnet]), \
                f"controlnet should be a subset of {self.controlnet.keys()}"

        if len(controlnet) > 1:
            self.pipe.controlnet = MultiControlNetModel([self.controlnet[k] for k in controlnet]).to(self.device)
        elif len(controlnet) == 1:
            self.pipe.controlnet = self.controlnet[controlnet[0]].to(self.device)

        if 'width' not in kwargs and 'height' not in kwargs:
            kwargs.update({'width': image.width, 'height': image.height})

        with torch.inference_mode():
            results = postprocess(
                self.pipe(prompt=prompt, image=image, mask_image=mask_image,
                          control_image=control_image, **kwargs)
            )
            # save results to the same path of image_path
            result_paths = []
            for i, img in enumerate(results):
                img.save(image_path.replace('.', f'-result-{i}.'))
                result_paths.append(image_path.replace('.', f'-result-{i}.'))
        return result_paths


"""The main app in the backend application"""
from fastapi import FastAPI
from database.db import init_db
from routers.product_review import product_reviews_router

app = FastAPI(
    title="Beanie Rules",
    description="Some Dummy backend which has beanie as the ODM",
    version="0.10"
)

@app.on_event("startup")
async def start_db():
    """The call to initialize the database on startup"""
    await init_db()

@app.get(
    "/",
    tags=["Root"]
)
async def root() -> dict[str, str]:
    """The root endpoint for the application"""
    return {"message": "Hello World"}

app.include_router(product_reviews_router,
                   prefix="/product_reviews", tags=["Product Reviews"])


import numpy as np
import os
from PIL import ImageFont, ImageDraw, Image
from moviepy.audio.io.AudioFileClip import AudioFileClip
from moviepy.video.io.VideoFileClip import VideoFileClip
from moviepy.audio.AudioClip import AudioClip
from moviepy.video.VideoClip import ImageClip
from moviepy.editor import concatenate_videoclips, concatenate_audioclips, CompositeVideoClip, CompositeAudioClip, afx
from skimage.filters import gaussian

class Editor(object):
  def __init__(
    self,
    target_resolution: tuple,
    fps: int,
    font: str,
    text_size: int,
    text_color: str,
    text_stroke_width: int,
    text_stroke_color: str
  ):
    """
    Initialize Editor.

    Args:
        target_resolution (tuple): (width, height) of the editted video.
        fps (int): fps of the video.
        font (str): path to the font file.
        text_size (int): size of the subtitle.
        text_color (str): color of the subtitle.
        text_stroke_width (int): width of the stroke of the subtitle.
        text_stroke_color (str): color of the stroke of the subtitle.
    """
    self.target_resolution = target_resolution
    self.fps = fps
    self.text_size = text_size
    self.text_color = text_color
    self.font = ImageFont.truetype(font, size=text_size)
    self.max_text_cnt = target_resolution[0] // text_size
    self.text_stroke_width = text_stroke_width
    self.text_stroke_color = text_stroke_color

  def __is_image(self, image_name: str):
    """
    Determine whether given name is image or video.

    Args:
        image_name (str): file name

    Return:
        return (bool): True if image, False otherwise (of course, video.)
    """
    return image_name.endswith(("bmp", "jpg", "jpeg", "gif", "png", "tif", "wmf", "emf"))

  def __fit_image_or_video_in_screen(self, image: ImageClip | VideoFileClip):
    """
    Fit given image or video into target resolution.
    Ratio will be kept during resizing, and rest of the part would be blurred image.

    Args:
        image (ImageClip | VideoFileClip): image or video

    Return:
        return (VideoFileClip): fitted image or video.
    """
    # resize given image while keeping ratio
    if image.size[0] / image.size[1] > self.target_resolution[0] / self.target_resolution[1]:
      height = image.size[1] * self.target_resolution[0] / image.size[0]
      height = max(height, self.target_resolution[0])
    else:
      height = self.target_resolution[1]
    image = image.resize(height=height)

    # set image position: center
    image = image.set_position(("center", "center"))

    # set duration and position of the background
    background = image.resize(self.target_resolution)
    background = background.fl_image(lambda frame: gaussian(frame.astype(float), sigma=8))
    background = background.set_duration(image.duration)

    # composite with background image.
    return CompositeVideoClip([background, image])

  def __add_text_to_video(self, image: VideoFileClip, text: str):
    """
    Returns subtitle-adding function for moviepy.fl_image()

    Args:
        image (VideoFileClip): video to add subtitle
        text (str): subtitle

    Return:
        return (VideoFileClip): video with subtitle
    """
    def pipeline(subtitle):
      def fun(frame):
        frame = Image.fromarray(frame)
        draw = ImageDraw.Draw(frame)
        for idx, line in enumerate(subtitle):
          x = self.target_resolution[0] / 2
          # y = self.target_resolution[1] - self.text_size * (3 + 1.5 * (len(subtitle) - 1 - idx))
          # y = self.target_resolution[1] / 2 + self.text_size * (idx * 1.5 - 0.75)
          y = self.target_resolution[1] * 0.73 + self.text_size * idx * 1.4
          draw.text((x, y), line, self.text_color, self.font, "mm", stroke_width=self.text_stroke_width, stroke_fill=self.text_stroke_color)
        return np.array(frame)
      return fun

    # split text to put in the screen
    subtitles = []
    subtitle = ""
    for token in text.split():
      if len(subtitle) + len(token) > self.max_text_cnt:
        # enter: make new line
        subtitles.append(subtitle)
        subtitle = token
      else:
        # add token to current line
        subtitle += f" {token}"
        subtitle = subtitle.strip()
      if token.endswith((".", "!", "?", "\"")):
        # end of the line: make new empty line
        subtitles.append(subtitle)
        subtitle = ""
    if subtitle:
      subtitles.append(subtitle)

    # triple subtitles
    subtitles = list(zip(subtitles[::2], subtitles[1::2] + [""]))

    # add subtitles for each clips
    sub_clips = []
    get_subtitle_len = lambda l: sum(len(s) + (4 if s.endswith((".", "!", "?", "\"")) else 0) for s in l)
    total_len = sum(map(get_subtitle_len, subtitles))
    start = 0
    end = 0
    for idx, subtitle in enumerate(subtitles):
      if idx == len(subtitles) - 1:
        end = image.duration
      else:
        end = min(start + image.duration * (get_subtitle_len(subtitle) / total_len), image.duration)
      sub_clip = image.subclip(start, end)
      sub_clip = sub_clip.fl_image(pipeline(subtitle))
      sub_clips.append(sub_clip)
      start = end

    return concatenate_videoclips(sub_clips)

  def __edit_image_or_video_with_audio(self, line: dict, video_name: str):
    """
    Add a video to an audio, then return a video clip.

    Args:
        line (dict): must have audio_name, image_name, and duration.
        video_name (str): name of the video.

    Return:
        return (VideoClip): video clip.
    """
    # validation
    assert "audio_name" in line
    assert "image_name" in line
    assert "content" in line

    # audio clip
    audio = AudioFileClip(line["audio_name"])
    audio = audio.subclip(0, audio.duration - 0.05)
    audio = concatenate_audioclips([audio, AudioClip(lambda _: 0, duration=0.05)])

    # get image clip
    if self.__is_image(line["image_name"]):
      # if image, set duration as audio
      image = ImageClip(line["image_name"])
      image = image.set_duration(audio.duration)
    else:
      # if video, remove audio and set duration as audio (loop or cut)
      image = VideoFileClip(line["image_name"])
      image = image.without_audio()
      if image.duration < audio.duration:
        image = image.loop(duration=audio.duration)
      else:
        image = image.set_duration(audio.duration)

    # fit video into the screen
    image = self.__fit_image_or_video_in_screen(image)

    # add subtitles
    image = self.__add_text_to_video(image, line["content"])

    # add audio
    video = image.set_audio(audio)
    video = video.set_fps(self.fps)

    return video

  def __add_bgm_to_video(self, video: VideoFileClip, bgm: str, bgm_vol: float):
    # bgm
    bgm = AudioFileClip(bgm)
    bgm = bgm.volumex(bgm_vol)
    if bgm.duration < video.duration:
      bgm = afx.audio_loop(bgm, duration=video.duration)
    else:
      bgm = bgm.set_duration(video.duration)

    # compose bgm with audio of the video
    audio = CompositeAudioClip([video.audio, bgm])

    return video.set_audio(audio)

  def edit_video(self, scenes: list, dirpath: str, bgm: str, bgm_vol: float):
    """
    Edit and save the final video.

    Args:
        scenes (list): returned value by Image/Video Generator.
                       each line must have audio_name, video_name, and duration.
        dirpath (str): path of saved clips and video.
        bgm (str): path to the background music.
        bgm_vol (float): volume of the background music.

    Return:
        return (str): returns video_name if no error occurred.
    """
    # generate video clips for each lines
    video_clips = []
    for scene_idx, scene in enumerate(scenes):
      for line_idx, line in enumerate(scene):
        video_clips.append(self.__edit_image_or_video_with_audio(line=line, video_name=os.path.join(dirpath, f"clip-{scene_idx}-{line_idx}")))

    # make final video
    video = concatenate_videoclips(video_clips)
    video = self.__add_bgm_to_video(video, bgm, bgm_vol)
    video.write_videofile(os.path.join(dirpath, "video.mp4"))

    return os.path.join(dirpath, "video.mp4")


from django.urls import path, include
from . import views
from django.views.generic import TemplateView, RedirectView

urlpatterns = [
    path("", views.login, name="login"),
    path("login", RedirectView.as_view(permanent=True, url="/")),
    path("register", RedirectView.as_view(permanent=True, url="/")),
    # path("register", views.register, name="register"),
    path("home", views.home, name="home"),
    path("update_zendesk", views.update_zendesk, name="update_zendesk"),
    path("call_zendesk_api", views.call_zendesk_api, name="call_zendesk_api"),
]


import psutil
import pyttsx3
import warnings
import speech_recognition as sr
import os
import webbrowser
import wikipedia
import random
import subprocess
import string
import time
from sys import platform
import psutil as p
import datetime
from src.va import speak, ask

# engine = pyttsx3.init()
# voices = engine.getProperty("voices")
# engine.setProperty("voice", voices[1].id)

def open_chrome():

    if platform == "linux" or platform == "linux2":
        subprocess.run("chrome", shell=True)
    elif platform == "darwin":
        subprocess.run("open -a Chrome --args -private-window", shell=True)
    elif platform == "win32":
        subprocess.run("start chrome", shell=True)
    else:
        speak("Unsupported OS")

    # webbrowser.register('chrome', None, webbrowser.BackgroundBrowser(chrome_path))
    # webbrowser.get('chrome').open_new_tab('https://google.com')

def chrome_caller(input_string):
    open_chrome()

def main_date():

    now = datetime.datetime.now()
    speak("Sir, the Current Date is : " + now.strftime("%m-%d-%Y") + "\n")
    print("Sir, the Current Date is : ")
    print(now.strftime("%d-%m-%Y") + "\n")
    print("")

def date_caller(input_string):
    main_date()

def search_wiki():
    speak("What do you want to search on wikipedia ?")
    sinwiki = ask()
    sinwiki = sinwiki.replace("wikipedia", "")
    results = wikipedia.summary(sinwiki, sentences=2)
    print(results)
    speak("According to Wikipedia,")
    speak(results)

def search_wiki_caller(input_string):
    search_wiki()

hash_201801012 = {
    "Open Chrome": chrome_caller,
    "open chrome": chrome_caller,
    "start chrome": chrome_caller,
    "begin chrome": chrome_caller,
    "begin the chrome": chrome_caller,
    "initiate chrome": chrome_caller,
    "open chrome": chrome_caller,
    "start chrome": chrome_caller,
    "what is date": date_caller,
    "show the date": date_caller,
    "show date": date_caller,
    "display the date": date_caller,
    "display date": date_caller,
    "say date": date_caller,
    "search on wiki": search_wiki_caller,
    "search on wikipedia": search_wiki_caller,
}


import os
import uuid
import concurrent
import shutil

import pygit2
import tornado.web

from modules.Base import BaseHandler
from modules.Database import User, Repo
from modules.Config import *
from modules.Git import *
from modules.Lib import safe_markdown, admin

class RepoImportHandler(BaseHandler):
	@tornado.web.authenticated
	def get(self):
		action = self.get_argument("action", None)
		if action == "progress":
			self.get_progress()
		else:
			self.render("repoimport.htm")

	def get_progress(self):
		pass

	@tornado.web.authenticated
	def post(self):

		user = self.current_user
		assert user

		userpath = os.path.join(REPO_ROOT, user.username)
		os.makedirs(userpath, exist_ok=True)
		assert os.path.exists(userpath)

		repourl = self.get_argument("repo.url")
		reponame = self.get_argument("repo.reponame", "")
		private = self.get_argument("repo.private", False)
		intro = self.get_argument("repo.intro", "")

		if not reponame:
			reponame = repourl.split("/")[-1]
			if reponame.endswith(".git"):
				reponame = reponame[:-4]

		if os.path.exists(os.path.join(userpath, reponame)):
			return self.info("reponame already exists!")

		def import_repo(remote_path, local_path):
			try:
				repo = pygit2.clone_repository(remote_path, local_path, bare=True)
				return repo
			except Exception as e:
				print(e)
				return str(e)

		repo = Repo()
		repo.rid = uuid.uuid4()
		repo.user_id = user.uid
		repo.reponame = reponame
		repo.intro = intro or "Cloned from "+repourl
		repo.public = (not private)
		repo.save(force_insert=True)

		task = concurrent.futures.ThreadPoolExecutor(1)
		future = task.submit(import_repo, repourl, os.path.join(userpath, reponame))
		ASYNC_TASK[str(repo.rid)] = future

		self.redirect("/{}/{}".format(user.username, repo.reponame))

class RepoDeleteHandler(BaseHandler):
	@tornado.web.authenticated
	def get(self, username, reponame):
		user = User.get_or_none(User.username == username)
		if not user:
			return self.info("user not exists")

		repo = Repo.get_or_none(Repo.reponame == reponame)
		if not repo:
			return self.info("repo not exists")

		if username != self.current_user.username:
			return self.info("not authorized")

		repopath = os.path.join(REPO_ROOT, user.username, repo.reponame)

		try:
			Repo.delete().where(Repo.rid == repo.rid).execute()
		except Exception as e:
			return self.info("failed to delete repo from database!"+str(e))

		try:
			if os.path.exists(repopath):
				shutil.rmtree(repopath, ignore_errors=False, onerror=None)
		except OSError as e:
			return self.info("failed to delete repo from filesystem!"+str(e))

		return self.redirect("/")

class RepoEditHandler(BaseHandler):
	@tornado.web.authenticated
	def get(self, username, reponame):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		self.render("repoedit.htm", repo = repo)

	@tornado.web.authenticated
	def post(self, username, reponame):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		newreponame = self.get_argument("repo.reponame", None)
		intro = self.get_argument("repo.intro", "")
		private = self.get_argument("repo.private", False)

		if not newreponame:
			return self.info("reponame cannot be empty")

		if Repo.get_or_none((Repo.reponame == newreponame) &(Repo.rid != repo.rid)):
			return self.info("reponame already exists")

		try:
			os.rename(os.path.join(REPO_ROOT, user.username, repo.reponame),
				os.path.join(REPO_ROOT, user.username, newreponame))
		except OSError as e:
			return self.info("failed to rename the repo. " + str(e))

		repo.reponame = newreponame
		repo.intro = intro
		repo.public = (not bool(private))
		repo.save()

		return self.redirect("/{}/{}".format(user.username, repo.reponame))

class RepoHooksHandler(BaseHandler):
	@tornado.web.authenticated
	def get(self, username, reponame):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		if repo.owner.uid != self.current_user.uid:
			return self.info("not authorized")

		hook_id = self.get_argument("hid", None)
		action = self.get_argument("action", None)

		if hook_id:
			hook = RepoHook.get_or_none(RepoHook.hid == int(hook_id))
			if not hook:
				return self.info("invalid hook id")
			if action == "disable":
				hook.disabled = True
				hook.save()
				return self.redirect("/{}/{}/edit/hooks".format(user.username, repo.reponame))
			elif action == "enable":
				hook.disabled = False
				hook.save()
				return self.redirect("/{}/{}/edit/hooks".format(user.username, repo.reponame))
			elif action == "delete":
				RepoHook.delete().where(RepoHook.hid == hook_id).execute()
				return self.redirect("/{}/{}/edit/hooks".format(user.username, repo.reponame))
			else:
				pass
		else:
			hook = RepoHook()

		self.render("repohook.htm", repo = repo, current = hook)

	@tornado.web.authenticated
	def post(self, username, reponame):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		if repo.owner.uid != self.current_user.uid:
			return self.info("not authorized")

		url = self.get_argument("hook.url")
		credential = self.get_argument("hook.credential", "")
		method = self.get_argument("hook.method", "GET")

		hook_id = self.get_argument("hid", None)
		if hook_id:
			hook = RepoHook.get_or_none(RepoHook.hid == hook_id)
		else:
			hook = RepoHook()

		hook.url = url
		hook.credential = credential
		hook.post = (method == "GET")
		hook.repo_id = repo.rid

		hook.save()

		self.redirect("/{}/{}/edit/hooks".format(user.username, repo.reponame))

class RepoAddHandler(BaseHandler):
	@tornado.web.authenticated
	def get(self):
		self.render("repoadd.htm")

	@tornado.web.authenticated
	def post(self):
		user = self.current_user
		reponame = self.get_argument("repo.reponame").strip()
		private = self.get_argument("repo.private", False)
		intro = self.get_argument("repo.intro", "")
		repopath = os.path.join(REPO_ROOT, user.username, reponame)
		if os.path.exists(repopath):
			return self.write("Name already taken.")

		try:
			cmd = GitCommand("git")
			cmd.addParam("init")
			cmd.addParam("--bare")
			cmd.addParam(repopath)
			cmd.run()
		except GitCommandException as e:
			print(e)
			shutil.rmtree(repopath, ignore_errors=True)
			return self.info("Failed to create repo, exception happened!")

		repo = Repo()
		repo.rid = uuid.uuid4()
		repo.user_id = user.uid
		repo.reponame = reponame
		repo.intro = intro
		repo.public = (not private)
		ret = repo.save(force_insert=True)

		self.redirect("/{}/{}".format(repo.owner.username, repo.reponame))

class RepoBlobHandler(BaseHandler):
	def get(self, username, reponame, ref, path):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		try:
			repo.git = pygit2.Repository(os.path.join(REPO_ROOT, user.username, reponame))
		except Exception as e:
			print(e)
			return self.info("Invalid repository.")

		path = "/"+path.strip("/") if path else ""
		if not ref: ref = repo.git.head.shorthand

		for a in repo.git.listall_references():
			if ref == a.split("/")[-1]:
				ref = a
				break

		repo.url = os.path.join("http://", self.request.host, username, reponame)
		repo.path = path
		repo.ref = ref
		repo.ref_short = ref.split("/")[-1]

		target_node = repo.git.revparse_single(ref)
		if hasattr(target_node, "tree"):
			target_node = target_node.tree
		elif isinstance(target_node, pygit2.Tag):
			target_node = target_node.get_object().tree
		else:
			return self.info("Failed to get tree of type:", target_node)

		if path:
			for p in path.split("/"):
				if not p: continue
				target_node = target_node[p]
				if target_node.type == "tree":
					target_node = repo.git.get(target_node.id)

		repo.filename = target_node.name
		repo.items = []
		blob = repo.git.get(target_node.id)
		try:
			repo.filedata = blob.data.decode("utf-8")
			if repo.filename.endswith(".md"):
				repo.filedata = safe_markdown(repo.filedata)
			else:
				repo.filedata = tornado.escape.xhtml_escape(repo.filedata)
		except UnicodeDecodeError as e:
			repo.filedata = "binary data"

		self.render("repotree.htm", repo=repo, lastcommit=None)

class RepoHistoryHandler(BaseHandler):
	def get(self, username, reponame, ref = None, path = None):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		try:
			repo.git = pygit2.Repository(os.path.join(REPO_ROOT, user.username, reponame))
		except Exception as e:
			print(e)
			return self.info("Invalid repository.")

		path = "/"+path.strip("/") if path else ""
		if not ref: ref = repo.git.head.shorthand

		for a in repo.git.listall_references():
			if ref == a.split("/")[-1]:
				ref = a
				break

		repo.url = os.path.join("http://", self.request.host, username, reponame)
		repo.path = path
		repo.ref = ref
		repo.ref_short = ref.split("/")[-1]

		history = list(GitLog(os.path.join(REPO_ROOT, user.username, repo.reponame),
			ref = repo.ref_short, item = path.lstrip("/")))

		self.render("history.htm", repo = repo, history = history)

class RepoTreeHandler(BaseHandler):
	def get(self, username, reponame, ref = None, path = None):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		if str(repo.rid) in ASYNC_TASK:
			if ASYNC_TASK[str(repo.rid)].done():
				result = ASYNC_TASK[str(repo.rid)].result()
				if not isinstance(result, pygit2.Repository):
					print("import failure", result)
				del ASYNC_TASK[str(repo.rid)]
			else:
				return self.info("repo is importing")

		try:
			repo.git = pygit2.Repository(os.path.join(REPO_ROOT, user.username, repo.reponame))
		except Exception as e:
			print(e)
			return self.info("Invalid repository.")

		if repo.git.is_empty:
			return self.render("repoempty.htm", repo = repo, host=self.request.host)

		path = "/"+path.strip("/") if path else ""
		if not ref: ref = repo.git.head.shorthand

		for a in repo.git.listall_references():
			if ref == a.split("/")[-1]:
				ref = a
				break

		repo.url = os.path.join("http://", self.request.host, username, reponame)
		repo.path = path
		repo.ref = ref
		repo.ref_short = ref.split("/")[-1]

		target_tree = repo.git.revparse_single(ref)
		if hasattr(target_tree, "tree"):
			target_tree = target_tree.tree
		elif isinstance(target_tree, pygit2.Tag):
			target_tree = target_tree.get_object().tree
		else:
			return self.info("Failed to get tree of type:", target_tree)

		if path:
			for p in path.split("/"):
				if not p: continue
				target_tree = repo.git.get(target_tree[p].oid)

		lastcommit = next(GitLog(os.path.join(REPO_ROOT, user.username, repo.reponame),
			ref=repo.ref_short, item=path, n=1), None)

		repo.items = []
		repo.filename = None
		repo.filedata = None

		if target_tree:
			for node in repo.git.get(target_tree.oid):
				x = repo.git.get(node.id)

				if tornado.options.options.perf:
					commit = None
				else:
					commit = next(GitLog(os.path.join(REPO_ROOT, user.username, repo.reponame),
						ref=repo.ref_short, item=os.path.join(path, node.name), n=1))

				repo.items.append({
					"name": node.name,
					"type": str(node.type),
					"lastcommit": commit,
				})

				if node.name.upper() == "README.MD":
					repo.filename = node.name
					blob = repo.git.get(node.id)
					repo.filedata = safe_markdown(blob.data.decode("utf-8"))

		self.render("repotree.htm", repo=repo, lastcommit=lastcommit)

class RepoCommitHandler(BaseHandler):
	def get(self, username, reponame, oid):
		user, repo, error = self.user_repo_access(username, reponame)
		if error:
			return self.info(error)

		try:
			repo.git = pygit2.Repository(os.path.join(REPO_ROOT, user.username, reponame))
		except Exception as e:
			print(e)
			return self.info("Invalid repository.")

		if repo.git.is_empty:
			return self.render("repoempty.htm", repo = repo, host=self.request.host)

		repo.url = os.path.join("http://", self.request.host, username, reponame)
		repo.path = ""
		repo.ref = ""
		repo.ref_short = ""

		self.render("commit.htm", repo = repo, commit = repo.git.get(oid))


import os
import hashlib
from io import open

import emails
import markdown
from bs4 import BeautifulSoup

from mdmail.helpers import sanitize_email_address, is_string

def send(email, subject=None,
         from_email=None, to_email=None,
         cc=None, bcc=None, reply_to=None,
         smtp=None):
    """Send markdown email

    Args:
        email (str/obj): A markdown string or EmailContent object
        subject (str): subject line
        from_email (str): sender email address
        to_email (str/list): recipient email addresses
        cc (str/list): CC email addresses (string or a list)
        bcc (str/list): BCC email addresses (string or a list)
        reply_to (str): Reply-to email address
        smtp (dict): SMTP configuration (dict)

    Schema of smtp dict:
        host (str): SMTP server host. Default: localhost
        port (int): SMTP server port. Default: 25
        tls (bool): Use TLS. Default: False
        ssl (bool): Use SSL. Default: False
        user (bool): SMTP login user. Default empty
        password (bool): SMTP login password. Default empty
    """
    if is_string(email):
        email = EmailContent(email)

    from_email = sanitize_email_address(from_email or email.headers.get('from'))
    to_email = sanitize_email_address(to_email or email.headers.get('to'))
    cc = sanitize_email_address(cc or email.headers.get('cc'))
    bcc = sanitize_email_address(bcc or email.headers.get('bcc'))
    reply_to = sanitize_email_address(reply_to or email.headers.get('reply-to'))

    message_args = {
        'html': email.html,
        'text': email.text,
        'subject': (subject or email.headers.get('subject', '')),
        'mail_from': from_email,
        'mail_to': to_email
    }
    if cc:
        message_args['cc'] = cc
    if bcc:
        message_args['bcc'] = bcc
    if reply_to:
        message_args['headers'] = {'reply-to': reply_to}

    message = emails.Message(**message_args)

    for filename, data in email.inline_images:
        message.attach(filename=filename, content_disposition='inline', data=data)

    message.send(smtp=smtp)

class EmailContent(object):
    def __init__(self, content, css=None, image_root='.'):
        """ Constructor

        Args:
            content (str): Markdown text
            css (str): Custom CSS style. If not set, use default CSS style.
            image_root (str): Root directory for inline images.
        """
        self._md = markdown.Markdown(extensions=[
            'markdown.extensions.tables',
            'markdown.extensions.meta'])
        self._html = None
        self._inline_images = None
        self._convert(content, css, image_root)

    def _replace_inline_images(self, html, image_root):
        soup = BeautifulSoup(html, "lxml")
        imgs = soup.findAll('img')
        inlines = {}
        for img in imgs:
            src = img['src']
            if not src.startswith('http'):
                src_uid = '{}_{}'.format(
                    hashlib.md5(src.encode('utf-8')).hexdigest()[:6],
                    os.path.basename(src))
                if src_uid not in inlines:
                    img_path = os.path.join(image_root, src)
                    inlines[src_uid] = open(img_path, 'rb')
                img['src'] = 'cid:'+src_uid
        self._inline_images = inlines.items()
        return soup

    def _inline_css(self, html, css):
        if not css:
            default_css = os.path.join(os.path.dirname(__file__), 'default.css')
            css = open(default_css, encoding='utf-8').read()
        email_html_template = u"""
        <!doctype html>
        <html>
            <head>
                <meta charset="UTF-8">
                <style>
                    {css}
                </style>
            </head>
            <body>
                {content}
            </body>
        </html>"""
        return email_html_template.format(css=css, content=html)

    def _convert(self, content, css, image_root):
        raw_html = self._md.convert(content)
        html = self._replace_inline_images(raw_html, image_root)
        html = self._inline_css(html, css)
        self._html = BeautifulSoup(html, "lxml").prettify()

    @property
    def html(self):
        return self._html

    @property
    def text(self):
        return u'\n'.join(self._md.lines)

    @property
    def headers(self):
        return {k.lower(): (v[0] if len(v) == 1 else v) for k,v in self._md.Meta.items()}

    @property
    def inline_images(self):
        return self._inline_images


from django.shortcuts import render, redirect, get_object_or_404
from django.http import HttpResponse
from django.contrib.auth.forms import UserCreationForm
from django.conf import settings
from django.contrib.auth.models import User
from .models import DogPost, Comment, CommentVoteLog
from .forms import QuestionForm, CommentForm, CommentVoteLogForm
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_GET, require_POST
from django.utils import timezone

# Create your views here.

def homepage(request):
    if request.user.is_authenticated:
        dogposts = DogPost.objects.order_by('-created_at')
        return render(request, 'core/home.html', {'dogposts': dogposts})
    else:
        return redirect('accounts/login/')

@login_required
def profile(request):
    posts = DogPost.objects.filter(user = request.user)
    comments = Comment.objects.filter(commentUser = request.user)
    return render(request, 'core/profile.html', {'posts': posts, 'comments': comments, } )

@login_required
def postdetail(request, pk):
    dogpost = DogPost.objects.get(pk=pk)
    coments = Comment.objects.filter(dogpost_id=pk).order_by('-created_at')
    if request.method == "POST" and "subcomment":
        form = CommentForm(request.POST)
        commentBody = request.POST.get('body')
        if form.is_valid():
            post = form.save(commit=False)
            post.commentUser = request.user
            post.created_at = timezone.now()
            post.dogpost = dogpost
            post.save()
            return redirect(f'/goodboi/{pk}/')

    else:
        form = CommentForm()
    return render(request, 'core/dogpostdetail.html', {'dogpost': dogpost, 'comments': coments, 'form': form, })

@login_required
def vote(request, pk):
    if request.method == "POST" and "upvote":
        form = CommentVoteLogForm(request.POST)
        comment = get_object_or_404(Comment, pk=pk)
        if form.is_valid():
            obj, created = CommentVoteLog.objects.update_or_create(
                voter = request.user, comment = comment,
                defaults={'rankValue': 1}
            )
            # obj.save()
            return redirect(f'/goodboi/{comment.dogpost.pk}/')
    elif request.method == "POST" and "downvote":
        form = CommentVoteLogForm(request.POST)
        comment = get_object_or_404(Comment, pk=pk)
        if form.is_valid():
            obj, created = CommentVoteLog.objects.update_or_create(
                voter = request.user, comment = comment,
                defaults={'rankValue': -1}
            )
            # obj.save()
            return redirect(f'/goodboi/{comment.dogpost.pk}/')

@login_required
def create_post(request):
    if request.method == "POST":
        form = QuestionForm(request.POST)
        title = request.POST.get('title')
        body = request.POST.get('body')
        if form.is_valid():
            post = form.save(commit=False)
            post.user = request.user
            post.created_at = timezone.now()
            post.save()
            return redirect('profile')
    else:
        form = QuestionForm()
    return render(request, 'core/newbork.html', {'form': form})

@login_required
def createComment(request):

    pass


from turtle import Turtle
from paddle import Paddle
from bricks import Bricks
from scoreboard import Scoreboard

TRAVEL_DIST = 10
paddle = Paddle()
bricks = Bricks()
score = Scoreboard()

class Ball(Turtle):
    def __init__(self):
        super().__init__()
        self.shape("circle")
        self.color("alice blue")
        self.penup()
        self.x_move_dist = TRAVEL_DIST
        self.y_move_dist = TRAVEL_DIST
        self.reset()

    def move(self):
        new_x = self.xcor() + self.x_move_dist
        new_y = self.ycor() + self.y_move_dist
        self.goto(new_x, new_y)

    def bounce(self, x_bounce, y_bounce):
        if x_bounce:
            self.x_move_dist *= -1
        if y_bounce:
            self.y_move_dist *= -1

    def reset(self):
        self.goto(x=0, y=-240)
        self.y_move_dist = 10


# Patrick Komiske, Eric Metodiev, MIT, 2017
#
# Contains two basic Keras models, a convolutional one with three convolutional
# layers and one dense layer and a dense model with an extensible number of
# layers.

from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D, SpatialDropout2D
from keras import backend as K
from keras.callbacks import EarlyStopping

# ensure Theano dimension ordering
K.set_image_dim_ordering('th')

def conv_net_construct(hps):

    nb_conv = hps['nb_conv']
    nb_pool = hps['nb_pool']
    img_size = hps['img_size']
    nb_filters = hps['nb_filters']
    nb_channels = hps['nb_channels']
    nb_neurons = hps['nb_neurons']
    dropout = hps['dropout']
    act = hps.setdefault('act', 'relu')

    model = Sequential()
    model.add(Convolution2D(nb_filters[0], nb_conv[0], nb_conv[0],
                            input_shape = (nb_channels, img_size, img_size),
                            init = 'he_uniform', border_mode = 'valid'))
    model.add(Activation(act))
    model.add(MaxPooling2D(pool_size = (nb_pool[0], nb_pool[0])))
    model.add(SpatialDropout2D(dropout[0]))

    model.add(Convolution2D(nb_filters[1], nb_conv[1], nb_conv[1],
                            init='he_uniform', border_mode = 'valid'))
    model.add(Activation(act))
    model.add(MaxPooling2D(pool_size=(nb_pool[1], nb_pool[1])))
    model.add(SpatialDropout2D(dropout[1]))

    model.add(Convolution2D(nb_filters[2], nb_conv[2], nb_conv[2],
                            init='he_uniform', border_mode = 'valid'))
    model.add(Activation(act))
    model.add(MaxPooling2D(pool_size=(nb_pool[2], nb_pool[2])))
    model.add(SpatialDropout2D(dropout[2]))

    model.add(Flatten())

    model.add(Dense(nb_neurons))
    model.add(Activation(act))
    model.add(Dropout(dropout[3]))

    model.add(Dense(2))
    model.add(Activation('softmax'))

    model.compile(loss = 'categorical_crossentropy', optimizer = 'adam',
                  metrics = ['accuracy'])
    model.summary()

    return model

def dense_net_construct(hps):
    layer_dims = hps['layer_dims']
    input_dim = hps['input_dim']

    model = Sequential()
    for i,dim in enumerate(layer_dims):
        if i == 0:
            model.add(Dense(dim, input_dim = input_dim, init = 'he_uniform'))
        else:
            model.add(Dense(dim, init = 'he_uniform'))
        model.add(Activation(hps.setdefault('act', 'relu')))

    model.add(Dense(2))
    model.add(Activation('softmax'))

    model.compile(loss = 'categorical_crossentropy', optimizer = 'adam',
                  metrics = ['accuracy'])
    model.summary()

    return model


from django.test import TestCase
from django.test import Client
from django.contrib.auth.models import User
from forward_app.core_models import Policy
from forward_app.core_models import Comment
from forward_app.core_models import Popularity
from forward_app.core_models import Thread

"""
Tests for all functionality with Policy object
"""
class PolicyTest(TestCase):

    def setUp(self):
        Policy.objects.create(category=1, name='Universal Health Care', statement='Healthcare for all', description='Medical services to all citizens')

    def testCategory(self):
        policy = Policy.objects.get(name='Universal Health Care')
        self.assertEqual(policy.category, 1)

    def testStatement(self):
        policy = Policy.objects.get(name='Universal Health Care')
        self.assertEqual(policy.statement, 'Healthcare for all')

    def testDescription(self):
        policy = Policy.objects.get(description='Medical services to all citizens')
        self.assertEqual(policy.description, 'Medical services to all citizens')

"""
Tests for all functionality with Popularity object
"""

class PopularityTest(TestCase):

    def setUp(self):
        Policy.objects.create(category=1, name='Universal Health Care', statement='Healthcare for all', description='Medical services to all citizens')

    def testPopularity(self):
        policy = Policy.objects.get(name='Universal Health Care')
        popularity = Popularity.objects.create(policy=policy, likes=200)
        self.assertEquals(popularity.likes, 200)
        self.assertEqual(popularity.policy, policy)

"""
Tests for all functionality with Comment object
"""

class CommentTest(TestCase):

    def setUp(self):
        Policy.objects.create(category=1, name='Universal Health Care', statement='Healthcare for all', description='Medical services to all citizens')

    def testComment(self):
        user = User.objects.create_user(username='brian')
        policy = Policy.objects.get(name='Universal Health Care')
        popularity = Popularity.objects.create(policy=policy, likes=20)
        thread = Thread.objects.create(popularity=popularity)

        comment = Comment.objects.create(user=user, thread=thread, content='good policy', likes=5)
        thread.lead_comment_id = comment.id
        comment.next_comment_id = comment.id

        self.assertEqual(thread.popularity, popularity)
        self.assertEqual(comment.user, user)
        self.assertEqual(comment.content, 'good policy')
        self.assertEqual(comment.likes, 5)

    def testThread(self):
        user = User.objects.create_user(username='brian')
        policy = Policy.objects.get(name='Universal Health Care')
        popularity = Popularity.objects.create(policy=policy, likes=20)
        thread = Thread.objects.create(popularity=popularity)
        comment = Comment.objects.create(user=user, thread=thread, content='good policy', likes=5)
        thread.lead_comment_id = comment.id
        comment.next_comment_id = comment.id

        comment2 = Comment.objects.create(user=user, thread=thread, content='I agree', likes=2)
        comment2.next_commend_id = comment2.id
        comment.next_comment_id = comment2.id

        self.assertEqual(Comment.objects.get(id=thread.lead_comment_id), comment)
        self.assertEqual(comment.next_comment_id, comment2.id)


# -*- coding: utf-8 -*-
"""
# -*- coding: utf-8 -*-
@author: cami_
Created on Thu Feb 27 11:02:16 2020
NOISYmputer python FILTERING ON HTZ %, MISSING DATA %, MAF
@author: Chrystian Sosa
@Correspondence author: Dr Mathias Lorieux
"""
#IMPORTING MODULES
import pandas as pd
import os.path
from tqdm import tqdm

#nthreads = 4
#size = 10**5  # CHANGED

def filtering_snp_step_chr(chr_dir,n_chrs,maxFreqMD,maxFreqH,minFreqA,minFreqB,log_dir):
    for i in tqdm(range(len(n_chrs)),desc="filtering snps"):
        n_chr=str(n_chrs[i])
        print(n_chr)
        filtering_snp_step(chr_dir,n_chr,maxFreqMD,maxFreqH,minFreqA,minFreqB)
    return("DONE!")

def filtering_snp_step(chr_dir,n_chr,maxFreqMD,maxFreqH,minFreqA,minFreqB):

    export_file_path = (chr_dir+"/"+"Chr"+n_chr+"_step_2.txt")
    export_file_path_filt = (chr_dir+"/"+"Chr"+n_chr+"_step_3.txt")

    retain =[]
    with open(export_file_path) as fp:
       for line in fp:
            nbA = line.count("A")
            nbB = line.count("B")
            nbH = line.count("H")
            nbMD = line.count("-")
            Npop = nbA + nbB + nbH + nbMD

            if Npop > 0:
                freqA = nbA / Npop
                freqB = nbB / Npop
                freqH = nbH / Npop
                freqMD = nbMD / Npop

                if freqH < maxFreqH and freqMD < maxFreqMD and freqA > minFreqA and freqB > minFreqB:
                    retain.append(True)
                else:
                    retain.append(False)
            else:
                retain.append(False)

    #retain.remove(0)
    del retain[0]
    if(os.path.isfile(export_file_path_filt)):
        return(print(export_file_path+" already processed!"))
    else:

        split_file2 = pd.read_csv(export_file_path, sep=" ")
        split_file = pd.read_csv(export_file_path, sep=" ")
        split_file['filtered'] = retain
        split_file = split_file[split_file.filtered==True]
        split_file = split_file.drop(columns=['filtered'])
#        split_file = split_file.drop_duplicates()
        split_file.to_csv(export_file_path_filt,index = False, header=True,sep=" ")

        changed =split_file2.shape[0]-split_file.shape[0]
        data =[["step","prefiltering"],
                                  ["chr",n_chr],
                                  ["changes",changed]]
        log_file_df = pd.DataFrame(data,columns=["item","status"])
        log_file_df.to_csv(log_dir+"/"+"Chr"+n_chr+"_step3.log",index = False, header=True)

        return(print(export_file_path+" filtered!"))
########################################

#chr_dir = "E:/CHR"
#maxFreqMD = 0.666
#maxFreqH = 0.800
#minFreqA = 0.010
#minFreqB = 0.010
#n_chrs=12
#n_chr = str(1)
#x_filtered = filtering_snp_step(chr_dir,n_chr,maxFreqMD,maxFreqH,minFreqA,minFreqB)

#filtering_snp_step_chr(chr_dir,n_chrs,maxFreqMD,maxFreqH,minFreqA,minFreqB)