#Import required modules
import cv2
import dlib
import numpy as np
import os
import csv
import statistics

#Set up some required objects

#video_capture = cv2.VideoCapture(0) #Webcam object
#video_capture = cv2.VideoCapture('SourceShort4.mp4')
fourcc = cv2.VideoWriter_fourcc(*'XVID')
video_capture = cv2.VideoCapture('0001-0483.avi')
video_output = cv2.VideoWriter('output.avi', fourcc, 60.0, (1920,1080))
detector = dlib.get_frontal_face_detector() #Face detector
predictor = dlib.shape_predictor("Dataset/shape_predictor_68_face_landmarks.dat") #Landmark identifier. Set the filename to whatever you named the downloaded file
cv2.useOptimized()
#---------------------------------------------------
#Parameters:

frame_interval = 3
pitch_normalisation_value = 0.01
yaw_normalisation_value = 0.005
pupil_threshold = 30
EAR_blink_threshold = 0.16

#---------------------------------------------------

# POSE ESTIMATION:
# 3D model points for head tracking.
model_points = np.array([
    (0.0, 0.0, 0),  # Nose tip
    (0.0, -210.0, -100.0),  # Chin
    (-130.0, 90.0, -110.0),  # Left eye left corner
    (0, 90.0, -70.0),  # Nose Bridge
    (130.0, 90.0, -110.0),  # Right eye right corner
    (-180.0, -100.0, -280.0),  # Left Jaw
    (180.0, -100.0, -280.0)  # Right Jaw

])

# Camera internals for head tracking:

focal_length = video_capture.get(3)
center = (video_capture.get(3) / 2, video_capture.get(4))
camera_matrix = np.array(
    [[focal_length, 0, center[0]],
     [0, focal_length, center[1]],
     [0, 0, 1]], dtype="double"
)

pitch =[]
yaw = []
roll = []
NER_L = []
NER_R = []
EAR_L = []
EAR_R = []
EX = []
EY = []
IMAR = []
OMAR = []
N2MAR_L = []
N2MAR_R = []
blink = []

head_output_buffer = [pitch,yaw,roll, NER_L, NER_R, EAR_L, EAR_R, EX, EY, IMAR, OMAR, blink, N2MAR_R, N2MAR_L]
final_output = []

while(video_capture.isOpened()):
    ret, frame = video_capture.read()
    if ret:
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)) #Histogram Equalisation
        clahe_image = clahe.apply(gray)
        detections = detector(clahe_image, 1) #Detect the faces in the image

        if (detections):
            shape = predictor(clahe_image, detections[0]) #Get coordinates

        for i in range(0,68): #There are 69 landmark points on each face
            cv2.circle(frame, (shape.part(i).x, shape.part(i).y), 2, (0,0,0), thickness=1) #For each point, draw a red circle with thickness2 on the original frame

        for i in range(0,16):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on Jaw

        for i in range(17,21):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on Eyebrow

        for i in range(22,26):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on Eyebrow

        for i in range(27,35):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on Nose

        cv2.line(frame, (shape.part(35).x, shape.part(35).y), (shape.part(30).x, shape.part(30).y), (0, 0, 0), thickness=1, lineType=8)  #Line on Nose
        cv2.line(frame, (shape.part(27).x, shape.part(27).y), (shape.part(31).x, shape.part(31).y), (0, 0, 0), thickness=1, lineType=8)  # Last line on Nose
        cv2.line(frame, (shape.part(27).x, shape.part(27).y), (shape.part(35).x, shape.part(35).y), (0, 0, 0), thickness=1, lineType=8)  # Last line on Nose

        for i in range(48,59):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on Mouth
        cv2.line(frame, (shape.part(59).x, shape.part(59).y), (shape.part(48).x, shape.part(48).y), (0, 0, 0), thickness=1, lineType=8)  # Line on Mouth

        for i in range(60,67):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on inner Mouth
        cv2.line(frame, (shape.part(67).x, shape.part(67).y), (shape.part(60).x, shape.part(60).y), (0, 0, 0), thickness=1, lineType=8)  # Line on inner Mouth

        for i in range(36,41):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on eye
        cv2.line(frame, (shape.part(41).x, shape.part(41).y), (shape.part(36).x, shape.part(36).y), (0, 0, 0), thickness=1, lineType=8)  # Line on eye

        for i in range(42,47):
            cv2.line(frame, (shape.part(i).x, shape.part(i).y), (shape.part(i+1).x, shape.part(i+1).y), (0,0,0),thickness=1,lineType = 8) #Line on eye
        cv2.line(frame, (shape.part(47).x, shape.part(47).y), (shape.part(42).x, shape.part(42).y), (0, 0, 0), thickness=1, lineType=8)  # Line on eye

#------------------------------------------------------------------------------------
        #Head Tracking:

        # 2D points for Head Tracking
        image_points = np.array([
            (shape.part(30).x, shape.part(30).y),  # Nose tip
            (shape.part(8).x, shape.part(8).y),  # Chin
            (shape.part(45).x, shape.part(45).y),  # Left eye left corner
            (shape.part(27).x, shape.part(27).y),  # Nose Bridge
            (shape.part(36).x, shape.part(36).y),  # Right eye right corner
            (shape.part(12).x, shape.part(12).y),  # Left Jaw
            (shape.part(4).x, shape.part(4).y)  # Right Jaw
        ], dtype="double")

        dist_coeffs = np.zeros((4, 1))  # Assuming no lens distortion
        (success, rotation_vector, translation_vector) = cv2.solvePnP(model_points, image_points, camera_matrix,
                                                                          dist_coeffs)

        # Project a 3D point (0, 0, 1000.0) onto the image plane.
        # We use this to draw a line sticking out of the nose

        (nose_end_point2D, jacobian) = cv2.projectPoints(np.array([(0.0, 0.0, 1000.0)]), rotation_vector,
                                                             translation_vector, camera_matrix, dist_coeffs)

        for p in image_points:
            cv2.circle(frame, (int(p[0]), int(p[1])), 2, (255, 0, 0), -1)

        nose_tip = (int(image_points[0][0]), int(image_points[0][1]))
        nose_target = (int(nose_end_point2D[0][0][0]), int(nose_end_point2D[0][0][1]))

        cv2.arrowedLine(frame, nose_tip, nose_target, (255, 0, 0), 1)

        rotation_matrix = np.zeros((3, 3))
        cv2.Rodrigues(rotation_vector,rotation_matrix,jacobian)

        projection_matrix = np.array(
            [[rotation_matrix[0][0], rotation_matrix[0][1], rotation_matrix[0][2] , translation_vector[0]],
            [rotation_matrix[1][0], rotation_matrix[1][1], rotation_matrix[1][2], translation_vector[1]],
            [rotation_matrix[2][0], rotation_matrix[2][1], rotation_matrix[2][2], translation_vector[2]]], dtype="double"
        )

        euler_angles = cv2.decomposeProjectionMatrix(projection_matrix)
        euler_angles = euler_angles[6]

        nose_bridge = np.array([shape.part(27).x, shape.part(27).y])

        yaw = -euler_angles[1][0]
        pitch = -(nose_tip[1] - nose_target[1]) * 0.05

        if abs(euler_angles[2][0] % 180) <= abs(180 - (euler_angles[2][0] % 180)):
            roll = (euler_angles[2][0] % 180)
        else:
            roll = -1*(180 - (euler_angles[2][0] % 180))

        roll = - roll

        cv2.putText(frame,
                    'yaw = '+ str(yaw), (32, 32), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)
        cv2.putText(frame,
                    'pitch = ' + str(pitch), (32, 64), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)
        cv2.putText(frame,
                    'roll = ' + str(roll), (32, 96), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)

#--------------------------------------------------------------------------------------------
#   NER - normalized eyebrow ratios for eyebrow movement

        cv2.line(frame, (shape.part(17).x, shape.part(17).y), (shape.part(27).x, shape.part(27).y), (0, 255, 0),
                 thickness=1, lineType=8)  # Line between outer eyebrow and nose bridge
        cv2.line(frame, (shape.part(27).x, shape.part(27).y), (shape.part(26).x, shape.part(26).y), (0, 255, 0),
                 thickness=1, lineType=8)  # Line between outer eyebrow and nose bridge

        outer_brow1 = np.array([shape.part(17).x, shape.part(17).y])
        outer_brow2 = np.array([shape.part(26).x, shape.part(26).y])
        nose_bridge = np.array([shape.part(27).x, shape.part(27).y])

        p3 = np.array([shape.part(18).x, shape.part(18).y])
        d1 = np.linalg.norm(np.cross(outer_brow1 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow1 - nose_bridge)

        p3 = np.array([shape.part(19).x, shape.part(19).y])
        d2 = np.linalg.norm(np.cross(outer_brow1 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow1 - nose_bridge)

        p3 = np.array([shape.part(20).x, shape.part(20).y])
        d3 = np.linalg.norm(np.cross(outer_brow1 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow1 - nose_bridge)

        p3 = np.array([shape.part(21).x, shape.part(21).y])
        d4 = np.linalg.norm(np.cross(outer_brow1 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow1 - nose_bridge)

        ner_r =(d1 + d2 + d3 + d4) / (4 * np.linalg.norm(outer_brow1 - nose_bridge))
        ner_r = ner_r * (1 + (pitch * pitch_normalisation_value))

        p3 = np.array([shape.part(25).x, shape.part(25).y])
        d1 = np.linalg.norm(np.cross(outer_brow2 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow2 - nose_bridge)

        p3 = np.array([shape.part(24).x, shape.part(24).y])
        d2 = np.linalg.norm(np.cross(outer_brow2 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow2 - nose_bridge)

        p3 = np.array([shape.part(23).x, shape.part(23).y])
        d3 = np.linalg.norm(np.cross(outer_brow2 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow2 - nose_bridge)

        p3 = np.array([shape.part(22).x, shape.part(22).y])
        d4 = np.linalg.norm(np.cross(outer_brow2 - nose_bridge, nose_bridge - p3)) / np.linalg.norm(outer_brow2 - nose_bridge)

        ner_l = (d1 + d2 + d3 + d4) / (4 * np.linalg.norm(outer_brow2 - nose_bridge))
        ner_l = ner_l * (1 + (pitch * pitch_normalisation_value))

        cv2.putText(frame,
                    'NER_L = ' + str(ner_l), (32, 128), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)
        cv2.putText(frame,
                    'NER_R = ' + str(ner_r), (32, 160), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)

#--------------------------------------------------------------------------------------------
# EAR -Eye Aspect Ratio for eyes closing

        p1 = np.array([shape.part(36).x, shape.part(36).y])
        p2 = np.array([shape.part(37).x, shape.part(37).y])
        p3 = np.array([shape.part(38).x, shape.part(38).y])
        p4 = np.array([shape.part(39).x, shape.part(39).y])
        p5 = np.array([shape.part(40).x, shape.part(40).y])
        p6 = np.array([shape.part(41).x, shape.part(41).y])

        ear_l = (np.linalg.norm(p2 - p6) + np.linalg.norm(p3 - p5)) / (2 * np.linalg.norm(p1 - p4))
        ear_l = ear_l * (1 - (np.abs(yaw)*yaw_normalisation_value))

        p1 = np.array([shape.part(42).x, shape.part(42).y])
        p2 = np.array([shape.part(43).x, shape.part(43).y])
        p3 = np.array([shape.part(44).x, shape.part(44).y])
        p4 = np.array([shape.part(45).x, shape.part(45).y])
        p5 = np.array([shape.part(46).x, shape.part(46).y])
        p6 = np.array([shape.part(47).x, shape.part(47).y])

        ear_r = (np.linalg.norm(p2 - p6) + np.linalg.norm(p3 - p5)) / (2 * np.linalg.norm(p1 - p4))
        ear_r = ear_r * (1 - (np.abs(yaw) * yaw_normalisation_value))

        blink = 0

        # and (final_output[-1][11] == 0) ?
        if (final_output and ((ear_r + ear_l )/ 2) < EAR_blink_threshold):
            blink = 1

        cv2.putText(frame,
                    'EAR_L = ' + str(ear_l), (32, 192), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)
        cv2.putText(frame,
                    'EAR_R = ' + str(ear_r), (32, 224), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)

        # -------------------------------------------
        # Eye Tracking
        left_eye_region = np.array([(shape.part(42).x, shape.part(42).y),
                                    (shape.part(43).x, shape.part(43).y),
                                    (shape.part(44).x, shape.part(44).y),
                                    (shape.part(45).x, shape.part(45).y),
                                    (shape.part(46).x, shape.part(46).y),
                                    (shape.part(47).x, shape.part(47).y)], np.int32)

        height, width, _ = frame.shape
        mask = np.ones((height, width), np.uint8)
        cv2.polylines(mask, [left_eye_region], True, 255, 2)
        cv2.fillPoly(mask, [left_eye_region], 255)
        left_eye = cv2.bitwise_and(gray, gray, mask=mask)

        min_x = np.min(left_eye_region[:, 0])
        max_x = np.max(left_eye_region[:, 0])
        min_y = np.min(left_eye_region[:, 1])
        max_y = np.max(left_eye_region[:, 1])
        gray_eye = left_eye[min_y: max_y, min_x: max_x]
        gray_eye = cv2.GaussianBlur(gray_eye, (3,3), 0)

        _, threshold_eye = cv2.threshold(gray_eye, pupil_threshold, 255, cv2.THRESH_BINARY_INV)
        M = cv2.moments(threshold_eye)

        if (M["m10"] != 0):
            cX_la = M["m10"] / M["m00"]
            cY_la = M["m01"] / M["m00"]
            cX_l = cX_la / (max_x - min_x)
            cY_l = cY_la / (max_y - min_y)
        else:
            cX_l = 0.5
            cY_l = 0.5
            cX_la = (max_x - min_x) / 2
            cY_la = (max_y - min_y) / 2

        cv2.circle(frame, (min_x + int(cX_la), min_y + int(cY_la)), 2, (0, 0, 255), thickness=1)

        right_eye_region = np.array([(shape.part(36).x, shape.part(36).y),
                                    (shape.part(37).x, shape.part(37).y),
                                    (shape.part(38).x, shape.part(38).y),
                                    (shape.part(39).x, shape.part(39).y),
                                    (shape.part(40).x, shape.part(40).y),
                                    (shape.part(41).x, shape.part(41).y)], np.int32)

        cv2.polylines(mask, [right_eye_region], True, 255, 2)
        cv2.fillPoly(mask, [right_eye_region], 255)
        right_eye = cv2.bitwise_and(gray, gray, mask=mask)

        min_x = np.min(right_eye_region[:, 0])
        max_x = np.max(right_eye_region[:, 0])
        min_y = np.min(right_eye_region[:, 1])
        max_y = np.max(right_eye_region[:, 1])
        gray_eye = right_eye[min_y: max_y, min_x: max_x]
        gray_eye = cv2.GaussianBlur(gray_eye, (3, 3), 0)

        _, threshold_eye = cv2.threshold(gray_eye, pupil_threshold, 255, cv2.THRESH_BINARY_INV)
        M = cv2.moments(threshold_eye)

        if (M["m10"] != 0):
            cX_ra = M["m10"] / M["m00"]
            cY_ra = M["m01"] / M["m00"]
            cX_r = cX_ra / (max_x - min_x)
            cY_r = cY_ra / (max_y - min_y)
        else:
            cX_r = 0.5
            cY_r = 0.5
            cX_ra = (max_x - min_x) / 2
            cY_ra = (max_y - min_y) / 2

        cv2.circle(frame, (min_x + int(cX_ra), min_y + int(cY_ra)), 2, (0, 0, 255), thickness=1)

        eX = (cX_l + cX_r) / 2
        eY = min((cY_l + cY_r) / 2, 1)

        cv2.putText(frame,
                    'EX = ' + str(eX), (32, 256), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)
        cv2.putText(frame,
                    'EY = ' + str(eY), (32, 288), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)

        if final_output and final_output[-1][11] == 1.0:
            cv2.putText(frame,
                        'BLINK', (32, 384), cv2.FONT_HERSHEY_SIMPLEX,
                        0.75, (0, 0, 255), 1, cv2.LINE_AA)

# --------------------------------------------------------------------------------------------
# OMAR - Outer Mouth Aspect Ratio for mouth stretching, smiling

        p1 = np.array([shape.part(48).x, shape.part(48).y])
        p2 = np.array([shape.part(49).x, shape.part(49).y])
        p3 = np.array([shape.part(50).x, shape.part(50).y])
        p4 = np.array([shape.part(51).x, shape.part(51).y])
        p5 = np.array([shape.part(52).x, shape.part(52).y])
        p6 = np.array([shape.part(53).x, shape.part(53).y])
        p7 = np.array([shape.part(54).x, shape.part(54).y])
        p8 = np.array([shape.part(55).x, shape.part(55).y])
        p9 = np.array([shape.part(56).x, shape.part(56).y])
        p10 = np.array([shape.part(57).x, shape.part(57).y])
        p11 = np.array([shape.part(58).x, shape.part(58).y])
        p12 = np.array([shape.part(59).x, shape.part(59).y])
        p13 = np.array([shape.part(61).x, shape.part(61).y])
        p14 = np.array([shape.part(62).x, shape.part(62).y])
        p15 = np.array([shape.part(63).x, shape.part(63).y])
        p16 = np.array([shape.part(65).x, shape.part(65).y])
        p17 = np.array([shape.part(66).x, shape.part(66).y])
        p18 = np.array([shape.part(67).x, shape.part(67).y])

        omar = (np.linalg.norm(p2 - p12) + np.linalg.norm(p3 - p13) + np.linalg.norm(p4 - p14) +
                np.linalg.norm(p5 - p15) + np.linalg.norm(p6 - p8) + np.linalg.norm(p16 - p9) +
                np.linalg.norm(p16 - p9) + np.linalg.norm(p17 - p10) + np.linalg.norm(p18 - p11)) /\
                (5* + np.linalg.norm(p1 - p7))

        omar = omar * (1 - (np.abs(yaw) * yaw_normalisation_value))

        cv2.putText(frame,
                    'OMAR = ' + str(omar), (32, 320), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)

# --------------------------------------------------------------------------------------------
# IMAR - Inner Mouth Aspect Ratio for showing teeth

        imar = (5 * (np.linalg.norm(p13 - p18) + np.linalg.norm(p14 - p17) + np.linalg.norm(p15 - p16))) /\
               (3 * (np.linalg.norm(p2 - p12) + np.linalg.norm(p3 - p11) + np.linalg.norm(p4 - p10) +
                np.linalg.norm(p5 - p9) + np.linalg.norm(p6 - p8)))

        cv2.putText(frame,
                    'IMAR = ' + str(imar), (32, 352), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (255, 0, 0), 1, cv2.LINE_AA)

# --------------------------------------------------------------------------------------------
# N2MAR - Nose to Mouth Ratios
        p1 = np.array([shape.part(27).x, shape.part(27).y]) #nose bridge
        p2 = np.array([shape.part(31).x, shape.part(31).y]) #right nose wing
        p3 = np.array([shape.part(35).x, shape.part(35).y]) #left nose wing
        p4 = np.array([shape.part(32).x, shape.part(32).y]) #right nostril
        p5 = np.array([shape.part(34).x, shape.part(34).y]) #left nostril
        p6 = np.array([shape.part(33).x, shape.part(33).y]) #nose bottom
        p7 = np.array([shape.part(48).x, shape.part(48).y]) # right lip corner
        p8 = np.array([shape.part(49).x, shape.part(49).y]) # right lip 1
        p9 = np.array([shape.part(50).x, shape.part(50).y]) # right lip 2
        p10 = np.array([shape.part(51).x, shape.part(51).y]) # lip center
        p11 = np.array([shape.part(52).x, shape.part(52).y]) # left lip 2
        p12 = np.array([shape.part(53).x, shape.part(53).y]) # left lip 1
        p13= np.array([shape.part(54).x, shape.part(54).y]) # left lip corner

        d1 = np.linalg.norm(np.cross(p2 - p3, p3 - p7)) / np.linalg.norm(
            p2 - p3)

        d2 = np.linalg.norm(np.cross(p2 - p3, p3 - p8)) / np.linalg.norm(
            p2 - p3)

        d3 = np.linalg.norm(np.cross(p2 - p3, p3 - p9)) / np.linalg.norm(
            p2 - p3)

        d4 = np.linalg.norm(np.cross(p2 - p3, p3 - p10)) / np.linalg.norm(
            p2 - p3)

        n2mar_r = (d1 + d2) / (d3 + d4)

        d1 = np.linalg.norm(np.cross(p2 - p3, p3 - p13)) / np.linalg.norm(
            p2 - p3)

        d2 = np.linalg.norm(np.cross(p2 - p3, p3 - p12)) / np.linalg.norm(
            p2 - p3)

        d3 = np.linalg.norm(np.cross(p2 - p3, p3 - p11)) / np.linalg.norm(
            p2 - p3)

        d4 = np.linalg.norm(np.cross(p2 - p3, p3 - p10)) / np.linalg.norm(
            p2 - p3)

        n2mar_l = (d1 + d2) / (d3 + d4)

        n2mar_l = n2mar_l * (1 + (pitch * pitch_normalisation_value * 2))
        n2mar_r = n2mar_r * (1 + (pitch * pitch_normalisation_value * 2))

        cv2.putText(frame,
                    'N2MAR_R = ' + str(n2mar_r), (32, 416), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (0, 255, 0), 1, cv2.LINE_AA)

        cv2.putText(frame,
                    'N2MAR_L = ' + str(n2mar_l), (32, 448), cv2.FONT_HERSHEY_SIMPLEX,
                    0.75, (0, 255, 0), 1, cv2.LINE_AA)

# --------------------------------------------------------------------------------------------

        cv2.imshow("image", frame) #Display the frame
        video_output.write(frame)

        head_output_buffer[0].append(yaw)
        head_output_buffer[1].append(pitch)
        head_output_buffer[2].append(roll)
        head_output_buffer[3].append(ner_l)
        head_output_buffer[4].append(ner_r)
        head_output_buffer[5].append(ear_l)
        head_output_buffer[6].append(ear_r)
        head_output_buffer[7].append(eX)
        head_output_buffer[8].append(eY)
        head_output_buffer[9].append(omar)
        head_output_buffer[10].append(imar)
        head_output_buffer[11].append(((ear_r + ear_l )/ 2))
        head_output_buffer[12].append(n2mar_r)
        head_output_buffer[13].append(n2mar_l)

        if (len(head_output_buffer[0]) >= frame_interval):
            print(str(statistics.median(head_output_buffer[0])) + ', ' + str(statistics.median(head_output_buffer[1])) + ', ' + str(statistics.median(head_output_buffer[2])) + ', ' + str(statistics.median(head_output_buffer[3])) + ', ' + str(statistics.median(head_output_buffer[4])))
            final_output.append([statistics.median(head_output_buffer[0]),
                                 statistics.median(head_output_buffer[1]),
                                 statistics.median(head_output_buffer[2]),
                                 statistics.median(head_output_buffer[3]),
                                 statistics.median(head_output_buffer[4]),
                                 statistics.median(head_output_buffer[5]),
                                 statistics.median(head_output_buffer[6]),
                                 statistics.median(head_output_buffer[7]),
                                 statistics.median(head_output_buffer[8]),
                                 statistics.median(head_output_buffer[9]),
                                 statistics.median(head_output_buffer[10]),
                                 np.min(head_output_buffer[11]),
                                 statistics.median(head_output_buffer[12]),
                                 statistics.median(head_output_buffer[13])])

            for datum in head_output_buffer:
                datum.clear()

        if cv2.waitKey(1) & 0xFF == ord('q'):  # Exit program when the user presses 'q'
            break
    else:
        if (len(head_output_buffer[0]) > 0):
            final_output.append([statistics.median(head_output_buffer[0]),
                                 statistics.median(head_output_buffer[1]),
                                 statistics.median(head_output_buffer[2]),
                                 statistics.median(head_output_buffer[3]),
                                 statistics.median(head_output_buffer[4]),
                                 statistics.median(head_output_buffer[5]),
                                 statistics.median(head_output_buffer[6]),
                                 statistics.median(head_output_buffer[7]),
                                 statistics.median(head_output_buffer[8]),
                                 statistics.median(head_output_buffer[9]),
                                 statistics.median(head_output_buffer[10]),
                                 np.max(head_output_buffer[11]),
                                 statistics.median(head_output_buffer[12]),
                                 statistics.median(head_output_buffer[13])])

        video_capture.release()
        video_output.release()
        cv2.destroyAllWindows()

with open('output.csv', mode='w') as employee_file:
    writer = csv.writer(employee_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

    writer.writerow(['yaw', 'pitch', 'roll', 'NER_L', 'NER_R', 'EAR_L', 'EAR_R', 'E_X', 'E_Y', 'OMAR', 'IMAR'])
    for row in final_output:
        writer.writerow(row)


import math
import copy
import numpy as np
from random import randrange
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

class Node:
    def __init__(self, probs, state, probs_mask, reward, terminal, parent = None):
        self.state = state
        self.P = probs * probs_mask
        probs_sum = self.P.sum().item()
        length = len(self.P)

        if probs_sum == 0.0:
            mask_sum = probs_mask.sum().item()
            if mask_sum == 0.0:
                self.P = torch.ones(length, dtype = torch.float32)
            else:
                self.P = probs_mask * (1.0 / mask_sum)
        else:
            self.P = self.P / probs_sum

        self.N = torch.zeros(length, dtype = torch.int16)
        self.Q = torch.zeros(length, dtype = torch.float32)
        self.R = reward
        self.T = terminal

        self.children = {}
        self.parent = parent
class MCTS:
    def __init__(self, root, env, c1, c2, gamma):
        self.root = root
        self.env = env

        self.c1 = c1
        self.c2 = c2
        self.gamma = gamma

    def selection(self):
        node = self.root
        actions = []

        while True:
            if node.T == True:
                return node, -1, actions

            sum = node.N.sum().item()
            if sum > 0:
                c = self.c1 + math.log((sum + self.c2 + 1.0) / self.c2)
                u = node.Q + node.P * c * (math.sqrt(sum) / (1.0 + node.N))

                u_max = -10000
                a = 0
                for i in range(len(u)):
                    if node.P[i] > 0.0:
                        if u[i] > u_max:
                            a = i
                            u_max = u[i]
            else:
                a = torch.argmax(node.P).item()
            actions.append(a)

            if a in node.children:
                node = node.children[a]
            else:
                return node, a, actions
    def expansion(self, node, action, probs, state, probs_mask, reward, terminal):
        new_node = Node(probs, state, probs_mask, reward, terminal, node)
        node.children[action] = new_node
        return new_node
    def backup(self, node, value, actions):
        v = node.R + self.gamma * value * (1 - int(node.T))
        actions.reverse()

        for a in actions:
            node = node.parent
            v = node.R - self.gamma * v
            node.Q[a] = (node.N[a] * node.Q[a] + v) / (node.N[a] + 1)
            node.N[a] += 1
class Game:
    def __init__(self, states, values, policy):
        self.states = states
        self.target_values = values
        self.target_policy = policy
class ReplayBuffer:
    def __init__(self, max_size):
        self.max_size = max_size
        self.buffer = []

    def store(self, game):
        if len(self.buffer) > self.max_size:
            self.buffer.pop(0)
        self.buffer.append(game)

    def sample(self, batch_size):
        length = len(self.buffer)
        game_indexes = np.random.choice(length, batch_size)

        states, target_values, target_policy = [], [], []
        for index in game_indexes:
            game = self.buffer[index]
            action_index = randrange(len(game.states))
            states.append(game.states[action_index])
            target_values.append(game.target_values[action_index])
            target_policy.append(game.target_policy[action_index])

        return torch.stack(states), torch.stack(target_values), torch.stack(target_policy)

class AZAgent:
    def __init__(self, env, model, replay_buffer, actions, simulation_count = 800, gamma = 0.997, training_steps = 5000, c1 = 1.25, c2 = 19652.0, temperature_update_steps = 100000,
                 dirichlet_alpha = 0.1, exploration_fraction = 0.25, weight_decay = 0.0001, lr = 2e-2, arena_count = 25, self_play_steps = 100, batch_size = 512, name = 'az',
                 run_iteration = 1000, steps = 0):
        self.env = env
        self.model = model
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        print('device: ', self.device)
        self.model.to(self.device)
        self.weight_decay = weight_decay
        self.optimizer = optim.Adam(self.model.parameters(), lr=lr, weight_decay = weight_decay)

        self.replay_buffer = replay_buffer
        self.actions = actions

        self.dirichlet_alpha = dirichlet_alpha
        self.exploration_fraction = exploration_fraction

        self.simulation_count = simulation_count
        self.arena_count = arena_count
        self.self_play_steps = self_play_steps
        self.training_steps = training_steps
        self.steps = steps
        self.batch_size = batch_size
        self.run_iteration = run_iteration + 1

        self.name = name

        self.c1 = c1
        self.c2 = c2
        self.gamma = gamma
        self.lr = lr

        self.T = 1
        self.temperature_update_steps = temperature_update_steps
        self.update_parameters()

    def update_parameters(self):
        if self.steps == 3 * self.temperature_update_steps:
            self.T = 0.25
            self.lr = 1e-5
            self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay = self.weight_decay)
        elif self.steps == 2 * self.temperature_update_steps:
            self.T = 0.5
            self.lr = 1e-4
            self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay = self.weight_decay)
        elif self.steps == self.temperature_update_steps:
            self.lr = 1e-3
            self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay = self.weight_decay)

    def run_mcts(self, state, moves, env, model, training = False):
        logit, value = model.forward(state.unsqueeze(0).to(self.device).float())

        probs = F.softmax(logit.cpu(), dim=-1)
        if training == True:
            noise = torch.tensor(np.random.gamma(self.dirichlet_alpha, 1, self.actions))
            probs = probs[0] * (1 - self.exploration_fraction) + noise * self.exploration_fraction
        else:
            probs = probs[0]

        root = Node(probs, state.cpu(), moves, 0, False)
        mcts = MCTS(root, env, self.c1, self.c2, self.gamma)

        for simulation in range(self.simulation_count):
            node, a, actions = mcts.selection()
            if a > -1:
                new_state, reward, terminal, new_moves = env.step(a, node.state)
                if terminal:
                    probs, value = torch.tensor(self.actions).float(), 0
                else:
                    logit, value = model.forward(new_state.unsqueeze(0).to(self.device).float())
                    probs = F.softmax(logit.cpu(), dim=-1)[0]
                    value = value.cpu().item()

                new_node = mcts.expansion(node, a, probs, new_state, new_moves, reward, terminal)
                mcts.backup(new_node, value, actions)
            else:
                mcts.backup(node, 0, actions)

        print('Q', mcts.root.Q)
        print('p', mcts.root.P)
        return mcts.root.N

    def play(self, model1, model2):
        terminal = False
        state, moves = self.env.start()
        player = 1

        states, rewards, target_policy = [], [], []

        logs = ''
        while not terminal:
            with torch.no_grad():
                if player == 1:
                    visit_childs = self.run_mcts(state, moves, self.env, model1)
                else:
                    visit_childs = self.run_mcts(state, moves, self.env, model2)

            temperature_coef = 1.0/self.T
            visit_childs = visit_childs.float()**temperature_coef
            sum = visit_childs.sum().item()
            probs = visit_childs / sum
            action = torch.argmax(probs).item()

            logs += str(action) + ';' + str(probs) + '\n'

            state, reward, terminal, moves = self.env.step(action, state)
            player = - player

            states.append(state.float())
            rewards.append(reward)
            target_policy.append(probs)

        length = len(rewards)
        target_values = torch.zeros(length)

        value = 0
        for i in reversed(range(length)):
            value = rewards[i] - self.gamma * value
            target_values[i] = value

        self.replay_buffer.store(Game(states, target_values, target_policy))

        return reward, logs

    def self_play(self):
        terminal = False
        states, rewards, target_policy = [], [], []
        state, moves = self.env.start()
        while not terminal:
            with torch.no_grad():
                visit_childs = self.run_mcts(state, moves, self.env, self.model, True)
            temperature_coef = 1.0/self.T
            visit_childs = visit_childs.float()**temperature_coef
            sum = visit_childs.sum().item()
            probs = visit_childs / sum
            action = probs.multinomial(num_samples=1).detach().item()

            new_state, reward, terminal, moves = self.env.step(action, state)

            states.append(state.float())
            rewards.append(reward)
            target_policy.append(probs)
            state = new_state

        length = len(rewards)
        target_values = torch.zeros(length)

        value = 0
        for i in reversed(range(length)):
            value = rewards[i] - self.gamma * value
            target_values[i] = value

        self.replay_buffer.store(Game(states, target_values, target_policy))
        return rewards[length-1]

    def learn(self, batch_size):
        states, target_values, target_policy = self.replay_buffer.sample(batch_size)

        logits, values = self.model.forward(states.to(self.device))
        log_probs = F.log_softmax(logits.cpu(), dim=-1)[0]
        values = values.cpu()[0]

        loss_value = (values - target_values)**2
        loss_policy = - (target_policy * log_probs)
        loss = loss_value.mean() + loss_policy.mean()

        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

    def arena(self, model1, model2, iteration):
        wins1, wins2, draft = 0, 0, 0
        text = ''
        min_iteration = max(0, iteration - 10)
        game = 1

        for i in range(min_iteration, iteration):
            print(game)
            model2.load_state_dict(torch.load('models/' + self.name + str(i) + '.pt'))

            text += 'arena game ' + str(game) + '\n'
            result, logs = self.play(model1, model2)
            text += logs

            if result == 1:
                wins1 += 1
            elif result == 0:
                draft += 1
            else:
                wins2 += 1

            game += 1
            text += 'arena game ' + str(game) + '\n'
            result, logs = self.play(model2, model1)
            text += logs

            if result == 1:
                wins2 += 1
            elif result == 0:
                draft += 1
            else:
                wins1 += 1

            game += 1

        print('win 1: ', wins1, 'win 2:', wins2, 'draft: ', draft)
        return wins1 / float(wins1 + wins2 + 1e-10), text

    def run(self):
        for iteration in range(1, self.run_iteration):
            print('iteration: ', iteration)

            for i in range(self.self_play_steps):
                if i % 5 == 0:
                    print('self play ', i)
                self.self_play()

            torch.save(self.model.state_dict(), 'models/' + self.name + str(iteration - 1) + '.pt')

            for i in range(self.training_steps):
                if i % 250 == 0:
                    print('training ', i)
                self.learn(self.batch_size)

            self.steps += self.training_steps
            self.update_parameters()

            model2 = copy.deepcopy(self.model)
            model2.to(self.device)

            print('arena')
            ratio, text = self.arena(self.model, model2, iteration)
            print('arena ratio: ', ratio)

            f = open('logs/az/arena' + str(iteration) + '.txt', "w")
            f.write(text)
            f.close()

            del model2

    def run2(self):
        terminal = False
        rewards, target_policy = [], []
        state, moves = self.env.start2()
        for y in range(6):
            line = ' '
            for x in range(6):
                if state[0, y, x] == 1:
                    line += ' X '
                elif state[1, y, x] == 1:
                    line += ' O '
                else:
                    line += ' _ '
            print(line)

        print('m', moves.reshape(6, 6))
        while not terminal:
            with torch.no_grad():
                visit_childs = self.run_mcts(state, moves, self.env, self.model, False)

            temperature_coef = 1.0/self.T
            print('visit', visit_childs)
            visit_childs = visit_childs.float()**temperature_coef
            sum = visit_childs.sum().item()
            probs = visit_childs / sum
            print('visits probs', probs)
            action = probs.multinomial(num_samples=1).detach().item()

            new_state, reward, terminal, moves = self.env.step(action, state)

            rewards.append(reward)
            state = new_state

            for y in range(6):
                line = ' '
                for x in range(6):
                    if state[0, y, x] == 1:
                        line += ' X '
                    elif state[1, y, x] == 1:
                        line += ' O '
                    else:
                        line += ' _ '
                print(line)

            return -1

        length = len(rewards)
        target_values = torch.zeros(length)

        value = 0
        for i in reversed(range(length)):
            value = rewards[i] - self.gamma * value
            target_values[i] = value

        return rewards[length-1]


import pandas as pd
from scipy.interpolate import interp1d, interp2d
from scipy import stats
import numpy as np
from astropy.io import fits
from sys import exit

import matplotlib.pyplot as plt
import matplotlib.patches as patches
from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,
                               AutoMinorLocator)
import matplotlib.gridspec as gridspec

from glob import glob
import os

from astropy.visualization import (PercentileInterval, LinearStretch,
                                    ImageNormalize, ZScaleInterval)
from astropy import wcs
from astropy import units as u
from astropy.modeling.models import Ellipse2D
from astropy.coordinates import Angle

import WIFISSpectrum as WS

bluelow =  [4827.875, 4946.500, 5142.625]
bluehigh = [4847.875, 4977.750, 5161.375]
linelow =  [4847.875, 4977.750, 5160.125]
linehigh = [4876.625, 5054.000, 5192.625]
redlow =   [4876.625, 5054.000, 5191.375]
redhigh =  [4891.625, 5065.250, 5206.375]

c = 299792.458

def ellipse_region_sauron(cube, center, a, ecc, theta, annulus = False):
    ''' Function that returns a mask that defines the elements that lie
        within an ellipsoidal region. The ellipse can also be annular.

        Inputs:
            Cube:    The data array
            center:  Tuple of the central coordinates (x_0, y_0) (spaxels)
            a:       The semi-major axis length (spaxels).
            ecc:     The eccentricity of the ellipse (0<ecc<1)
            theta:   The rotation angle of the ellipse from vertical (degrees)
                     rotating clockwise.

        Optional Inputs:
            annulus: False if just simple ellipse, otherwise is the INNER
                     annular radius (spaxels)

    '''

    # Define angle, eccentricity term, and semi-minor axis
    an = Angle(theta, 'deg')
    e = np.sqrt(1 - (ecc**2))
    b = a * e

    # Create outer ellipse
    ell_region = Ellipse2D(amplitude=10, x_0 = center[0], y_0 = center[1],\
                a=a, b=b, theta=an.radian)
    x,y = np.mgrid[0:cube.shape[0], 0:cube.shape[1]]
    ell_good = ell_region(x,y)

    if annulus:
        # Define inner ellipse parameters and then create inner mask
        a2 = annulus
        b2 = a2 * e
        print(a2, b2)
        ell_region_inner = Ellipse2D(amplitude=10, x_0 = center[0],\
                y_0 = center[1], a=a2, b=b2, theta=an.radian)

        # Set region of outer mask and within inner ellipse to zero, leaving
        # an annular elliptical region
        ell_inner_good = ell_region_inner(x,y)
        ell_good[ell_inner_good > 0] = 0

    #fig, ax = mpl.subplots(figsize = (12,7))
    #mpl.imshow(np.nansum(cube, axis = 0), origin = 'lower', interpolation = None)
    #mpl.imshow(ell_good, alpha=0.2, origin = 'lower', interpolation = None)
    #mpl.show()

    # Return the mask
    return np.array(ell_good, dtype = bool)

def prepare_sauron(fl, ellipse, write=False, plot=False):

    ff = fits.open(fl)
    data = ff[0].data
    head = ff[0].header
    image = ff[1].data
    table = ff[2].data
    wl = np.arange(data.shape[1])*head['CDELT1'] + head['CRVAL1']

    x = table['A']
    y = table['D']
    SNflat = table['SN']

    xx, yy = np.meshgrid(np.sort(list(set(x))), np.sort(list(set(y))))
    zz = np.zeros(xx.shape)
    ss = np.zeros((xx.shape[0],xx.shape[1],data.shape[1]))
    SN = np.zeros(xx.shape) + 0.0001
    im = np.mean(data, axis = 1)
    #print(xx[0,:].shape, yy[:,0].shape,zz.shape,xx.shape, yy.shape,\
        #len(np.sort(list(set(x)))), len(np.sort(list(set(y)))))
    for i, val in enumerate(zip(x,y)):
        xi = np.where(xx[0,:] == val[0])
        yi = np.where(yy[:,0] == val[1])
        try:
            zz[yi,xi] = im[i]
            SN[yi,xi] = SNflat[i]
            ss[yi,xi,:] = data[i,:]
        except:
            print(xi,yi,i)

    SNspec = np.zeros(ss.shape)
    for i in range(ss.shape[-1]):
        SNspec[:,:,i] = ss[:,:,i] / SN * np.sqrt(ss.shape[-1])

    #ellipse = (cx[0][0],cx[1][0],-28.,5,0.759,False)
    cx = np.where(np.logical_and(xx == 0, yy == 0))
    whgood = ellipse_region_sauron(zz, (cx[0][0], cx[1][0]),\
                    ellipse[3], ellipse[4], ellipse[2], \
                    annulus = ellipse[5])
    whgoodflat = whgood.flatten()

    ssnew = np.moveaxis(ss, 2, 0)
    ssflat = ssnew.reshape(ssnew.shape[0], -1)
    specslice = ssflat[:,whgoodflat]

    specmeanorig = np.nanmean(specslice, axis = 1)
    goodi = np.ones(specslice.shape[1], dtype=bool)
    for i in range(specslice.shape[1]):
        if np.median(specslice[:,i]) < 0.05:
            goodi[i] = False
    specslice = specslice[:,goodi]

    specmean = np.nanmean(specslice, axis = 1)
    specsum = np.nansum(specslice, axis = 1)

    SNnew = np.moveaxis(SNspec, 2, 0)
    SNflat = SNnew.reshape(SNnew.shape[0], -1)
    noiseslice = SNflat[:,whgoodflat]
    noiseslice = noiseslice[:,goodi]
    noisemean = np.sqrt(np.sum(noiseslice**2.0, axis = 1)) / specslice.shape[1]
    noisesum = np.sqrt(np.sum(noiseslice**2.0, axis = 1))

    if plot:
        gg = np.zeros(zz.shape)
        gg[whgood] = 1.0
        fig, ax = plt.subplots(figsize=(8,8))
        #ax.scatter(x,y,c = np.mean(data, axis = 1), s=50)
        ax.imshow(zz, origin='lower', extent = (np.min(x)-0.4,np.max(x)+0.4,\
            np.min(y)-0.4, np.max(y)+0.4), interpolation=None)

        if ellipse[5]:
            e= np.sqrt(1 - (ellipse[4]**2.0))
            an = Angle(ellipse[2],'deg')
            b = ellipse[3]*e
            el = patches.Ellipse((0,0), 2*ellipse[3],2*b, angle=-an.degree+90,
                        linewidth=2, facecolor='none', edgecolor='m')
            ax.add_patch(el)

            b = ellipse[5]*e
            el2 = patches.Ellipse((0,0), 2*ellipse[5],2*b, angle=-an.degree+90,
                        linewidth=2, facecolor='none', edgecolor='r')
            ax.add_patch(el2)

        else:
            e= np.sqrt(1 - (ellipse[4]**2.0))
            an = Angle(ellipse[2],'deg')
            b = ellipse[3]*e
            el = patches.Ellipse((0,0), 2*ellipse[3],2*b, angle=-an.degree+90,
                        linewidth=2, facecolor='none', edgecolor='r')
            ax.add_patch(el)

        plt.show()

        fig, ax = plt.subplots(figsize=(10,6))

        ax.plot(wl, specsum, label='CorrMean')
        #ax.plot(wl, specmeanorig, label='Mean')
        ax.fill_between(wl, specsum - noisesum, y2 = specsum + noisesum,\
                 alpha = 0.3, color='#333333')
        ax.legend()
        plt.show()

        fig, ax = plt.subplots(figsize=(14,10))

        for i in range(specslice.shape[1]):
            ax.plot(wl, specslice[:,i]/specslice[20,i])
        plt.show()

    if write:
        print("Writing extracted spectrum....")
        hdu = fits.PrimaryHDU(specsum)
        hdu2 = fits.ImageHDU(wl, name = 'WL')
        hdu3 = fits.ImageHDU(noisesum, name = 'ERR')
        hdul = fits.HDUList([hdu,hdu2,hdu3])

        hdul.writeto(write, overwrite=True)
        print("Wrote to: ", write)

    return wl, specsum, noisesum


from urllib.request import urlopen
import json
json_usrl = 'https://raw.githubusercontent.com/muxuezi/btc/master/btc_close_2017.json'
response = urlopen(json_usrl)
req = response.read()
with open('btc_colose_2017_urllib.json', 'wb') as f:
    f.write(req)
file_urllib = json.loads(req)

import requests

json_url = 'https://raw.githubusercontent.com/muxuezi/btc/master/btc_close_2017.json'
req = requests.get(json_url)
with open('btc_close_2017_request.json', 'w') as f:
    f.write(req.text)
file_requests = req.json()
print(file_urllib == file_requests)


from django.shortcuts import render, HttpResponse, redirect
from django.contrib.auth import login
from django.contrib.auth.forms import UserCreationForm, AuthenticationForm
from django.contrib.auth.models import User
from .forms import ProfileForm, PostForm
from .models import Profile, Post, City
from django.contrib.auth.decorators import login_required
from django.core.mail import send_mail
from django.conf import settings

# Create your views here.
def home(request):
    return render(request, 'home.html')

def city_index(request):
    cities = City.objects.all()
    return render(request, 'city/city_base.html', {'cities': cities})

def city_detail(request, city_id):
    city = City.objects.get(id = city_id)
    posts = Post.objects.filter(city_id = city.id).order_by('-id')
    context = {
        'city': city,
        'posts': posts,
    }
    return render(request, 'city/detail.html', context)

def city_post(request):
    return render(request, 'city/post.html')

def signup(request):
    error_message = 'Error'
    form = UserCreationForm()
    context = {
        'form': form,
        'error_message': error_message,
    }
    if request.method == 'POST':
        form = UserCreationForm(request.POST)
        form2 = ProfileForm(request.POST)
        print(form)
        if form.is_valid():
            user = form.save()
            login(request, user)
            profile = form2.save(commit=False)
            profile.user_id = user.id
            profile.save()
            return redirect('/profile/', {'profile': profile} )
        else:
            global error
            error = 'User account already exists'
            return render(request, 'registration/signup.html', context)
    else:
        form = UserCreationForm()
        return render(request, 'registration/signup.html', {'form': form})

@login_required
def update(request):
    profile = Profile.objects.get(user=request.user.id)
    if request.method == 'POST':
        form = ProfileForm(request.POST, instance=profile)
        if form.is_valid():
            profile = form.save()
            return redirect('profile')
        else:
            return render(request, 'registration/update.html', {'form': form, 'profile': profile})
    else:
        form = ProfileForm(instance=profile)
        return render(request, 'registration/update.html', {'form': form, 'profile': profile})

@login_required
def profile(request):
    profile = Profile.objects.get(user = request.user.id)
    posts = Post.objects.filter(profile = profile.id)
    return render(request, 'profile.html', {'profile':profile, 'posts': posts})

@login_required
def post_new(request, city_id):
    city = City.objects.get(id = city_id)
    profile = Profile.objects.get(user = request.user.id)
    if request.method == 'POST':
        form = PostForm(request.POST)
        if form.is_valid():
            post = form.save(commit=False)
            post.city_id = city_id
            post.save()
            return redirect('city_detail', city.id)
        else:
            return redirect('/post/new')
    else:
        form = PostForm()
        return render(request, 'post_new.html', {'form': form, 'city': city, 'profile': profile })

@login_required
def post_edit(request, post_id):
    post = Post.objects.get(id=post_id)
    if request.method == 'POST':
        form = PostForm(request.POST, instance=post)
        if form.is_valid():
            post = form.save()
            return redirect('post_detail', post_id)
        else:
            print(form)
            return redirect('post_edit', post_id)
    else:
        form = PostForm()
        return render(request, 'post_edit.html', {'form': form, 'post': post})

def post_detail(request, post_id):
    post = Post.objects.get(id = post_id)
    profile = Profile.objects.get(id = post.profile.id)
    city = City.objects.get(id = post.city.id)
    context = {
        'post': post,
        'profile': profile,
        'city': city,
    }
    return render(request, 'post.html', context)

@login_required
def post_delete(request, post_id):
    post = Post.objects.get(id=post_id)
    city_id = post.city.id
    post.delete()
    return redirect('city_detail', city_id)


from django.shortcuts import render
from products.models import ProductCategory, Product
from django.http.response import HttpResponseRedirect, HttpResponse
from checkout.views import _addCartInfos
from checkout.models import ShoppingCart, Order, OrderForm, STATE_NEW
import json
from django.db.models.query_utils import Q

def index(request):
    context = { }
    addCategories(context)
    _addCartInfos(context, request)

    context['showCase'] = Product.objects.filter(showcase=True)

    return render(request, 'home/index.html', context)

def productListByCategory(request, cat_name):
    prod_list = Product.objects.filter(category__name=cat_name)
    context = {'prod_list' : prod_list}

    addCategories(context)
    _addCartInfos(context, request)
    context['title'] = cat_name

    return render(request, 'home/prodlist.html', context)

def productDetail(request, cat_name, prod_id):
    context = { }
    if request.method == 'POST':
        form = OrderForm(request.POST)
        if form.is_valid():
            _addToCart(request, cat_name, prod_id, form.cleaned_data['size'])
            return HttpResponseRedirect('/home/')
    else:
        form = OrderForm()
        context ['prod'] = Product.objects.get(id=prod_id)
        addCategories(context)
        _addCartInfos(context, request)
        context['title'] =  Product.objects.get(id=prod_id).name

    context['form'] = form
    context['cat_name'] = cat_name
    return render(request, 'home/productDetail.html', context)

def _addToCart(request, cat_name, prod_id, size):

    cartByUser = ShoppingCart.objects.get_or_create(user=request.user, state=STATE_NEW)[0]

    order = Order.objects.create(product=Product.objects.get(id=prod_id))
    order.size = size
    order.save()

    cartByUser.orders.add(order)

    return HttpResponseRedirect('/home/')

def addCategories(context):
    cat_list = ProductCategory.objects.all()
    context['cat_list'] = cat_list

def search_prods(request):
    if request.is_ajax():
        q = request.GET.get('term', '')
        prods = Product.objects.filter(Q(name__icontains = q) |Q(desc__icontains = q))[:20]
        results = []
        for prod in prods:
            prod_json = {}
            prod_json['id'] = prod.id
            prod_json['label'] = prod.name
            prod_json['cat'] = prod.category.name
            results.append(prod_json)
        data = json.dumps(results)
    else:
        data = 'fail'
    mimetype = 'application/json'
    return HttpResponse(data, mimetype)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb 23 15:41:57 2019

@author: karthikchowdary
"""
from bs4 import BeautifulSoup
from bs4.element import Comment
import urllib.request
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer
from nltk import wordpunct_tokenize, pos_tag, ne_chunk, ngrams, FreqDist, sent_tokenize, word_tokenize

file=open("nlp_input.txt","r+")
fileData = file.read()

# Word Tokenization - to extarct each word from the text
tokens = word_tokenize(fileData)

# Applying Lemmatization
lemmatizer = WordNetLemmatizer()
lemmatizerOutput = []
print("Lemmatization Output : \n")
for tok in tokens:
    #itearting through each word and lematizing and appending it to list
    lemmatizerOutput.append(lemmatizer.lemmatize(str(tok)))
print(lemmatizerOutput)

# performing trigram on the  Lemmatizer Output
print("trigrams :\n")
trigramsOutput = []
for tri in ngrams(tokens, 3):
    # Fetching trigrams using 'ngrams' method and Iterating it and appending it to list
    trigramsOutput.append(tri)
print(trigramsOutput)

# triGram- Word Frequency
# Using trigramOutput fetch the WordFreq Details
wordFreq = FreqDist(trigramsOutput)
# Getting Most Common Words and Printing them - Will get the Counts from top to least
mostCommon = wordFreq.most_common()
print("triGrams Frequency (From Top to Least) : \n", mostCommon)
# Fetching the Top 10 trigrams
top10 = wordFreq.most_common(10)
print("Top 10 triGrams : \n", top10)

# Getting Sentences using Sentence Tokenization
sentTokens = sent_tokenize(fileData)
# Creating an Array to append the sentence
concatenatedArray = []
# Iterating the Sentences
for sentence in sentTokens:
    # Iterating the trirams present
    for a, b, c in trigramsOutput:
        # Iterating the Top 10 triGrams
        for ((d, e, f), length) in top10:
            # Comparing the each with each of the Top 10 trigram
            if(a == d, e, f):
                concatenatedArray.append(sentence)

print("Concatenated Array : ",concatenatedArray)
print("Maximum of Concatenated Array : ", max(concatenatedArray))


#!/env/python
# -*- coding: UTF-8 -*-

#输入index文件,输出与其相距为n的全部序列
from itertools import product
import sys

##计算两个字符串的汉明距离
def hanmingDistance(s1,s2):
    if len(s1) != len(s2):
        raise ValueError("Undefined for sequences of unequal length")
    return (sum(x1 != x2 for x1,x2 in zip(s1,s2)))

def diff_(s,n):
    s = s.upper()
    items = ["A","G","C","T"]

    lst_ = []
    for p in product(items,repeat=len(s)):
        lst_.append("".join(p))

    lst2_ = []
    for seq in lst_:
        if hanmingDistance(s,seq) == n:
            lst2_.append(seq)
    return lst2_
s = "AGCT"

idx_lst = []
with open(sys.argv[1],"r") as fi:
    for line in fi:
        line = line.strip("\n")
        if not line.startswith("#"):
            comb_idx = line.split("=")[0]+line.split("=")[1]
            idx_lst.append(comb_idx)
result = []
for idx in idx_lst:
    result.append(diff_(idx[-6:],1))

print(result)


from discord.ext import commands
import discord
from config import (
    BOT_COMMANDS_CHANNEL_ID,
    BOT_GAMES_CHANNEL_ID,
    CHALLENGE_HOST_ROLE_ID,
    CHALLENGE_HOST_HELPER_ROLE_ID,
    CHALLENGE_INFO_CHANNEL_ID,
    CHALLENGE_PARTICIPANT_ROLE_ID,
    CHALLENGE_SUBMIT_CHANNEL_ID,
    CHALLENGE_SUBMITTED_ROLE_ID,
    CHALLENGE_WINNER_ROLE_ID,
    STAFF_ROLE_ID,
)

def setup(bot):
    bot.add_cog(ChallengeHandler(bot))

class ChallengeHandler(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.has_any_role(
        STAFF_ROLE_ID, CHALLENGE_HOST_ROLE_ID, CHALLENGE_HOST_HELPER_ROLE_ID
    )
    @commands.group(name="challenges", aliases=("c",))
    async def challenges_group(self, ctx: commands.Context) -> None:
        """All of the Challenges commands"""
        if not ctx.invoked_subcommand:
            await ctx.send_help(ctx.command)

    @challenges_group.command(
        name="resubmit",
        aliases=("rs",),
        brief="Resubmit Command to remove submitted role",
    )
    async def challenges_resubmit(self, ctx: commands.Context, member: discord.Member):
        submitted_role = ctx.guild.get_role(CHALLENGE_SUBMITTED_ROLE_ID)

        if submitted_role in member.roles:
            await member.remove_roles(submitted_role)
            return await ctx.send(f"Submitted role removed from {member.mention}")

        return await ctx.send("Member doesn't have the submitted role")

    @commands.cooldown(1, 3600, commands.BucketType.user)
    @challenges_group.command(
        name="announce_winners",
        aliases=("aw", "announce"),
        brief="Command to announce the distribution of :pancakes:",
    )
    async def announce_winners(self, ctx: commands.Context):
        info_channel = ctx.guild.get_channel(CHALLENGE_INFO_CHANNEL_ID)

        return await info_channel.send(
            f"<@&{CHALLENGE_WINNER_ROLE_ID}> :pancakes: have been given out, "
            f"go deposit them in <#{BOT_GAMES_CHANNEL_ID}>. \n"
            f"Analysis for the challenge will be available shortly in <#{CHALLENGE_INFO_CHANNEL_ID}>",
            allowed_mentions=discord.AllowedMentions(
                roles=[discord.Object(CHALLENGE_WINNER_ROLE_ID)]
            ),
        )

    @commands.cooldown(1, 3600, commands.BucketType.user)
    @challenges_group.command(
        name="open_submissions",
        aliases=("os", "open"),
        brief="Command to open submissions",
    )
    async def open_submissions(self, ctx: commands.Context):
        info_channel = ctx.guild.get_channel(CHALLENGE_INFO_CHANNEL_ID)

        submit_channel = ctx.guild.get_channel(CHALLENGE_SUBMIT_CHANNEL_ID)

        # Allows people to submit
        overwrite = submit_channel.overwrites_for(ctx.guild.default_role)
        overwrite.send_messages = None
        await submit_channel.set_permissions(ctx.guild.default_role, overwrite=overwrite)

        return await info_channel.send(
            f"<@&{CHALLENGE_PARTICIPANT_ROLE_ID}> Submissions are open."
            f"Upload your code file with extension in <#{CHALLENGE_SUBMIT_CHANNEL_ID}>. "
            f"Send `t.tag submitting` in <#{BOT_COMMANDS_CHANNEL_ID}> for more details",
            allowed_mentions=discord.AllowedMentions(
                roles=[discord.Object(CHALLENGE_PARTICIPANT_ROLE_ID)]
            ),
        )

    @commands.cooldown(1, 3600, commands.BucketType.user)
    @challenges_group.command(
        name="close_submissions",
        aliases=("cs", "close"),
        brief="Command to close submissions",
    )
    async def close_submissions(self, ctx: commands.Context):
        info_channel = ctx.guild.get_channel(CHALLENGE_INFO_CHANNEL_ID)

        submit_channel = ctx.guild.get_channel(CHALLENGE_SUBMIT_CHANNEL_ID)

        # Disallows people to submit
        overwrite = submit_channel.overwrites_for(ctx.guild.default_role)
        overwrite.send_messages = False
        await submit_channel.set_permissions(ctx.guild.default_role, overwrite=overwrite)

        return await info_channel.send(
            f"<@&{CHALLENGE_SUBMITTED_ROLE_ID}> Submissions are closed. "
            "Testing will begin soon. See you in the next challenge",
            allowed_mentions=discord.AllowedMentions(
                roles=[discord.Object(CHALLENGE_SUBMITTED_ROLE_ID)]
            ),
        )

    @commands.Cog.listener()
    async def on_raw_reaction_add(self, payload):  # Participant role.
        if payload.emoji != discord.PartialEmoji(name="🖐"):
            return

        if payload.channel_id == 680851798340272141:
            submitted = self.bot.guild.get_role(687417501931536478)
            if submitted in payload.member.roles:
                return

            participant = self.bot.guild.get_role(687417513918857232)
            await self.bot.guild.get_member(payload.user_id).add_roles(participant)

        elif payload.channel_id == 713841395965624490:
            submitted = self.bot.guild.get_role(715676464573317220)
            if submitted in payload.member.roles:
                return

            participant = self.bot.guild.get_role(715676023387062363)
            await self.bot.guild.get_member(payload.user_id).add_roles(participant)

    @commands.Cog.listener()
    async def on_message(self, message):  # Submitted role.

        if message.channel.id in (680851820587122700, 713841306253656064):  # weekly

            if message.author.id == self.bot.user.id:
                return

            if message.author.bot:
                return await message.delete()

            if message.channel.id == 680851820587122700:  # weekly 1
                submitted = self.bot.guild.get_role(687417501931536478)
                submission_channel = self.bot.guild.get_channel(729453161885990924)

            else:  # weekly 2
                submitted = self.bot.guild.get_role(715676464573317220)
                submission_channel = self.bot.guild.get_channel(729453201081761862)

            if submitted not in message.author.roles:
                await message.delete()
                attach = message.attachments and message.attachments[0]

                if not attach:
                    msg = (
                        f"{message.author.mention} make sure to __upload a "
                        f"file__ that only includes the code required "
                        f"for the challenge!"
                    )
                    return await message.channel.send(msg, delete_after=10.0)

                filetype = attach.filename.split(".")[-1]

                if (
                    len(filetype) > 4 or len(filetype) == 0
                ):  # Most filetypes are 2-3 chars, 4 just to be safe
                    return await message.channel.send(
                        f"{message.author.mention} attachement file extension must be between 1 and 4 characters long",
                        delete_after=10.0,
                    )

                code = (await attach.read()).decode("u8")

                content = f"```{filetype}\n" + code.replace("`", "\u200b`") + "```"
                if len(content) > 4096:
                    # 4096 = max embed description size

                    msg = (
                        f"{message.author.mention} your submission can't be __more"
                        f" than {4096 - len(filetype) - 7} characters__."
                    )
                    return await message.channel.send(msg, delete_after=10.0)

                await message.author.add_roles(submitted)
                embed = discord.Embed(description=content, color=0x36393E)
                embed.set_author(
                    name=str(message.author), icon_url=message.author.avatar_url
                )
                embed.set_footer(
                    text=f"#ID: {message.author.id} • {len(code)} chars • Language: {filetype}"
                )
                await submission_channel.send(embed=embed)

        elif message.channel.id in [
            680851798340272141,
            713841395965624490,
        ]:  # Automatic reaction
            await message.add_reaction("🖐")


import argparse
import gym
import numpy as np
from itertools import count

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.distributions import Categorical
import matplotlib.pyplot as plt
# PATH = "/home/josephp/projects/def-dnowrouz/josephp/Pong/SURE/"
PATH = "C:\\Users\\Joseph\\PycharmProjects\\SURE\\"
parser = argparse.ArgumentParser(description='PyTorch REINFORCE example')
parser.add_argument('--gamma', type=float, default=0.99, metavar='G',
                    help='discount factor (default: 0.99)')
parser.add_argument('--seed', type=int, default=543, metavar='N',
                    help='random seed (default: 543)')
parser.add_argument('--render', action='store_true',
                    help='render the environment')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                    help='interval between training status logs (default: 10)')
args = parser.parse_args()
losses = []

env = gym.make('CartPole-v0')
env.seed(args.seed)
torch.manual_seed(args.seed)
TARGET_UPDATE = 100

class Policy(nn.Module):
    def __init__(self):
        super(Policy, self).__init__()
        self.affine1 = nn.Linear(4, 128)
        self.affine2 = nn.Linear(128, 2)

        self.saved_log_probs = []
        self.rewards = []
        self.states = []
        self.next_states = []
        self.dones = []

    def forward(self, x):
        x = x.float()
        x = F.relu(self.affine1(x))
        action_scores = self.affine2(x)
        return F.softmax(action_scores, dim=1)

policy = Policy()
target = Policy()
optimizer = optim.Adam(policy.parameters(), lr=1e-2)
eps = np.finfo(np.float32).eps.item()

def select_action(state):
    state = torch.from_numpy(state).float().unsqueeze(0)
    probs = policy(state)
    m = Categorical(probs)
    action = m.sample()
    policy.saved_log_probs.append(m.log_prob(action))
    return action.item()

def plotLoss():
    plot = plt.figure(num="Loss")
    plt.clf()
    losses_t = torch.tensor(losses, dtype=torch.float)
    plt.title('Loss...')
    plt.xlabel('Episode')
    plt.ylabel('Loss')
    plt.plot(losses_t.numpy())
    # plt.savefig("C:\\Users\\Joseph\\PycharmProjects\\SURE\\lossplt.png")
    plt.savefig(PATH + "cartpolelossplt.png")
    plt.close(plot)
def finish_episode():
    R = 0
    policy_loss = []
    rewards = []
    for r in policy.rewards[::-1]:
        R = r + args.gamma * R
        rewards.insert(0, R)
    rewards = torch.tensor(rewards)
    rewards = (rewards - rewards.mean()) / (rewards.std() + eps)
    for log_prob, reward in zip(policy.saved_log_probs, rewards):
        policy_loss.append(-log_prob * reward)
    optimizer.zero_grad()
    policy_loss = torch.cat(policy_loss).sum()
    losses.append(policy_loss)
    policy_loss.backward()
    optimizer.step()
    del policy.rewards[:]
    del policy.saved_log_probs[:]
def finish_episode_Q_func(gamma):
    states = torch.tensor(policy.states)
    next_states = torch.tensor(policy.next_states)
    rewards = torch.tensor(policy.rewards)
    state_actions = policy(states.unsqueeze(0)).detach().max(2)[0]
    next_actions = target(next_states.unsqueeze(0)).detach().max(2)[0]
    Q_target = rewards + gamma * next_actions * (1 - torch.tensor(policy.dones).float())

    loss = F.mse_loss(state_actions, Q_target)
    var_loss = torch.autograd.Variable(loss, requires_grad=True)
    optimizer.zero_grad()
    var_loss.backward()
    optimizer.step()
    del policy.rewards[:]
    del policy.states[:]
    del policy.next_states[:]
    del policy.dones[:]

def main():
    c = 0
    running_reward = 10
    for i_episode in count(1):
        state = env.reset()
        for t in range(10000):  # Don't infinite loop while learning
            action = select_action(state)
            next_state, reward, done, _ = env.step(action)
            if args.render:
                env.render()
            policy.rewards.append(reward)
            policy.states.append(state)
            policy.next_states.append(next_state)
            policy.dones.append(done)
            state = next_state
            if done:
                c += 1
                break
        if c % TARGET_UPDATE == 0:
            target.load_state_dict(policy.state_dict())
        running_reward = running_reward * 0.99 + t * 0.01
        finish_episode()
        if i_episode % args.log_interval == 0:
            print('Episode {}\tLast length: {:5d}\tAverage length: {:.2f}'.format(
                i_episode, t, running_reward))
        if running_reward > env.spec.reward_threshold:
            print("Solved! Running reward is now {} and "
                  "the last episode runs to {} time steps!".format(running_reward, t))
            break

if __name__ == '__main__':
    main()
    plotLoss()


from typing import List

from test_framework import generic_test

def is_pattern_contained_in_grid(grid: List[List[int]],
                                 pattern: List[int]) -> bool:
    m, n = len(grid), len(grid[0])
    M = [[0 for _ in range(n)] for _ in range(m)]

    for p in range(len(pattern)):
        for y in range(m):
            for x in range(n):
                if grid[y][x] == pattern[p]:
                    if p == 0:
                        M[y][x] = 1
                    else:
                        for dx, dy in [(-1, 0), (1, 0), (0, -1), (0, 1)]:
                            if 0 <= x + dx < n and 0 <= y + dy < m and \
                                    M[y + dy][x + dx] == p:
                                M[y][x] = p + 1
                                break
    for y in range(m):
        for x in range(n):
            if M[y][x] == len(pattern):
                return True
    return False

if __name__ == '__main__':
    exit(
        generic_test.generic_test_main('is_string_in_matrix.py',
                                       'is_string_in_matrix.tsv',
                                       is_pattern_contained_in_grid))


"""
将今日热榜的任务发送给爬虫
"""
import json
import random
import time
import datetime
import redis
import pgdb
from utils import (server_conn, job_pipeline_conn, CONTENT_TASKS_CHANNEL)

def get_content_jobs():
    now = datetime.datetime.now()
    six_minute_before = now - datetime.timedelta(minutes=9000)
    jobs_list = server_conn.query('''
          select
            sd.url,sj.content_regulation,sj.charset, sj.desc as job_desc
          from
            server_document as sd
          inner join
            server_job as sj
          on
            sd.job_id_id = sj.id
          where
            sd.createtime >= '%s'
          and
            sd.crawl_status = 'NO'
          and
            sj.content_regulation != ''
          and
            (sd.content is null or sd.content = '')
          and
            sj.desc in ('今日热榜')
          limit
            30;
        ''' % (six_minute_before, ))
    return jobs_list

def package_jobs_to_redis(jobs):
    job_list = []
    if jobs:
        for job in jobs:
            url = job.get('url', '')
            regulation = job.get('content_regulation', '')
            charset = job.get('charset', '')
            job_desc = job.get('job_desc', '')
            if url and regulation:
                job_list.append({
                    'url': url,
                    'regulation': regulation,
                    'charset': charset,
                    'job_desc': job_desc
                })
        job_list = json.dumps(job_list)
        job_pipeline_conn.publish(CONTENT_TASKS_CHANNEL, job_list)

if __name__ == '__main__':
    jobs = get_content_jobs()
    package_jobs_to_redis(jobs)
    # print(jobs)
    time.sleep(10)


import numpy as np
import os
import logging
from pprint import pformat

from settree.set_data import SetDataset, OPERATIONS, flatten_datasets
from exps.data import get_poker_dataset
import exps.eval_utils as eval

params = {
    'exp_name': 'single_new_ops',
    'seed': 0,
    'n_train': 25000,
    'n_test': 10000,
    'n_exp': 5
}

if __name__ == '__main__':

    log_dir = os.path.join(os.path.abspath('__file__' + '/../'), 'outputs', 'single')
    eval.create_logger(log_dir=log_dir,
                       log_name=params['exp_name'],
                       dump=True)

    logging.info(pformat(params))
    logging.info(pformat(params))

    mets = {'reg': [], 'reg_proc': [], 'set': []}
    for seed in range(params['n_exp']):
        np.random.seed(seed)
        logging.info('####### Exp {} #######'.format(seed))
        x_train, y_train, x_test, y_test = get_poker_dataset(pre_process_method='set_tree',
                                                             scenerio='bin',
                                                             n_test=params['n_test'],
                                                             seed=seed)
        # mode = 'bin_cls'
        mode = 'multi_cls'
        logging.info('There are {} train and {} test'.format(len(x_train), len(x_test)))

        ds_train = SetDataset(records=x_train, is_init=True)
        ds_test = SetDataset(records=x_test, is_init=True)

        shared_gbdt_params = {'n_estimators': 150,
                              'learning_rate': 0.1,
                              'max_depth': 6,
                              'max_features': None,
                              'subsample': 1,
                              'n_iter_no_change': 3,
                              'random_state': 42}

        logging.info('Shared params:\n' + pformat(shared_gbdt_params))

        set_params = {'n_estimators': shared_gbdt_params['n_estimators'],
                      'operations': OPERATIONS,
                      'splitter': 'sklearn',
                      'use_attention_set': True,
                      'attention_set_limit': 5,
                      'max_depth': shared_gbdt_params['max_depth'],
                      'max_features': shared_gbdt_params['max_features'],
                      'subsample': shared_gbdt_params['subsample'],
                      'random_state': shared_gbdt_params['random_state'],
                      # 'validation_fraction': 0.1,
                      # 'tol': 1e-3,
                      # 'n_iter_no_change': shared_gbdt_params['n_iter_no_change'],
                      'verbose': 3}

        xgboost_params = {  # 'objective': 'binary:logistic', # 'multi:softmax', binary:logistic
            'max_depth': shared_gbdt_params['max_depth'],
            'n_jobs': 10,
            'learning_rate': shared_gbdt_params['learning_rate'],
            'n_estimators': shared_gbdt_params['n_estimators'],
            'colsample_bytree': shared_gbdt_params['max_features'],
            'subsample': shared_gbdt_params['subsample'],
            'reg_lambda': 0,
            'reg_alpha': 0,
            'verbosity': 0,
            'random_state': shared_gbdt_params['random_state'],
            'seed': shared_gbdt_params['random_state']}

        x_train, x_test = flatten_datasets(ds_train, ds_test,
                                           operations_list=set_params['operations'],
                                           ds_val=None)

        ################################################################################################################
        xgboost_gbtd_proc, reg_proc_train_cc, reg_proc_test_acc = eval.train_and_predict_xgboost(xgboost_params,
                                                      x_train, y_train,
                                                      x_test, y_test,
                                                      val_x=None, val_y=None,
                                                      early_stopping_rounds=None,
                                                      out_metrics=True,
                                                      mode=mode)
        mets['reg_proc'].append(reg_proc_test_acc)
        ################################################################################################################

        xgboost_params = {  # 'objective': 'binary:logistic', # 'multi:softmax', binary:logistic
            'max_depth': shared_gbdt_params['max_depth'],
            'n_jobs': 10,
            'learning_rate': shared_gbdt_params['learning_rate'],
            'n_estimators': 300,
            'colsample_bytree': shared_gbdt_params['max_features'],
            'subsample': shared_gbdt_params['subsample'],
            'reg_lambda': 0,
            'reg_alpha': 0,
            'verbosity': 0,
            'random_state': shared_gbdt_params['random_state'],
            'seed': shared_gbdt_params['random_state']}
        xgboost_gbtd, reg_train_cc, reg_test_acc = eval.train_and_predict_xgboost(xgboost_params,
                                                      np.stack([r.flatten() for r in ds_train.records]), y_train,
                                                      np.stack([r.flatten() for r in ds_test.records]), y_test,
                                                      val_x=None, val_y=None,
                                                      early_stopping_rounds=None,
                                                      out_metrics=True,
                                                      mode=mode)
        mets['reg'].append(reg_test_acc)

        ################################################################################################################
        set_gbtd, set_train_cc, set_test_acc = eval.train_and_predict_set_gbdt(set_params,
                                                                               ds_train, y_train,
                                                                               ds_test, y_test,
                                                                               out_metrics=True,
                                                                               mode=mode)
        mets['set'].append(set_test_acc)
        if seed == 0:
            eval.save_pickle(set_gbtd, os.path.join(log_dir, '{}_set_model.pickle'.format(params['exp_name'])))

        ################################################################################################################
        logging.info('Seed: {} reg_proc acc: {:.5f} | reg acc: {:.5f} | set acc {:.5f}'.format(seed,
                                                                                reg_proc_test_acc,
                                                                                reg_test_acc,
                                                                                set_test_acc))
    ################################################################################################################
    # End of train
    ################################################################################################################
    reg_proc_res = np.array(mets['reg_proc'])
    logging.info('Reg-Proc: mean: {:.4f} | std: {:.4f}'.format(reg_proc_res.mean(), reg_proc_res.std()))
    reg_res = np.array(mets['reg'])
    logging.info('Reg: mean: {:.4f} | std: {:.4f}'.format(reg_res.mean(), reg_res.std()))
    set_res = np.array(mets['set'])
    logging.info('Set: mean: {:.4f} | std: {:.4f}'.format(set_res.mean(), set_res.std()))

    # eval.save_pickle(xgboost_gbtd, os.path.join(os.path.join(log_dir, params['exp_name'] + '_xbg_model.pkl')))
    # eval.save_pickle(set_gbtd, os.path.join(os.path.join(log_dir, params['exp_name'] + '_set_model.pkl')))

    ####################################################################################################################
    # ablation: train tree with permutations data
    ####################################################################################################################
    # x_test_numpy, y_test_numpy = pre_process_numpy(test[:10000])
    # x_perm_train, y_perm_train = pre_process_all_permutations(train)
    # logging.info('Flatten labels')
    # y_test_numpy[y_test_numpy != 0] = 1
    # y_perm_train[y_perm_train != 0] = 1
    # mode = 'bin_cls'
    #
    # x_perm_train, x_perm_val, y_perm_train, y_perm_val = train_test_split(x_perm_train, y_perm_train,
    #                                                                       test_size=0.15,
    #                                                                       random_state=params['seed'])
    #
    # xgboost_params = {'tree_method': 'gpu_hist',
    #                   'max_depth': 12,
    #                   'n_jobs': 10,
    #                   'learning_rate': 0.05,
    #                   'n_estimators': 500,
    #                   'colsample_bytree': None,
    #                   'subsample': 0.5,
    #                   'reg_lambda': 0,
    #                   'reg_alpha': 0,
    #                   'verbosity': 3}
    #
    # xgboost_gbtd = eval.train_and_predict_xgboost(xgboost_params,
    #                                               x_perm_train, y_perm_train,
    #                                               x_test_numpy, y_test_numpy,
    #                                               val_x=x_perm_val, val_y=y_perm_val,
    #                                               early_stopping_rounds=10,
    #                                               mode=mode)
    #


#################################################################
#
#   main.py
#   main file for the demonstration of the ART algorithm
#   written by: Walter Simson
#               Chair for Computer Aided Medical Procedures
#               & Augmented Reality
#               Technical University of Munich
#               27.10.2017
#   based on the work of Maximilian Baust
#
#################################################################

import warnings

import matplotlib.pyplot as plt
import numpy as np

from art import art
from helper import load_data

# Clean up
plt.close('all')

# Load system of equations
# A is the system matrix, b is the right hand side,
# x is the true solution, i.e. Ax=b
A, b, x = load_data("system.mat")

# Set number of iterations
iterations = 50

# Initialize plots
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2, figsize=(6, 6))
plt.tight_layout()
ax1.set_title('np.linalg.solve')
ax2.set_title('np.linalg.qr')
ax3.set_title('ART')
ax4.set_title('True Solution')

# Plot true solution
x = x.reshape((60, 60))
ax4.imshow(x, extent=[0, 1, 0, 1], cmap='bone')

# Solve LSE with numpy solver
x_np = np.linalg.solve(A, b)
x_np = x_np.reshape((60, 60))
# Warn like MATLAB
x_cond = np.linalg.cond(A)
warnings.warn("Warning: Matrix is close to singular or badly scaled." +
              " Results may be inaccurate. Condition = {0}.".format(x_cond))

ax1.imshow(x_np, extent=[0, 1, 0, 1], cmap='bone')

# Solve LSE with QR-Algorithm
q, r = np.linalg.qr(A)
y = np.matmul(q.T, b)
x_qr = np.linalg.solve(r, y)
x_qr = x_qr.reshape((60, 60))
ax2.imshow(x_qr, extent=[0, 1, 0, 1], cmap='bone')

# Solve LSE with ART
x_art = art(A, b, iterations)
x_art = x_art.reshape((60, 60))
ax3.imshow(x_art, extent=[0, 1, 0, 1], cmap='bone')

plt.show()


import codecs
from collections import Counter, OrderedDict
from json import dumps

# Need unit tests for all of these methods
# Might need to clean up unused / artifact methods

# Put this in a constants file
IGNORE_CHARACTERS = ['.', '!', '?', ',', ';', ':', '-', '(', ')', '{', '}', '[', ']', '<', '>', '"', '\'', '\n']

def launch_word_analysis(file_path, common_word_list, analyzed_path, count=50):
    word_map = get_word_map(file_path)
    top_words = get_top_words(word_map, common_word_list, count=count)
    analyzed_dir = open(analyzed_path, 'wb')
    json = dumps(top_words, sort_keys=False)
    print(json)
    # analyzed_dir.write(json)
    # analyzed_dir.flush()
    # analyzed_dir.close()

def get_word_map(file_path):
    word_map = {}
    with codecs.open(file_path, mode='r', encoding='utf-8') as file:
        for line in file:
            word_list = prepare_word_list(line)
            for word in word_list:
                if word in word_map:
                    word_map[word] = word_map[word] + 1
                else:
                    word_map[word] = 1
    return word_map

def prepare_word_list(line):
    line = remove_ignore_characters(line)
    line = line.lower()
    word_list = line.split()
    return word_list

def remove_ignore_characters(line):
    for char in IGNORE_CHARACTERS:
        line = line.replace(char, '')
    return line

def get_top_words(word_map, common_word_list=None, count=10):
    # We go from dictionary, to Counter, to OrderedDict - probably not efficient
    if common_word_list:
        word_map = remove_common_words(word_map, common_word_list)
    top_word_map = Counter(word_map)
    most_common_words = top_word_map.most_common(count)
    return OrderedDict(most_common_words)

def remove_common_words(word_map, common_word_list):
    new_word_map = {}
    for word in word_map:
        if word not in common_word_list:
            new_word_map[word] = word_map[word]
    return new_word_map

def print_results(top_words):
    rank = 1
    for word_tuple in top_words:
        print('%d: %s - %d' % (rank, word_tuple[0], word_tuple[1]))
        rank += 1

def generate_common_words_file(sample_file_path, common_words_file_path):
    common_word_list = generate_common_word_list(sample_file_path)
    with codecs.open(common_words_file_path, mode='w', encoding='utf-8') as common_words_file:
        for word in common_word_list:
            common_words_file.write(word + '\n')

def generate_common_word_list(sample_file_path):
    common_word_map = get_word_map(sample_file_path)
    top_fifty = get_top_words(common_word_map, 50)
    common_word_list = []
    for word_tuple in top_fifty:
        common_word_list.append(word_tuple[0])
    return common_word_list


import json
import datetime
from datetime import timedelta
from airflow.operators.bash_operator import BashOperator

from airflow import DAG
from airflow.operators.http_operator import SimpleHttpOperator
from airflow.operators.sensors import HttpSensor
from airflow.utils.dates import days_ago
from airflow.operators.python_operator import PythonOperator
from airflow.models import Variable

from airflow.hooks.base_hook import BaseHook
from airflow.contrib.operators.slack_webhook_operator import SlackWebhookOperator
from airflow import settings
from airflow.models import Connection

BASE_URL='http://192.168.1.5:8181/employee'
default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'email': ['avinash.kachhwaha@oracle.com'],
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=1),
    'provide_context': True

}

dag = DAG(
    'http_operator_rest_curl',
    default_args=default_args,
    tags=['example'],
    start_date=datetime.datetime.now() - datetime.timedelta(days=1)
    # schedule_interval='*/1 * * * *',
)

dag.doc_md = __doc__

health_check_task = BashOperator(
    task_id='status-check',
    bash_command='curl -X GET {BASE_URL}/'.format(BASE_URL = BASE_URL),
    dag=dag
)

get_all_employee_task = BashOperator(
    task_id='get-all-employee',
    bash_command='curl -X GET {BASE_URL}/get-all'.format(BASE_URL = BASE_URL),
    xcom_push=True,
    dag=dag
)

get_employee_by_id_task = BashOperator(
    task_id='get-employee-by-id',
    bash_command='curl -X GET {BASE_URL}/get-by-id/3/employee'.format(BASE_URL = BASE_URL),
    xcom_push=True,
    dag=dag
)

jsondata='{"name" : "Vikram3"}'
save_employee_task = BashOperator(
    task_id='save-employee',
    bash_command='curl -X POST  --data \'{jsondata}\' -H "Content-Type: application/json"  {BASE_URL}/save'.format(jsondata=jsondata, BASE_URL = BASE_URL),
    xcom_push=True,
    provide_context=True,
    dag=dag
)

printing_data = BashOperator(
    task_id='t2',
    bash_command='echo "{{ json.loads(ti.xcom_pull("save-employee")) }}"',
    dag=dag
)

health_check_task >> save_employee_task >> printing_data >> get_all_employee_task
health_check_task >> save_employee_task >> get_employee_by_id_task


"""Tests for pytype.pytd.parse.builtins."""

from pytype.pytd import pytd
from pytype.pytd import visitors
from pytype.pytd.parse import builtins
import unittest

class UtilsTest(unittest.TestCase):

  PYTHON_VERSION = (2, 7)

  @classmethod
  def setUpClass(cls):
    cls.builtins = builtins.GetBuiltinsPyTD(cls.PYTHON_VERSION)

  def testGetBuiltinsPyTD(self):
    self.assertIsNotNone(self.builtins)
    # Will throw an error for unresolved identifiers:
    self.builtins.Visit(visitors.VerifyLookup())

  def testHasMutableParameters(self):
    append = self.builtins.Lookup("__builtin__.list").Lookup("append")
    self.assertIsNotNone(append.signatures[0].params[0].mutated_type)

  def testHasCorrectSelf(self):
    update = self.builtins.Lookup("__builtin__.dict").Lookup("update")
    t = update.signatures[0].params[0].type
    self.assertIsInstance(t, pytd.GenericType)
    self.assertEqual(t.base_type, pytd.ClassType("__builtin__.dict"))

  def testHasObjectSuperClass(self):
    cls = self.builtins.Lookup("__builtin__.memoryview")
    self.assertEqual(cls.parents, (pytd.ClassType("__builtin__.object"),))
    cls = self.builtins.Lookup("__builtin__.object")
    self.assertEqual(cls.parents, ())

if __name__ == "__main__":
  unittest.main()


from __future__ import print_function
from flask import Flask
from flask import render_template
from flask import request
from flask import abort, redirect, url_for
from werkzeug.utils import secure_filename
from flask import send_from_directory
import os
from fraudModel.fraud import *
import sys
import copy
import time

UPLOAD_FOLDER = os.getcwd()+'/uploads'
ALLOWED_EXTENSIONS = set(['csv'])

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/', methods=['POST', 'GET'])
def upload_file():
    if request.method == 'POST':
        # check if the post request has the file part
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        file = request.files['file']
        modelType = request.form['select']
        #print(modelType,file=sys.stderr)
        # if user does not select file, browser also
        # submit a empty part without filename
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            #print(filename,file=sys.stderr)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            Use_Model(Clustering=True, baseDir = os.getcwd(), modelType = modelType, filename = filename)
            if modelType == 'RF':
                time.sleep(6.5)
            elif modelType == 'LRsgd':
                time.sleep(4.5)
            elif modelType == 'LRlbfgs':
                time.sleep(5)
            elif modelType == 'GBDT':
                time.sleep(5.8)
            elif modelType == 'SMV':
                time.sleep(5.0)
            else:
                pass
            return redirect(url_for('uploaded_file',
                                    filename=filename+'_'+modelType +'_predicted.csv'))
    return render_template('hello.html')

@app.route('/downloads/<filename>')
def uploaded_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'],
                               filename)

if __name__ == '__main__':
  app.run()


import time
import pyrix as px
import copy
import matplotlib.pyplot as pt
import numpy as np
from numpy.linalg import inv
benchdata = []
inputsize = list(range(1000))

def timerfunc(func):
    """
    A timer decorator
    """
    def function_timer(*args, **kwargs):
        """
        A nested function for timing other functions
        """
        start = time.time()
        value = func(*args, **kwargs)
        end = time.time()
        runtime = end - start
        benchdata.append(runtime)
        return value
    return function_timer

@timerfunc
def pyrixbased(scale):
    mat = px.identityMatrix(nrow=scale, ncol=scale)
    mat.invertMatrix()

for i in inputsize:
    pyrixbased(i+1)
pyrixdata = copy.deepcopy(benchdata)
benchdata = []

@timerfunc
def numpybased(scale):
    mat = np.identity(scale, dtype=int)
    inv(mat)

for i in inputsize:
    numpybased(i+1)
numpydata = copy.deepcopy(benchdata)

ax = pt.subplot()
ax.plot(inputsize, pyrixdata,label="Pyrix(ST+IDT)")
ax.plot(inputsize, numpydata,label="Numpy(MT+Cache)")
pt.xlabel('Size of Matrix(nxn)')
# naming the y axis
pt.ylabel('Time taken in seconds(Lower is Better)')
# giving a title to my graph
pt.title('Inversion of Identity Matrix Benchmark\nIntel J2900@2.41Ghz/Linux mint @ CPython3.6.9')
ax.legend()
pt.show()


"""dentymz URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.conf import settings
from django.conf.urls.static import static
from api import views
from chat import views as chat_view
from app import views as app_views

urlpatterns = [
    path('admin/', admin.site.urls),
    path('api/auth/login/', views.login),
    path('api/collect-schools/', views.collect_schools),
    path('api/collect-year-groups/', views.collect_year_groups),
    path('api/account/sign-up/', views.sign_up),
    path('api/account/complete-sign-up/', views.complete_sign_up),
    path('api/account/upload-photo/', views.upload_profile_photo),
    path('api/account/change-password/', views.change_password),
    path('api/post/load/', views.load_posts, name='load-posts'),
    path('api/post/text/', views.post_text),
    path('api/post/photo/', views.post_photo),
    path('api/post/load-likes/', views.load_likes),
    path('api/post/load-comments/', views.load_comments),
    path('api/post/like/', views.like_post, name='like-post'),
    path('api/post/comment/', views.comment_on_post),
    path('api/post/delete/', views.delete_post),
    path('api/post/report/', views.report_post),
    path('api/comment/edit/', views.edit_comment),
    path('api/comment/delete/', views.delete_comment),
    path('api/school/follow/', views.follow_school),
    path('api/school/get/', views.get_school),
    path('api/school/collect-students/', views.collect_school_students),
    path('api/school/report-missing/', views.report_missing_school),
    path('api/user/load-data/', views.load_user_data),
    path('api/user/set-status/', views.set_status),
    path('api/user/set-online-status/', views.set_online_status),
    path('api/user/update-fcm-token/', views.update_user_fcm_token),
    path('api/user/update-app-version/', views.update_user_app_version),
    path('api/user/collect-following/', views.collect_following),
    path('chat/', chat_view.index),
    path('chat/room/', chat_view.room, name='room'),
    path('api/chat/load/', views.load_chats),
    path('api/chat/load-messages/', views.load_messages),
    path('api/chat/send-message/', views.send_message),
    path('api/chat/request/', views.chat_request),
    path('api/chat/load-chat-requests/', views.load_chat_requests),
    path('api/chat/load-blocked-users/', views.load_blocked_users),
    path('api/support/report-problem/', views.report_problem),
    path('api/contact/send-feedback/', views.send_feedback),
    path('', app_views.index, name='index'),
    path('login/', app_views.LoginView.as_view(), name="login"),
    path('signup/', app_views.RegisterView.as_view(), name="signup"),
    path('complete-signup/', app_views.CompleteRegistrationView.as_view(), name="complete-signup"),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from pyspark.sql import SparkSession
from src.parser.csv_parser import load_as_df
from src.schema.twitter_schema import twitter_schema
from src.preprocess.preprocessor import shape_df
from src.featurizer.bert_featurizer import BertFeaturizer
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

if __name__ == '__main__':

    spark = SparkSession.builder \
        .appName('Spark SQL and DataFrame') \
        .getOrCreate()
    # Load training data
    dataPath = '../../data/twitter/20190528sentences_data_integrated.csv'
    # dataPath = '../../data/twitter_2020-03-10.csv'
    df = load_as_df(dataPath, twitter_schema)
    converted_df = shape_df(spark, df).drop("age")
    converted_df.show(3)
    # model_path = "../../param/word2vec/entity_vector/entity_vector.model.bin"
    # wv = Word2VecFeaturizer(spark, model_path)
    # feat_df = wv.featurize(converted_df)
    model_path = "../../param/bert/Japanese_L-24_H-1024_A-16_E-30_BPE_WWM_transformers"
    bert = BertFeaturizer(spark, model_path)
    feat_df = bert.featurize(converted_df)
    # Split the data into training and test sets (30% held out for testing)
    (trainingData, testData) = feat_df.randomSplit([0.7, 0.3], seed=2)

    dt = DecisionTreeClassifier(labelCol="label", featuresCol="features", maxDepth=6)

    # Train model.  This also runs the indexers.
    model = dt.fit(trainingData)

    # Make predictions.
    predictions = model.transform(testData)

    # Select example rows to display.
    predictions.select("prediction", "label", "features").show(5)

    # Select (prediction, true label) and compute test error
    evaluator = MulticlassClassificationEvaluator(
        labelCol="label", predictionCol="prediction", metricName="accuracy")
    accuracy = evaluator.evaluate(predictions)
    print("test accuracy: " + str(accuracy))