# -- coding: utf-8 --
# @File : test_wechat.py
import time

from appium import webdriver

class TestWeChart:
    def setup(self):
        condition = {
            "platformName": "android",
            "deviceName": "111", #一定不能少这个参数
            "udid": "127.0.0.1:7555", #如果只有一个设备可以不写，多个设备就要写进行指定
            "appPackage": "com.tencent.wework",
            "appActivity": ".launch.WwMainActivity",
            "settings[waitForIdleTimeout]": 0, #把动态网页，当作静态处理，减少元素定位时间，一定是[]不能是()
            "noReset": "true",  # 记录之前的操作或登录信息，不清除缓存
            "skipDeviceInitialization": "true"  # 跳过设备初始化
        }
        self.driver = webdriver.Remote("http://127.0.0.1:4723/wd/hub", condition)  # ip port num不要忘记！！
        self.driver.implicitly_wait(5)

    def test_wechat(self):
        self.driver.find_element_by_xpath("//*[@text='工作台']").click()
        #这里打卡页面，打卡元素在下面，无法直接定位，只能通过滚动定位
        self.driver.find_element_by_android_uiautomator('new UiScrollable(new UiSelector().'
                                                        'scrollable(true).instance(0)).'
                                                        'scrollIntoView(new UiSelector().'
                                                        'text("打卡").'
                                                        'instance(0));').click()
        self.driver.find_element_by_id('com.tencent.wework:id/i1x').click()
        #文本属性包含外出的进行定位
        time.sleep(2)
        self.driver.find_element_by_android_uiautomator('new UiSelector().textContains("次外出")').click() #注意Contains是大写
        #也可以这么写
        # self.driver.find_element_by_xpath("//*[contains(@text,'次外出')]")
        ele = self.driver.find_element_by_id('com.tencent.wework:id/pu').text
        assert '外出打卡成功' == ele


from django.conf import settings
from django.contrib import messages
from django.core.exceptions import PermissionDenied, SuspiciousOperation
from django.db.models.query import QuerySet
from django.http import FileResponse, HttpResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.utils.translation import gettext as _
from django.views.decorators.http import require_GET, require_POST
from django.views.generic import DetailView, TemplateView

from evap.evaluation.auth import (
    grade_downloader_required,
    grade_publisher_or_manager_required,
    grade_publisher_required,
)
from evap.evaluation.models import Course, EmailTemplate, Evaluation, Semester
from evap.evaluation.tools import get_object_from_dict_pk_entry_or_logged_40x, ilen
from evap.grades.forms import GradeDocumentForm
from evap.grades.models import GradeDocument

@grade_publisher_required
class IndexView(TemplateView):
    template_name = "grades_index.html"

    def get_context_data(self, **kwargs):
        return super().get_context_data(**kwargs) | {
            "semesters": Semester.objects.filter(grade_documents_are_deleted=False),
            "disable_breadcrumb_grades": True,
        }

def course_grade_document_count_tuples(courses: QuerySet[Course]) -> list[tuple[Course, int, int]]:
    courses = courses.prefetch_related("degrees", "responsibles", "evaluations", "grade_documents")

    return [
        (
            course,
            ilen(gd for gd in course.grade_documents.all() if gd.type == GradeDocument.Type.MIDTERM_GRADES),
            ilen(gd for gd in course.grade_documents.all() if gd.type == GradeDocument.Type.FINAL_GRADES),
        )
        for course in courses
    ]

@grade_publisher_required
class SemesterView(DetailView):
    template_name = "grades_semester_view.html"
    model = Semester
    pk_url_kwarg = "semester_id"

    object: Semester

    def get_object(self, *args, **kwargs):
        semester = super().get_object(*args, **kwargs)
        if semester.grade_documents_are_deleted:
            raise PermissionDenied
        return semester

    def get_context_data(self, **kwargs):
        courses = (
            self.object.courses.filter(evaluations__wait_for_grade_upload_before_publishing=True)
            .exclude(evaluations__state=Evaluation.State.NEW)
            .distinct()
        )
        courses = course_grade_document_count_tuples(courses)

        return super().get_context_data(**kwargs) | {
            "courses": courses,
            "disable_breadcrumb_semester": True,
        }

@grade_publisher_or_manager_required
class CourseView(DetailView):
    template_name = "grades_course_view.html"
    model = Course
    pk_url_kwarg = "course_id"

    def get_object(self, *args, **kwargs):
        course = super().get_object(*args, **kwargs)
        if course.semester.grade_documents_are_deleted:
            raise PermissionDenied
        return course

    def get_context_data(self, **kwargs):
        return super().get_context_data(**kwargs) | {
            "semester": self.object.semester,
            "grade_documents": self.object.grade_documents.all(),
            "disable_breadcrumb_course": True,
        }

def on_grading_process_finished(course):
    evaluations = course.evaluations.all()
    if all(evaluation.state == Evaluation.State.REVIEWED for evaluation in evaluations):
        for evaluation in evaluations:
            assert evaluation.grading_process_is_finished
        for evaluation in evaluations:
            evaluation.publish()
            evaluation.save()

        EmailTemplate.send_participant_publish_notifications(evaluations)
        EmailTemplate.send_contributor_publish_notifications(evaluations)

@grade_publisher_required
def upload_grades(request, course_id):
    course = get_object_or_404(Course, id=course_id)
    semester = course.semester
    if semester.grade_documents_are_deleted:
        raise PermissionDenied

    final_grades = request.GET.get("final") == "true"  # if parameter is not given, assume midterm grades

    grade_document = GradeDocument(course=course)
    if final_grades:
        grade_document.type = GradeDocument.Type.FINAL_GRADES
        grade_document.description_en = settings.DEFAULT_FINAL_GRADES_DESCRIPTION_EN
        grade_document.description_de = settings.DEFAULT_FINAL_GRADES_DESCRIPTION_DE
    else:
        grade_document.type = GradeDocument.Type.MIDTERM_GRADES
        grade_document.description_en = settings.DEFAULT_MIDTERM_GRADES_DESCRIPTION_EN
        grade_document.description_de = settings.DEFAULT_MIDTERM_GRADES_DESCRIPTION_DE

    form = GradeDocumentForm(request.POST or None, request.FILES or None, instance=grade_document)

    if form.is_valid():
        form.save(modifying_user=request.user)

        if final_grades:
            on_grading_process_finished(course)

        messages.success(request, _("Successfully uploaded grades."))
        return redirect("grades:course_view", course.id)

    template_data = {
        "semester": semester,
        "course": course,
        "form": form,
        "final_grades": final_grades,
        "show_automated_publishing_info": final_grades,
    }
    return render(request, "grades_upload_form.html", template_data)

@require_POST
@grade_publisher_required
def set_no_grades(request):
    course = get_object_from_dict_pk_entry_or_logged_40x(Course, request.POST, "course_id")

    try:
        status = bool(int(request.POST["status"]))
    except (KeyError, TypeError, ValueError) as e:
        raise SuspiciousOperation from e

    if course.semester.grade_documents_are_deleted:
        raise PermissionDenied

    course.gets_no_grade_documents = status
    course.save()

    if course.gets_no_grade_documents:
        on_grading_process_finished(course)

    return HttpResponse()  # 200 OK

@require_GET
@grade_downloader_required
def download_grades(request, grade_document_id):
    grade_document = get_object_or_404(GradeDocument, id=grade_document_id)
    if grade_document.course.semester.grade_documents_are_deleted:
        raise PermissionDenied

    return FileResponse(grade_document.file.open(), filename=grade_document.filename(), as_attachment=True)

@grade_publisher_required
def edit_grades(request, grade_document_id):
    grade_document = get_object_or_404(GradeDocument, id=grade_document_id)
    course = grade_document.course
    semester = course.semester
    if semester.grade_documents_are_deleted:
        raise PermissionDenied

    form = GradeDocumentForm(request.POST or None, request.FILES or None, instance=grade_document)

    final_grades = (
        grade_document.type == GradeDocument.Type.FINAL_GRADES
    )  # if parameter is not given, assume midterm grades

    if form.is_valid():
        form.save(modifying_user=request.user)
        messages.success(request, _("Successfully updated grades."))
        return redirect("grades:course_view", course.id)

    template_data = {
        "semester": semester,
        "course": course,
        "form": form,
        "show_automated_publishing_info": False,
        "final_grades": final_grades,
    }
    return render(request, "grades_upload_form.html", template_data)

@require_POST
@grade_publisher_required
def delete_grades(request):
    grade_document = get_object_from_dict_pk_entry_or_logged_40x(GradeDocument, request.POST, "grade_document_id")
    grade_document.delete()
    return HttpResponse()  # 200 OK


# M4 - Autonomous fruit searching

# basic python packages
import sys, os
import cv2
import numpy as np
import json
import ast
import argparse
import time
from Astar import AStarPlanner
sys.path.insert(0,"{}/network/".format(os.getcwd()))
sys.path.insert(0,"{}/network/scripts".format(os.getcwd()))
from network.scripts.detector import Detector
import TargetPoseEst as TPE

# import SLAM components
sys.path.insert(0, "{}/slam".format(os.getcwd()))
from slam.ekf1 import EKF
from slam.robot import Robot
import slam.aruco_detector as aruco

# import utility functions
#sys.path.insert(0, "util")
from util.pibot import Alphabot
import util.measure as measure

def read_true_map(fname):
    """Read the ground truth map and output the pose of the ArUco markers and 3 types of target fruit to search

    @param fname: filename of the map
    @return:
        1) list of target fruits, e.g. ['redapple', 'greenapple', 'orange']
        2) locations of the target fruits, [[x1, y1], ..... [xn, yn]]
        3) locations of ArUco markers in order, i.e. pos[9, :] = position of the aruco10_0 marker
    """
    with open(fname, 'r') as f:
        try:
            gt_dict = json.load(f)
        except ValueError as e:
            with open(fname, 'r') as f:
                gt_dict = ast.literal_eval(f.readline())
        fruit_list = []
        fruit_true_pos = []
        aruco_true_pos = np.empty([10, 2])

        # remove unique id of targets of the same type
        for key in gt_dict:
            x = np.round(gt_dict[key]['x'], 1)
            y = np.round(gt_dict[key]['y'], 1)

            if key.startswith('aruco'):
                if key.startswith('aruco10'):
                    aruco_true_pos[9][0] = x
                    aruco_true_pos[9][1] = y
                else:
                    marker_id = int(key[5])
                    aruco_true_pos[marker_id-1][0] = x
                    aruco_true_pos[marker_id-1][1] = y
            else:
                fruit_list.append(key[:-2])
                if len(fruit_true_pos) == 0:
                    fruit_true_pos = np.array([[x, y]])
                else:
                    fruit_true_pos = np.append(fruit_true_pos, [[x, y]], axis=0)

        return fruit_list, fruit_true_pos, aruco_true_pos

def read_search_list():
    """Read the search order of the target fruits

    @return: search order of the target fruits
    """
    search_list = []
    with open('search_list.txt', 'r') as fd:
        fruits = fd.readlines()

        for fruit in fruits:
            search_list.append(fruit.strip())

    return search_list

def print_target_fruits_pos(search_list, fruit_list, fruit_true_pos):
    """Print out the target fruits' pos in the search order

    @param search_list: search order of the fruits
    @param fruit_list: list of target fruits
    @param fruit_true_pos: positions of the target fruits
    """

    print("Search order:")
    n_fruit = 1
    for fruit in search_list:
        for i in range(3):
            if fruit == fruit_list[i]:
                print('{}) {} at [{}, {}]'.format(n_fruit,
                                                  fruit,
                                                  np.round(fruit_true_pos[i][0], 1),
                                                  np.round(fruit_true_pos[i][1], 1)))
        n_fruit += 1

def turn_to_point(delta_rot, pose_angle, turn_vel, scale, baseline):
    # find optimal turn direction
    dir = delta_rot - pose_angle*180/np.pi

    #print("dir: ",dir)

    if dir > 0 :
        turn_direc = [-turn_vel, turn_vel]

    elif dir < 0:
        turn_direc = [turn_vel, -turn_vel]
        # offset for right turn beecause it is shorter than left turn
        ppi.set_velocity([0,1],turn_direc,time=0.15)
    else:
        turn_direc = [0, 0]

    # calculate time it takes for a full 360 spin
    full_turn_time = (baseline * np.pi) / (scale * turn_vel)

    # est turn_time from baseline (requires good calibration)
    turn_time = full_turn_time * (abs(dir)/360)
    if dir == 0:
        turn_time = 0

    return turn_direc, turn_time

# Waypoint navigation
# the robot automatically drives to a given [x,y] coordinate
def drive_to_point(scale, baseline, waypoint, robot_pose):
    move_1, move_2 = [0,0,0], [0,0,0]
    wheel_vel = [23.6, 25] # tick to move the robot
    turn_vel = 10

    # get dist to waypoint
    delta_x = (waypoint[0][0] - robot_pose[0][0])
    delta_y = (waypoint[0][1] - robot_pose[0][1])
    delta_dist = np.sqrt( delta_x**2  +  delta_y**2 )

    # get angle to waypoint
    delta_rot = np.arctan2(delta_y, delta_x) * 180 / np.pi
    #print("delta rot: ",delta_rot)
    if delta_x == 0 and delta_y == 0:
        delta_rot = 360

    # limit delta_rot between 0 and 360 deg
    delta_rot = delta_rot % 360
    if delta_rot == 360:
        delta_rot = 0

    # get drive time (forward)
    drive_time = delta_dist / (scale * wheel_vel[1])

    # get turn direction and time to face waypoint direction
    turn_direc1, turn_time1 = turn_to_point(delta_rot, robot_pose[0][2], turn_vel, scale, baseline)

    # turn towards the waypoint
    if turn_time1 > 0.01:
        print("Turning for {} seconds speed {}".format(turn_time1, turn_direc1))
        move_1 = ppi.set_velocity([0, 1], tick=turn_direc1, time=turn_time1)
    img_1 = ppi.get_image()

    DL, DR = ppi.get_IR() # stop robot from driving forward if detect object after turning
    if DL == 0 or DR == 0:
        drive_time = 0

    if drive_time > 0.01:
        # after turning, drive straight to the waypoint
        print("Driving for {} seconds speed {}".format(drive_time, wheel_vel))
        move_2 = ppi.set_velocity([1, 0], tick=wheel_vel, time=drive_time)
    img_2 = ppi.get_image()

    ppi.set_velocity([0,0],tick=[0,0])

    return move_1, move_2, img_1, img_2

def get_robot_pose(move_1, move_2, img_1, img_2):

    # update the robot pose [x,y,theta]
    robot_pose = [[0.0,0.0,0.0]] # replace with your calculation

    if len(move_1) > 0:
        lv, rv, moving_time = move_1
        drive_meas = measure.Drive(lv, rv, moving_time)
        lms, aruco_img = aruco_det.detect_marker_positions(img_1)
        ekf.predict(drive_meas)
        ekf.update(lms)

    if len(move_2) > 0:
        lv, rv, moving_time = move_2
        drive_meas = measure.Drive(lv, rv, moving_time)

        lms, aruco_img = aruco_det.detect_marker_positions(img_2)
        ekf.predict(drive_meas)
        ekf.update(lms)

    robot_pose = ekf.robot.state

    return robot_pose.T

def check_IR(robot_pose):
    # check IR sensor for obstacle nearby
    DL, DR = ppi.get_IR()
    triggered = 0
    if DL == 0 or DR == 0:
        print("IR detected object")
        move_1 = ppi.set_velocity([0,-1],[14.7,15],time=1)
        img_1 = ppi.get_image()
        robot_pose = get_robot_pose(move_1, [0,0,0], img_1, img_1)
        triggered = 1
    # spin, check and update position of robot
    return robot_pose, triggered

def calib_Rob_Pose():
    # to define turn duration for calibration
    full_turn_time = (baseline * np.pi) / (scale * 10)
    turn_time = full_turn_time  * 30/360
    img_1 = ppi.get_image()

    #print("Checking Pose")

    # keep spinning to check position until robot detects at least 2 markers
    lms, aruco_img = aruco_det.detect_marker_positions(img_1)
    while len(lms) < 2:
        move_1 = ppi.set_velocity([0,1],tick=[10,-10],time=turn_time)
        img_1 = ppi.get_image()
        lv, rv, moving_time = move_1
        drive_meas = measure.Drive(lv, rv, moving_time)
        lms, aruco_img = aruco_det.detect_marker_positions(img_1)
        ekf.predict(drive_meas)
        ekf.update(lms)
        time.sleep(1)
        robot_pose = ekf.robot.state.T

    print("Calibrated robot pose: ", robot_pose, waypoint)

    return robot_pose

def path_planning(sx,sy,gx,gy,ox,oy, grid_size, robot_radius, offset):
    a_star = AStarPlanner(ox, oy, grid_size, robot_radius)
    # current output goes to position of fruit directly, need to adjust offset by radius around the fruit
    rx, ry = a_star.planning(sx, sy, gx, gy) # Returns coordinates from goal to start
    # get distance between rx,ry[0] and rx,ry[1]
    rx_dist = rx[0] - rx[1]
    ry_dist = ry[0] - ry[1]
    # find the intersecting point between the two where distance from robot to rx,ry = offset
    # update rx,ry[0] to intersect position
    rx[0] = rx[1] + rx_dist*offset/grid_size
    ry[0] = ry[1] + ry_dist*offset/grid_size
    # update start position for next fruit
    sx = rx[1]
    sy = ry[1]

    return sx, sy, rx, ry

def det_fruits(img, camera_matrix, missing_fruits):
    fx = []
    fy = []
    target_pose_dict = {}
    # get yolo detection
    detector_output, network_vis = detector.yolo_detection(img)

    # get estimated positions of detected fruits
    completed_img_dict = TPE.get_image_info(detector_output, ekf.robot.state)
    target_pose_dict = TPE.estimate_pose(camera_matrix, completed_img_dict)
    for fruit in missing_fruits:
        if fruit in target_pose_dict:
            fx.append(np.round(target_pose_dict[fruit]['x']/0.4)*0.4)
            fy.append(np.round(target_pose_dict[fruit]['y']/0.4)*0.4)
            print("detected,", fruit, " at ", np.round(target_pose_dict[fruit]['x']/0.4)*0.4, np.round(target_pose_dict[fruit]['y']/0.4)*0.4)

    return fx, fy

# main loop
if __name__ == "__main__":
    parser = argparse.ArgumentParser("Fruit searching")
    parser.add_argument("--map", type=str, default='M4_true_map.txt')
    parser.add_argument("--ip", metavar='', type=str, default='localhost')
    parser.add_argument("--port", metavar='', type=int, default=8000)
    parser.add_argument("--ckpt", default='network/fruits_model/best.pt')
    args, _ = parser.parse_known_args()

    ppi = Alphabot(args.ip,args.port)
    detector = Detector(args.ckpt, use_gpu=False)

    # read in the true map
    full_fruits_list = ['redapple', 'greenapple', 'orange', 'mango', 'capsicum']
    fruits_list, fruits_true_pos, aruco_true_pos = read_true_map(args.map)
    search_list = read_search_list()
    missing_fruits = []
    for fruit in full_fruits_list: # list out all unknown fruits positions
        if (fruit in fruits_list) == False:
            missing_fruits.append(fruit)

    fruits_true_pos_seq = [[],[],[]] # number of fruits to drive to and sequence

    # import all param files for Robot()
    datadir = 'calibration/param/'
    fileK = "{}intrinsic.txt".format(datadir)
    camera_matrix = np.loadtxt(fileK, delimiter=',')
    fileD = "{}distCoeffs.txt".format(datadir)
    dist_coeffs = np.loadtxt(fileD, delimiter=',')
    fileS = "{}scale.txt".format(datadir)
    scale = np.loadtxt(fileS, delimiter=',')
    fileB = "{}baseline.txt".format(datadir)
    baseline = np.loadtxt(fileB, delimiter=',')

    # intialize ekf, slam and robot
    robot = Robot(baseline, scale, camera_matrix, dist_coeffs)
    ekf = EKF(robot)
    aruco_det = aruco.aruco_detector(ekf.robot, marker_length = 0.06)

    #initialize variables
    x, y = 0.0, 0.0
    sx, sy = 0.0, 0.0
    waypoint = [[0.0, 0.0, 0.0]]
    robot_pose = [[sx, sy, 0.0]]
    move_1 = np.empty((0,3))
    move_2 = np.empty((0,3))
    img_1 = np.zeros([480, 640, 3], dtype=np.uint8)
    img_2 = np.zeros([480, 640, 3], dtype=np.uint8)

    grid_size, robot_radius, offset = 0.2, 0.12, 0.0  # [m]
    pose_error = 0.2

    # obstacle coordinates
    ox, oy = [], []

    # arrange search order sequence
    for i in range(0,len(search_list)):
        for j in range(0,len(fruits_list)):
            if search_list[i] == fruits_list[j]:
                fruits_true_pos_seq[i] = fruits_true_pos[j]

    # store all known obstacle positions in ox and oy
    for marker in range(0,len(aruco_true_pos)):
        ox.append(aruco_true_pos[marker][0])
        oy.append(aruco_true_pos[marker][1])

    # check initial postion of robot(in case robot wasn't placed perfectly at 0,0,0)
    img_1 = ppi.get_image()
    robot_pose = get_robot_pose([0,0,0],[0,0,0],img_1,img_1)
    sx, sy = robot_pose[0][0], robot_pose[0][1]

    while len(fruits_true_pos_seq) != 0: # while still have fruit to drive to

        ppi.set_velocity([0, 0], tick=[0,0])
        time.sleep(1)

        gx = fruits_true_pos_seq[0][0]
        gy = fruits_true_pos_seq[0][1]

        sx,sy,rx,ry = path_planning(sx,sy,gx,gy,ox,oy,grid_size,robot_radius,offset)

        print("path")
        print(rx)
        print(ry)
        print("Start drive")

        for j in range(len(rx)-2,-1,-1): # move robot to specified waypoint
            waypoint[0][0] = rx[j]
            waypoint[0][1] = ry[j]

            if [waypoint[0][0],waypoint[0][1]] == [rx[0],ry[0]] or j == 1:
                move_1, move_2, img_1, img_2 = drive_to_point(scale, baseline, waypoint,robot_pose)
                robot_pose = get_robot_pose(move_1, move_2, img_1, img_2)
                print(j, " robot pose: ", robot_pose, " waypoint: ",waypoint)

                print("Fruit Reached")
                ppi.set_Buzzer(1)
                ppi.set_velocity([0, 0], tick=[0,0])
                fruits_true_pos_seq.pop(0)
                time.sleep(1)
                ppi.set_Buzzer(0)
                break

            if [waypoint[0][0],waypoint[0][1]] != [rx[0],ry[0]]: # if haven't reached destination
                move_1, move_2, img_1, img_2 = drive_to_point(scale, baseline, waypoint,robot_pose)
                robot_pose = get_robot_pose(move_1, move_2, img_1, img_2)
                print(j, " robot pose: ", robot_pose, " waypoint: ",waypoint)
                ppi.set_velocity([0, 0], tick=[0,0])

                """
                robot_pose, triggered = check_IR(robot_pose)
                if triggered == 1:
                    sx, sy = robot_pose[0][0], robot_pose[0][1]
                    break
                """

                # check if unknown fruits are detected
                fx,fy = det_fruits(img_2, camera_matrix, missing_fruits)
                if len(fx) !=0: # update obstacle list to include position of unknown fruits
                    for marker in range(0,len(fx)):
                        ox.append(fx[marker])
                        oy.append(fy[marker])
                    sx, sy = robot_pose[0][0], robot_pose[0][1]
                    break # quit for loop to redo path planning and restart travelling

                #robot_pose = calib_Rob_Pose() # check robot position
                # check if position is as expected if not, correct position and replan path
                """
                if waypoint[0][0]-pose_error>robot_pose[0][0] or waypoint[0][0]+pose_error<robot_pose[0][0] or waypoint[0][1]-pose_error>robot_pose[0][1] or waypoint[0][1]+pose_error<robot_pose[0][1]:
                    print("position majorly wrong, replanning path")
                    move_1, move_2, img_1, img_2 = drive_to_point(scale, baseline, waypoint,robot_pose)
                    robot_pose = get_robot_pose(move_1, move_2, img_1, img_2)
                    sx = waypoint[0][0]
                    sy = waypoint[0][1]
                    break
                """

            ppi.set_velocity([0, 0], tick=[0,0])
            time.sleep(1)

    ppi.set_velocity([0, 0], tick=[0,0])


import re
import datetime

import snscrape.modules.twitter as sntwitter

from .scraper import Scraper, Content

class TwitterScraper(Scraper):
	def __init__(self) -> None:
		pass

	def _query_constructor(
		self,
		ticker: str,
		min_date: datetime.datetime,
		max_date: datetime.datetime,
		min_faves: int = 300) -> str:
		format_date = lambda date: (
			f'{date.year}-'
			f'{str(date.month).rjust(2, "0")}-'
			f'{str(date.day).rjust(2, "0")}')

		return (
			f'${ticker} min_faves:{min_faves} until:{format_date(max_date)} '
			f'since:{format_date(min_date)}')

	def fetch_content_for_day(
		self,
		ticker: str,
		day: datetime.datetime) -> list[Content]:
		return self._fetch_content_for_day(ticker=ticker, day=day)

	def _fetch_content_for_day(
		self,
		ticker: str,
		day: datetime.datetime,
		min_faves: int = 300,
		limit: int = 25) -> list[Content]:
		query = self._query_constructor(
			ticker=ticker,
			min_faves=min_faves,
			min_date=day,
			max_date=day + datetime.timedelta(days=1))

		raw_tweets = sntwitter.TwitterSearchScraper(query).get_items()

		tweets = list()
		for _ in range(limit):
			curr_tweet = next(raw_tweets, None)
			if not curr_tweet:
				break

			tweet_content_trunc = curr_tweet.rawContent[:127]
			tweets.append(Content(content=tweet_content_trunc, date=day))

		return tweets


from data import load_original_gmd_dataset_pickle

# Load 2bar gmd dataset as a dictionary
gmd_dict = load_original_gmd_dataset_pickle(
    gmd_pickle_path="data/gmd/resources/storedDicts/groove_2bar-midionly.bz2pickle")

gmd_dict.keys()          # dict_keys(['train', 'test', 'validation'])
gmd_dict['train'].keys()        #dict_keys(['drummer', 'session', 'loop_id', 'master_id', 'style_primary', 'style_secondary', 'bpm', 'beat_type', 'time_signature', 'full_midi_filename', 'full_audio_filename', 'midi', 'note_sequence'])

# =================================================================================================
# Extract HVO_Sequences from the dictionaries

from data import extract_hvo_sequences_dict, get_drum_mapping_using_label

hvo_dict = extract_hvo_sequences_dict (
    gmd_dict=gmd_dict,
    beat_division_factor=[4],
    drum_mapping=get_drum_mapping_using_label("ROLAND_REDUCED_MAPPING"))

# =================================================================================================

# Load GMD Dataset in `HVO_Sequence` format using a single command
from data import load_gmd_hvo_sequences
test_set = load_gmd_hvo_sequences(
    dataset_setting_json_path="data/dataset_json_settings/4_4_Beats_gmd.json",
    subset_tag="test",
    force_regenerate=False)

# =================================================================================================
# Load dataset as torch.utils.data.Dataset
from data import MonotonicGrooveDataset

# load dataset as torch.utils.data.Dataset
training_dataset = MonotonicGrooveDataset(
    dataset_setting_json_path="data/dataset_json_settings/4_4_Beats_gmd.json",
    subset_tag="train",
    max_len=32,
    tapped_voice_idx=2,
    collapse_tapped_sequence=False,
    load_as_tensor=True,
    sort_by_metadata_key="loop_id",
    genre_loss_balancing_beta=0.5,
    voice_loss_balancing_beta=0.5
)

training_dataset.__getitem__(0)
training_dataset.get_voice_counts()
training_dataset.get_genre_distributions_dict()
# training_dataset.visualize_global_hit_count_ratios_heatmap()
#training_dataset.visualize_genre_distributions(show_inverted_weights=True)
# =================================================================================================

# use the above dataset in the training pipeline, you need to use torch.utils.data.DataLoader
from torch.utils.data import DataLoader
train_dataloader = DataLoader(training_dataset, batch_size=32, shuffle=True)

epochs = 10
for epoch in range(epochs):
    # in each epoch we iterate over the entire dataset
    for batch_count, (inputs, outputs, indices) in enumerate(train_dataloader):
        print(f"Epoch {epoch} - Batch #{batch_count} - inputs.shape {inputs.shape} - "
              f"outputs.shape {outputs.shape} - indices.shape {indices.shape} ")


import torch.nn as nn
import torch
import numpy as np
import torch.nn.functional as F
from einops import rearrange, repeat

class ScaledDotProductAttention(nn.Module):
    """Scaled dot-product attention mechanism."""

    def __init__(self, attention_dropout=0.0):
        super(ScaledDotProductAttention, self).__init__()
        self.dropout = nn.Dropout(attention_dropout)
        self.softmax = nn.Softmax(dim=2)

    def forward(self, q, k, v, scale=None, attn_mask=None):
        """
        前向传播.
        Args:
        	q: Queries张量，形状为[B, L_q, D_q]
        	k: Keys张量，形状为[B, L_k, D_k]
        	v: Values张量，形状为[B, L_v, D_v]，一般来说就是k
        	scale: 缩放因子，一个浮点标量
        	attn_mask: Masking张量，形状为[B, L_q, L_k]

        Returns:
        	上下文张量和attention张量
        """
        attention = torch.bmm(q, k.transpose(1, 2))
        if scale:
            attention = attention * scale
        if attn_mask is not None:
            # 给需要mask的地方设置一个负无穷
            # attention = attention.cuda()
            attention = attention.masked_fill_(attn_mask.cuda(), -np.inf)
        # 计算softmax
        attention = self.softmax(attention)
        # 添加dropout
        attention = self.dropout(attention)
        # 和V做点积
        context = torch.bmm(attention, v)
        return context, attention

class MultiHeadAttention(nn.Module):

    def __init__(self, model_dim=512, num_heads=8, dropout=0.0):
        super(MultiHeadAttention, self).__init__()

        self.dim_per_head = model_dim // num_heads
        self.num_heads = num_heads
        self.linear_k = nn.Linear(model_dim, self.dim_per_head * num_heads)
        self.linear_v = nn.Linear(model_dim, self.dim_per_head * num_heads)
        self.linear_q = nn.Linear(model_dim, self.dim_per_head * num_heads)

        self.dot_product_attention = ScaledDotProductAttention(dropout)
        self.linear_final = nn.Linear(model_dim, model_dim)
        self.dropout = nn.Dropout(dropout)

        # multi-head attention之后需要做layer norm
        self.layer_norm = nn.LayerNorm(model_dim)

    def forward(self, key, value, query, attn_mask=None):
        # 残差连接
        residual = query
        dim_per_head = self.dim_per_head
        num_heads = self.num_heads
        batch_size = key.size(0)

        # linear projection
        key = self.linear_k(key)
        value = self.linear_v(value)
        query = self.linear_q(query)

        # split by heads
        key = key.view(batch_size * num_heads, -1, dim_per_head)
        value = value.view(batch_size * num_heads, -1, dim_per_head)
        query = query.view(batch_size * num_heads, -1, dim_per_head)

        if attn_mask is not None:
            attn_mask = attn_mask.repeat(num_heads, 1, 1)

        # scaled dot product attention
        scale = (key.size(-1)) ** -0.5
        context, attention = self.dot_product_attention(
            query, key, value, scale, attn_mask)

        # concat heads
        context = context.view(batch_size, -1, dim_per_head * num_heads)

        # final linear projection
        output = self.linear_final(context)

        # dropout
        output = self.dropout(output)

        # add residual and norm layer
        output = self.layer_norm(residual + output)

        return output, attention

def padding_mask(seq_k, seq_q):
    # seq_k 和 seq_q 的形状都是 [B,L]
    len_q = seq_q.size(1)
    # `PAD` is 0
    pad_mask = seq_k.eq(0)
    pad_mask = pad_mask.unsqueeze(1).expand(-1, len_q, -1)  # shape [B, L_q, L_k]
    return pad_mask

def sequence_mask(seq):
    batch_size, seq_len, _ = seq.size()
    mask = torch.triu(torch.ones((seq_len, seq_len), dtype=torch.uint8),
                    diagonal=1)
    mask = mask.unsqueeze(0).expand(batch_size, -1, -1)  # [B, L, L]
    return mask

class PositionalEncoding(nn.Module):

    def __init__(self, d_model, max_seq_len):
        """初始化。
        Args:
            d_model: 一个标量。模型的维度，论文默认是512
            max_seq_len: 一个标量。文本序列的最大长度
        """
        super(PositionalEncoding, self).__init__()

        # 根据论文给的公式，构造出PE矩阵
        position_encoding = np.array([
            [pos / np.power(10000, 2.0 * (j // 2) / d_model) for j in range(d_model)]
            for pos in range(max_seq_len)])
        # 偶数列使用sin，奇数列使用cos
        position_encoding[:, 0::2] = np.sin(position_encoding[:, 0::2])
        position_encoding[:, 1::2] = np.cos(position_encoding[:, 1::2])

        # 在PE矩阵的第一行，加上一行全是0的向量，代表这`PAD`的positional encoding
        # 在word embedding中也经常会加上`UNK`，代表位置单词的word embedding，两者十分类似
        # 那么为什么需要这个额外的PAD的编码呢？很简单，因为文本序列的长度不一，我们需要对齐，
        # 短的序列我们使用0在结尾补全，我们也需要这些补全位置的编码，也就是`PAD`对应的位置编码
        pad_row = torch.zeros([1, d_model])
        position_encoding = torch.cat((pad_row, position_encoding))

        # 嵌入操作，+1是因为增加了`PAD`这个补全位置的编码，
        # Word embedding中如果词典增加`UNK`，我们也需要+1。看吧，两者十分相似
        self.position_encoding = nn.Embedding(max_seq_len + 1, d_model)
        self.position_encoding.weight = nn.Parameter(position_encoding,
                                                     requires_grad=False)

    def forward(self, input_len):
        """神经网络的前向传播。

        Args:
          input_len: 一个张量，形状为[BATCH_SIZE, 1]。每一个张量的值代表这一批文本序列中对应的长度。

        Returns:
          返回这一批序列的位置编码，进行了对齐。
        """

        # 找出这一批序列的最大长度
        max_len = torch.max(input_len)
        tensor = torch.cuda.LongTensor if input_len.is_cuda else torch.LongTensor
        # 对每一个序列的位置进行对齐，在原序列位置的后面补上0
        # 这里range从1开始也是因为要避开PAD(0)的位置
        input_pos = tensor(
            [list(range(1, int(len) + 1)) + [0] * (max_len - len) for len in input_len])
        return self.position_encoding(input_pos)

class PositionalWiseFeedForward(nn.Module):

    def __init__(self, model_dim=512, ffn_dim=2048, dropout=0.0):
        super(PositionalWiseFeedForward, self).__init__()
        self.w1 = nn.Conv1d(model_dim, ffn_dim, 1)
        self.w2 = nn.Conv1d(ffn_dim, model_dim, 1)
        self.dropout = nn.Dropout(dropout)
        self.layer_norm = nn.LayerNorm(model_dim)

    def forward(self, x):
        output = x.transpose(1, 2)
        output = self.w2(F.relu(self.w1(output)))
        output = self.dropout(output.transpose(1, 2))

        # add residual and norm layer
        output = self.layer_norm(x + output)
        return output

class EncoderLayer(nn.Module):
    """Encoder的一层。"""

    def __init__(self, model_dim=512, num_heads=8, ffn_dim=2048, dropout=0.0):
        super(EncoderLayer, self).__init__()

        self.attention = MultiHeadAttention(model_dim, num_heads, dropout)
        self.feed_forward = PositionalWiseFeedForward(model_dim, ffn_dim, dropout)

    def forward(self, inputs, attn_mask=None):

        # self attention
        context, attention = self.attention(inputs, inputs, inputs, attn_mask)

        # feed forward network
        output = self.feed_forward(context)

        return output, attention

class Encoder(nn.Module):
    """多层EncoderLayer组成Encoder。"""

    def __init__(self,
               num_layers=6,
               model_dim=512,
               num_heads=8,
               ffn_dim=2048,
               dropout=0.0):
        super(Encoder, self).__init__()

        self.encoder_layers = nn.ModuleList(
          [EncoderLayer(model_dim, num_heads, ffn_dim, dropout) for _ in
           range(num_layers)])

    def forward(self, inputs):
        output = inputs
        # self_attention_mask = padding_mask(inputs, inputs)

        attentions = []
        for encoder in self.encoder_layers:
            output, attention = encoder(output)
            attentions.append(attention)

        return output, attentions

class DecoderLayer(nn.Module):

    def __init__(self, model_dim, num_heads=8, ffn_dim=2048, dropout=0.0):
        super(DecoderLayer, self).__init__()

        self.attention = MultiHeadAttention(model_dim, num_heads, dropout)
        self.feed_forward = PositionalWiseFeedForward(model_dim, ffn_dim, dropout)

    def forward(self,
              dec_inputs,
              enc_outputs,
              self_attn_mask=None,
              context_attn_mask=None):
        # self attention, all inputs are decoder inputs
        dec_output, self_attention = self.attention(
          dec_inputs, dec_inputs, dec_inputs, self_attn_mask)

        # context attention
        # query is decoder's outputs, key and value are encoder's inputs
        dec_output, context_attention = self.attention(
          enc_outputs, enc_outputs, dec_output, context_attn_mask)

        # decoder's output, or context
        dec_output = self.feed_forward(dec_output)

        return dec_output, self_attention, context_attention

class Decoder(nn.Module):

    def __init__(self,
               num_layers=6,
               model_dim=512,
               num_heads=8,
               ffn_dim=2048,
               dropout=0.0):
        super(Decoder, self).__init__()

        self.num_layers = num_layers

        self.decoder_layers = nn.ModuleList(
          [DecoderLayer(model_dim, num_heads, ffn_dim, dropout) for _ in
           range(num_layers)])

    def forward(self, inputs, enc_output, context_attn_mask=None):

        # self_attention_padding_mask = padding_mask(inputs, inputs)
        seq_mask = sequence_mask(inputs)
        self_attn_mask = torch.gt((seq_mask), 0)
        self_attn_mask = None
        self_attentions = []
        context_attentions = []
        output = inputs
        for decoder in self.decoder_layers:
            output, self_attn, context_attn = decoder(output, enc_output, self_attn_mask, context_attn_mask)
            self_attentions.append(self_attn)
            context_attentions.append(context_attn)

        return output, self_attentions, context_attentions

class Transformer(nn.Module):
    def __init__(self,
               N,
               dim,
               model_dim=512,
               num_layers=6,
               num_heads=8,
               ffn_dim=2048,
               dropout=0.2,
               with_attn=False,
               condition=False
               ):
        super(Transformer, self).__init__()
        self.N = N
        self.dim = dim
        self.with_attn = with_attn
        self.condition = condition
        self.patch_to_embedding = nn.Linear(dim, model_dim)
        self.pos_embedding = nn.Parameter(torch.randn(1, N, model_dim)) # 1, N, dim
        if self.condition:
            # dim+dim : 上采样通道扩增
            self.condition_to_embedding = nn.Linear(dim, model_dim)
            self.condition_pos_embedding = nn.Parameter(torch.randn(1, N, model_dim)) # 1, N, dim
        self.encoder = Encoder(num_layers, model_dim, num_heads, ffn_dim, dropout)
        self.decoder = Decoder(num_layers, model_dim, num_heads, ffn_dim, dropout)

        self.linear = nn.Linear(model_dim, dim, bias=False)
    def forward(self, x, condition=None):
        b, N, C = x.size()
        # print('x.size()',x.size())
        # condition的利用方式存疑？
        if condition is None:
            x = self.patch_to_embedding(x) # b, N, dim
            x += self.pos_embedding
            x, enc_self_attn = self.encoder(x)
            output, dec_self_attn, ctx_attn = self.decoder(inputs=x, enc_output=x)
        elif self.condition:
            '''考虑condition'''
            x = self.patch_to_embedding(x) # b, N, dim
            x += self.pos_embedding
            condition = self.condition_to_embedding(condition)
            condition += self.condition_pos_embedding
            condition, enc_self_attn = self.encoder(condition)
            output, dec_self_attn, ctx_attn = self.decoder(inputs=x, enc_output=condition)
        # elif self.condition:
        #     x = torch.cat([x, condition], dim=2)
        #     x = self.condition_to_embedding(x) # b, N, dim
        #     x += self.pos_embedding
        #     x, enc_self_attn = self.encoder(x)
        #     output, dec_self_attn, ctx_attn = self.decoder(inputs=x, enc_output=x)
        output = self.linear(output)
        if self.with_attn:
            return output, enc_self_attn, dec_self_attn, ctx_attn
        else:
            return output

class TransformerLayer(nn.Module):
    def __init__(self, size, patch_size, channel_transform=False, transpose_transform=False, condition=False, MiniTransFormer=None, with_mask=False):
        super(TransformerLayer, self).__init__()
        # size: [c, h, w]
        self.input_dim, self.H, self.W = size
        self.p = patch_size # patch_size
        self.channel_transform = channel_transform
        self.transpose_transform = transpose_transform
        self.patch_dim = self.input_dim*(self.p*self.p)
        self.patch_num = (self.H//self.p)*(self.W//self.p)

        model_dim = 256
        num_layers = 6
        num_heads = 8
        ffn_dim = 512
        if MiniTransFormer is not None:
            model_dim, num_layers, num_heads, ffn_dim = MiniTransFormer
        self.condition = condition
        self.transformer_h = Transformer(
            N=self.patch_num,
            dim=self.patch_dim,
            model_dim=model_dim,
            num_layers=num_layers,
            num_heads=num_heads,
            ffn_dim=ffn_dim,
            condition=self.condition
        )

        if self.transpose_transform:
            self.transformer_w = Transformer(
                N=self.patch_num,
                dim=self.patch_dim,
                model_dim=model_dim,
                num_layers=num_layers,
                num_heads=num_heads,
                ffn_dim=ffn_dim,
                condition=self.condition
            )
        if self.channel_transform:
            self.transformer_channel = Transformer(
                N=self.patch_dim,
                dim=self.patch_num,
                model_dim=model_dim,
                num_layers=num_layers,
                num_heads=num_heads,
                ffn_dim=ffn_dim,
                condition=self.condition
            )
        self.out_channel = self.input_dim*(1+self.channel_transform+self.transpose_transform)
        self.to_input_dim = None
        if self.out_channel != self.input_dim:
            self.to_input_dim = nn.Conv2d(self.out_channel, self.input_dim, kernel_size=1)
    def forward(self, x, condition=None):
        b, c, h, w = x.size()
        p = self.p

        if self.transpose_transform:
            x_trans = x.transpose(2, 3)
            condition_trans = None
            if condition is not None:
                condition_trans = condition_trans.transpose(2, 3)
        # P = 8, patch_size 大小， 32 * 32
        x = rearrange(x, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = p, p2 = p) # b, N, p^2*C -> b, 1024, 320
        if condition is not None:
            condition = rearrange(condition, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = p, p2 = p)

        '''channel 并行'''
        if self.channel_transform:
            x_channel = x.permute(0, 2, 1)
            condition_channel = None
            if condition is not None:
                condition_channel = condition.permute(0, 2, 1)
            x_channel = self.transformer_channel(x_channel, condition_channel)
            x_channel = x_channel.permute(0, 2, 1)
        '''结束'''

        x = self.transformer_h(x, condition)
        x = rearrange(x, ' b (h w) (p1 p2 c) -> b c (h p1) (w p2)', h = h//p, w = w//p, p1 = p, p2 = p) # b, c, h, w

        if self.transpose_transform:
            x_trans = rearrange(x_trans, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = p, p2 = p) # b, N, p^2*C
            if condition is not None:
                condition_trans = rearrange(condition_trans, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = p, p2 = p) # b, N, p^2*C -> b, 1024, 320
            x_trans = self.transformer_w(x_trans, condition_trans)
            x_trans = rearrange(x_trans, ' b (h w) (p1 p2 c) -> b c (h p1) (w p2)', h = h//p, w = w//p, p1 = p, p2 = p) # b, c, h, w

            x_trans_origin = x_trans.transpose(2, 3)
            x = torch.cat([x, x_trans_origin], dim=1) # b, c, h, w
        if self.channel_transform:
            x = torch.cat([x, x_channel], dim=1) # b, c, h, w
        if self.to_input_dim is not None:
            return self.to_input_dim(x)
        else:
            return x


import argparse
import os
import pty
import shlex
import sys
from typing import List, Optional

TRT_DOCKER_FILES = {
    "8.4": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_6_tensorrt8_4",
    "8.5": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_5",
    "8.6": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_6",
    "BIN": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt_bin",
}

def run_cmd(cmd: List[str]) -> Optional[int]:
    """
    Runs a shell command and returns the process's return code.

    :param cmd: List of command strings.

    :return: The return code.
    """

    escaped_cmd = " ".join(map(shlex.quote, cmd))
    print(f"[CMD] {escaped_cmd}\n")

    return pty.spawn(cmd)

def get_common_docker_build_args(args: argparse.Namespace) -> List[str]:
    """
    Returns a list of common 'docker build' command-line arguments/options.

    :param args: Arguments to the script.

    :return: A list of common 'docker build' arguments.
    """

    return [
        "--no-cache",
        "-t",
        f"{args.image_name}",
        "--build-arg",
        f"CMAKE_CUDA_ARCHITECTURES={args.cuda_arch}",
        "--build-arg",
        f"ONNXRUNTIME_BRANCH={args.branch}",
    ]

def is_valid_ver_str(version: str, min_comps: int = 0, max_comps: int = 0) -> bool:
    """
    Returns a boolean indicating if the string argument is a 'valid' version string.

    :param version: String representing a version (e.g., 8.2.3).
    :param min_comps: The minimum number of expected version components.
                      Set to 0 to ignore.
    :param max_comps: The maximum number of expected version components.
                      Set to 0 to ignore.

    :return: True if the string is valid (i.e., positive integers separated by dots)
    """

    if not version:
        return False

    ver_nums = version.split(".")
    num_comps = len(ver_nums)

    if num_comps < min_comps:
        return False

    if num_comps > max_comps > 0:
        return False

    return all(num.isdecimal() for num in ver_nums)

def docker_build_trt(args: argparse.Namespace):
    """
    Builds a Docker image that installs TensorRT from a public repository.

    :param args: The arguments to this script.
    """

    if not is_valid_ver_str(args.trt_version, min_comps=2, max_comps=4):
        print(f"[ERROR]: Invalid TensorRT version '{args.trt_version}'", file=sys.stderr)
        sys.exit(1)

    vers_comps = args.trt_version.split(".")
    trt_ver_key = f"{vers_comps[0]}.{vers_comps[1]}"

    if trt_ver_key not in TRT_DOCKER_FILES:
        print(f"[ERROR]: TensorRT version '{args.trt_version}' is currently unsupported", file=sys.stderr)
        sys.exit(1)

    docker_file = TRT_DOCKER_FILES[trt_ver_key]
    docker_file_path = os.path.normpath(os.path.join(args.repo_path, docker_file))

    if not os.path.isfile(docker_file_path):
        print(f"[ERROR]: Invalid docker file path '{docker_file_path}'", file=sys.stderr)
        sys.exit(1)

    common_args = get_common_docker_build_args(args)
    cmd_ret = run_cmd(["docker", "build", *common_args, "-f", f"{docker_file_path}", "."])

    if cmd_ret != 0:
        print(f"[ERROR]: docker build command failed with return code {cmd_ret}", file=sys.stderr)
        sys.exit(1)

def docker_build_trt_bin(args: argparse.Namespace):
    """
    Builds a Docker image that installs TensorRT from a tar.gz package containing binaries.
    See: https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#installing-tar

    :param args: The arguments to this script.
    """

    docker_file = TRT_DOCKER_FILES["BIN"]
    docker_file_path = os.path.normpath(os.path.join(args.repo_path, docker_file))

    if not is_valid_ver_str(args.trt_version, 4, 4):
        print(
            "[ERROR]: Must specify a valid TensorRT version for binary TensorRT installs (e.g., 8.x.x.x)",
            file=sys.stderr,
        )
        sys.exit(1)

    if not is_valid_ver_str(args.tar_cuda_version, 2, 2):
        print("[ERROR]: Must specify a valid CUDA version for binary TensorRT installs (e.g., 11.x)", file=sys.stderr)
        sys.exit(1)

    if not is_valid_ver_str(args.tar_cudnn_version, 2, 2):
        print("[ERROR]: Must specify a valid cuDNN version for binary TensorRT installs (e.g., 8.x)", file=sys.stderr)
        sys.exit(1)

    if not os.path.isfile(docker_file_path):
        print(f"[ERROR]: Invalid docker file path '{docker_file_path}'", file=sys.stderr)
        sys.exit(1)

    if not args.trt_bins_dir or not os.path.isdir(args.trt_bins_dir):
        print(f"[ERROR]: Invalid TensorRT bin directory '{args.trt_bins_dir}'", file=sys.stderr)
        sys.exit(1)

    common_args = get_common_docker_build_args(args)
    cmd_ret = run_cmd(
        [
            "docker",
            "build",
            *common_args,
            "--build-arg",
            f"TAR_TRT_VERSION={args.trt_version}",
            "--build-arg",
            f"TAR_CUDA_VERSION={args.tar_cuda_version}",
            "--build-arg",
            f"TAR_CUDNN_VERSION={args.tar_cudnn_version}",
            "--build-arg",
            f"TRT_BINS_DIR={args.trt_bins_dir}",
            "-f",
            f"{docker_file_path}",
            ".",
        ]
    )

    if cmd_ret != 0:
        print(f"[ERROR]: docker build command failed with return code {cmd_ret}", file=sys.stderr)
        sys.exit(1)

def parse_arguments() -> argparse.Namespace:
    """
    Parses command-line arguments and returns an object with each argument as a field.

    :return: An object whose fields represent the parsed command-line arguments.
    """

    parser = argparse.ArgumentParser()
    parser.add_argument("-r", "--repo_path", required=True, help="Path to the onnxruntime repository")
    parser.add_argument("-i", "--image_name", required=True, help="The resulting Docker image name")
    parser.add_argument("-b", "--branch", default="main", help="Name of the onnxruntime git branch to checkout")
    parser.add_argument("-t", "--trt_version", default="8.4.1.5", help="TensorRT version (e.g., 8.4.1.5)")
    parser.add_argument("-a", "--cuda_arch", default="75", help="CUDA architecture (e.g., 75)")

    # Command-line options for installing TensorRT from binaries.
    parser.add_argument(
        "--install_bin",
        action="store_true",
        default=False,
        help="Enable to install TensorRT from tar.gz binary package",
    )
    parser.add_argument(
        "--tar_cuda_version",
        default="",
        help="CUDA version (e.g., 11.8) used to find TensorRT EA binary tar.gz package",
    )
    parser.add_argument(
        "--tar_cudnn_version",
        default="",
        help="CUDA version (e.g., 8.6) used to find TensorRT EA binary tar.gz package",
    )
    parser.add_argument("--trt_bins_dir", default="", help="Directory containing TensorRT tar.gz package")

    return parser.parse_args()

def main() -> int:
    """
    Script entry point. Builds an Ubuntu-based Docker image with TensorRT.

    :return: 0 on success, 1 on error.
    """

    args = parse_arguments()

    if args.install_bin:
        docker_build_trt_bin(args)
    else:
        docker_build_trt(args)

    return 0

if __name__ == "__main__":
    main()


from django.shortcuts import render, HttpResponse
from .models import PokemonSpawn, NestMigrations

from .config_parser import get_gmaps_api_key
from .tasks import update_migration_date
from .utils.google_storage import download_xml

def show_map(request):
    nxt = NestMigrations.objects.get_last_datetime()
    if not nxt:
        nxt = update_migration_date()

    api_key = get_gmaps_api_key()
    PokemonSpawn.objects.filter(on_map=True)
    return render(request, template_name='map.html', context={'api_key': api_key,
                                                              'file': 'poke_spawn_test.xml',
                                                              'next_migration_date': nxt.strftime(
                                                                  "%Y-%m-%dT%H:%M:%SZ")})

def proxy(request):
    xml = download_xml(request.GET['url'])
    return HttpResponse(xml, content_type='text/xml')


"""
Szyfrowanie tekstu korzystające z ECC.

Autor: Maciej Milewski
Źródło algorytmu szyfrującego: https://pypi.org/project/pycrypto/
"""

from tinyec import registry
from Crypto.Cipher import AES
import hashlib, secrets, binascii
import timeit
import os

def encrypt_AES_GCM(msg, secretKey):
    """ encrypt AES in GCM mode """
    aesCipher = AES.new(secretKey, AES.MODE_GCM)
    ciphertext, authTag = aesCipher.encrypt_and_digest(msg)
    return (ciphertext, aesCipher.nonce, authTag)

def decrypt_AES_GCM(ciphertext, nonce, authTag, secretKey):
    """ decrypt AES in GCM mode """
    aesCipher = AES.new(secretKey, AES.MODE_GCM, nonce)
    plaintext = aesCipher.decrypt_and_verify(ciphertext, authTag)
    return plaintext

def ecc_point_to_256_bit_key(point):
    """ setting generator point for elliptic curve """
    sha = hashlib.sha256(int.to_bytes(point.x, 32, 'big'))
    sha.update(int.to_bytes(point.y, 32, 'big'))
    return sha.digest()

def encrypt_ECC(msg, pubKey, curve):
    """ Encryption using ECC with 256 bit key and GCM mode to chain cipher blocks """
    ciphertextPrivKey = secrets.randbelow(curve.field.n)
    sharedECCKey = ciphertextPrivKey * pubKey
    secretKey = ecc_point_to_256_bit_key(sharedECCKey)
    ciphertext, nonce, authTag = encrypt_AES_GCM(msg, secretKey)
    ciphertextPubKey = ciphertextPrivKey * curve.g
    return (ciphertext, nonce, authTag, ciphertextPubKey)

def decrypt_ECC(encryptedMsg, privKey):
    """ ECC decryption """
    (ciphertext, nonce, authTag, ciphertextPubKey) = encryptedMsg
    sharedECCKey = privKey * ciphertextPubKey
    secretKey = ecc_point_to_256_bit_key(sharedECCKey)
    plaintext = decrypt_AES_GCM(ciphertext, nonce, authTag, secretKey)
    return plaintext

def main():
    """ Crypt&Decrypt message using ECC & AES """
    curve = registry.get_curve('brainpoolP256r1')
    msg = input("Podaj wiadomość: ")
    msg = msg.encode('utf-8')
    print("Oryginalna wiadomość:", msg.decode("ISO-8859-1").rstrip('\0'))

    privKey = secrets.randbelow(curve.field.n)
    pubKey = privKey * curve.g

    starttime = timeit.default_timer()
    encryptedMsg = encrypt_ECC(msg, pubKey, curve)
    encryptedMsgObj = {
        'ciphertext': binascii.hexlify(encryptedMsg[0]),
        'nonce': binascii.hexlify(encryptedMsg[1]),
        'authTag': binascii.hexlify(encryptedMsg[2]),
        'ciphertextPubKey': hex(encryptedMsg[3].x) + hex(encryptedMsg[3].y % 2)[2:]
    }
    print("Czas szyfrowania używając ECC: ", timeit.default_timer() - starttime, "s")

    starttime = timeit.default_timer()
    decryptedMsg = decrypt_ECC(encryptedMsg, privKey)
    print("Czas odszyfrowania ECC: ", timeit.default_timer() - starttime, "s")

    print("Zaszyfrowana wiadomość:", encryptedMsgObj)
    print("Odszyfrowana wiadomość:", decryptedMsg.decode("ISO-8859-1").rstrip('\0'))

if __name__ == "__main__":
    main()


#!/usr/bin/python
import json

import psycopg2

import db_connection
import programminglanguage

def db_connect():
    conn_string = db_connection.get_connection_string()
    # get a connection, if a connect cannot be made an exception will be raised here
    conn = psycopg2.connect(conn_string)
    # conn.cursor will return a cursor object, you can use this cursor to perform queries
    return conn

def make_query(query):
    connection = db_connect()
    cursor = connection.cursor()
    cursor.execute(query)
    objects = create_objects(cursor.fetchall())
    cursor.close()
    connection.close()
    return objects

def update(program):
    conn = db_connect()
    cursor = conn.cursor()
    sql = "UPDATE programmeerimiskeel SET nimi = %s, loomise_aasta = %s, disainer = %s WHERE id = %s"
    cursor.execute(sql, (program.name, program.year, program.designer, program.id))
    conn.commit()
    cursor.close()
    conn.close()

def insert(program):
    conn = db_connect()
    cursor = conn.cursor()
    sql = "INSERT INTO programmeerimiskeel (nimi, loomise_aasta, disainer) VALUES (%s, %s, %s)"
    cursor.execute(sql, (program.name, program.year, program.designer))
    conn.commit()
    cursor.close()
    conn.close()

def create_objects(records):
    result = "["
    size = len(records)
    for index, record in enumerate(records):
        prlng = programminglanguage.ProgrammingLanguage(record[0], record[1], int(record[2]), record[3])
        result += (json.dumps(prlng.serialize()))
        if index < size -1:
            result += ','
    result += "]"
    return result

def delete(id):
    conn = db_connect()
    cursor = conn.cursor()
    sql = "DELETE FROM programmeerimiskeel WHERE programmeerimiskeel.id = " + str(id)
    cursor.execute(sql)
    conn.commit()
    cursor.close()
    conn.close()

def search(id, name, designer, year):
    conn = db_connect()
    cursor = conn.cursor()
    where_clause = ""
    id = id.encode('utf-8')
    name = name.encode('utf-8')
    designer = designer.encode('utf-8')
    year = year.encode('utf-8')
    if not id is "":
        where_clause = " id = " + id
    if not name is '':
        if where_clause is "":
            where_clause += " UPPER(nimi) LIKE UPPER('%" + name + "%')"
        else:
            where_clause += " AND UPPER(nimi) LIKE UPPER('%" + name + "%')"
    if not designer is "":
        if where_clause is "":
            where_clause += " UPPER(disainer) LIKE UPPER('%" + designer + "%')"
        else:
            where_clause += " AND UPPER(disainer) LIKE UPPER('%" + designer + "%')"
    if not year is '':
        if where_clause is "":
            where_clause += " loomise_aasta = " + year
        else:
            where_clause += " AND loomise_aasta = " + year
    # If no search params inserted then return all
    if where_clause is '':
        sql = "SELECT * FROM programmeerimiskeel"
    else:
        sql = "SELECT * FROM programmeerimiskeel WHERE" + where_clause
    cursor.execute(sql)
    objects = create_objects(cursor.fetchall())
    cursor.close()
    conn.close()
    return objects


import json
import requests
import time

print("In order for this script to work, the channel ids, auth token, and username are required in the info.json file")

with open('info.json', 'r') as f:
    info = json.load(f)

username = info['username']
auth_token = info['auth_token']
channels = info['channels']

def get_all_user_messages(auth_token, channel_id):
    all_messages = []
    num_of_returned_messages = 100
    last_message_id = ''
    while num_of_returned_messages == 100:
        messages = get_100_messages(auth_token, channel_id, last_message_id)  # get 100 messages
        if messages:
            # last_message_id = min(messages, key=lambda x: x['timestamp'])
            last_message = sorted(messages, key=lambda x: x['timestamp'], reverse=True)[-1]  # get the last message
            last_message_id = last_message['id']
            print('Downloading messages... {}'.format(last_message['timestamp']), end='\r')
            filtered_messages = [m for m in messages if m['author']['username'] == username]  # filter for user's messages
            all_messages += filtered_messages  # add these messages to the list
            num_of_returned_messages = len(messages)  # count how many messages we got this time
    return all_messages

def get_100_messages(auth, id, last=''):
    if not last:  # first method call, start from beginning (might be able to remove)
        messages = json.loads(requests.get("http://discordapp.com/api/v6/channels/" + id + "/messages",
                                           headers={"authorization": auth},
                                           params={"limit": 100}).content)
    else:
        messages = json.loads(requests.get("http://discordapp.com/api/v6/channels/" + id + "/messages",
                                           headers={"authorization": auth},
                                           params={"before": last, "limit": 100}).content)

    return messages

def delete_messages(auth, id, user, messages):
    print("deleting all messages in {} from username {}".format(id, user))
    for message in messages:
        print(message['content'])
        requests.delete("http://discordapp.com/api/v6/channels/" + id + "/messages/" + message["id"],
                        headers={"authorization": auth})
        time.sleep(0.2)  # avoid rate limiting
    print("all messages deleted")

if __name__ == '__main__':
    for channel_id in channels:
        user_messages = get_all_user_messages(auth_token, channel_id)
        delete_messages(auth_token, channel_id, username, user_messages)


import time
import os
import sys
import traceback

from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.common.exceptions import TimeoutException as SeleniumTimeoutException
from selenium import webdriver

from common import COLORS, say

def get_driver():
    options = webdriver.ChromeOptions()
    options.add_argument('--no-sandbox')

    options.add_argument('--disable-gpu')
    options.add_argument("--disable-extensions")
    options.add_argument("--disable-infobars")

    options.add_argument("--start-maximized")
    options.add_argument('--window-size=1000,700')

    options.add_argument("--headless")
    options.add_argument("--log-level=1")
    options.add_argument('--disable-dev-shm-usage')

    service = ChromeService(executable_path=ChromeDriverManager().install())
    return webdriver.Chrome(options=options, service=service)

def wait_for_scripts(driver):
    height = None
    while (height is None):
        height = driver.execute_script("return UI;")['window_height'];
        say(0, '~', end='', flush=True)
    say(1, '+')

def take_screenshots(driver, output_dir):
    screenshots = []
    error = None
    while (True):
        try:
            ts  = driver.execute_async_script("UI._set_redraw_hook(arguments[arguments.length-1]);")
            delta = abs(ts) - screenshots[~0]["ts"] if len(screenshots)>0 else 0
            #print('received pingback:', ts, ' +', delta)
            say(0, '.', end='', flush=True)
            file_name = f"{output_dir}/grab_{len(screenshots):03g}_{delta}.png"
            driver.save_screenshot(file_name)
            screenshots.append({
                "name": file_name
                ,"delta": delta
                ,"ts" : ts
            })
        except SeleniumTimeoutException as ex:
            say(0, '?', end='')
            say(0, COLORS.WARNING, "selenium timed out while waiting for redraw_hook callback", COLORS.ENDC)
            say(2, 'Exception:', ex)
            say(2, 'Trace:', traceback.format_exc())
            error = (ex, None)
        except Exception as ex:
            error = (ex, traceback.format_exc())

        if (ts < 0):
            say(0, '+', end='')
            say(1, "")
            say(1, "received stop signal")
            break

        if (error is not None):
            if (error[1] is None):
                break
            say(0, '!')
            say(0, "received error:", type(error[0]), error[0])
            say(0, "trace: ", error[1])
            break

    return screenshots, error

def main(user, ip, uri, speedup, output_dir):
    """
    ip = '172.17.0.2'
    uri = 'test-pen-1'
    speedup = '0.2'
    output_dir = '/chalkboard/tmp'
    """
    speedup = float(speedup)

    driver = get_driver()
    url = f'http://{ip}:5000/index.html#{uri}$play?speedup={speedup}'

    say(2, f"selenium fetching URL: {url} into {output_dir}")
    driver.get(url)

    say(2, "selenium waiting for scripts")
    wait_for_scripts(driver)

    say(2, "capturing screenshots")
    screenshots, error = take_screenshots(driver, output_dir)

    if (error is not None)and(error[1] is not None):
        pass
    else:
        say(0, "[s]", end='')
        say(2, "waiting for settlement")
        time.sleep(3)

    say(0, "[c]", end='')
    say(2, "taking final screenshot")
    driver.save_screenshot(f"{output_dir}/result.png")

    say(0, "[l]", end='')
    say(2, "saving js logs")
    logs = driver.execute_script("return UI.logger.logs")
    with open(f"{output_dir}/js_log.txt", "wt") as f:
        f.write("\n".join(map(lambda l:str(l[1]) + '::' + str(l[0]), logs[::-1])))

    say(0, "[d]", end='')
    say(2, "closing selenium driver")
    driver.quit()

    say(0, "[g]", end='')
    say(2, "converting screenshots into gif")
    convert_cmd = ("convert " + " ".join([
        f"-delay {int(f['delta'] * speedup / 10)} {f['name']}" for f in screenshots
    ]) + f" -loop 0  -layers Optimize {output_dir}/result.gif")
    os.system(convert_cmd)

    say(0, "[x]", end='')
    say(2, "cleaning up screenshots")
    for f in screenshots:
        say(2, '.', end='', flush=True)
        os.remove(f["name"])
    say(2, '')

    say(0, "[o]", end='')
    say(2, "switching ownership")
    os.system(f"chown -R {user} {output_dir}/*")

    return ((error is not None)and(error[1] is not None))*1

if __name__ == "__main__":
    if (len(sys.argv) != 7) or (sys.argv[1] == "?"):
        print("usage: play.py <owner> <ip> <uri> <speedup> <output_dir> <verbosity>")
        sys.exit(-1)

    say.__level = int(sys.argv[6])
    say(2, "args: ", sys.argv)

    try:
        code = main(*sys.argv[1:-1])
    except KeyboardInterrupt:
        code = -2

    sys.exit(code)


import pygame
import time
import math
from utils import scale_image, blit_rotate_center

GRASS = scale_image(pygame.image.load("imgs/grass.jpg"), 2.5)
TRACK = scale_image(pygame.image.load("imgs/track.png"), 0.9)

TRACK_BORDER = scale_image(pygame.image.load("imgs/track-border.png"), 0.9)
TRACK_BORDER_MASK = pygame.mask.from_surface(TRACK_BORDER)

FINISH = pygame.image.load("imgs/finish.png")
FINISH_MASK = pygame.mask.from_surface(FINISH)
FINISH_POSITION = (130, 250)

RED_CAR = scale_image(pygame.image.load("imgs/red-car.png"), 0.55)
GREEN_CAR = scale_image(pygame.image.load("imgs/green-car.png"), 0.55)

WIDTH, HEIGHT = TRACK.get_width(), TRACK.get_height()
WIN = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("Racing Game!")

FPS = 60

class AbstractCar:
    def __init__(self, max_vel, rotation_vel):
        self.img = self.IMG
        self.max_vel = max_vel
        self.vel = 0
        self.rotation_vel = rotation_vel
        self.angle = 0
        self.x, self.y = self.START_POS
        self.acceleration = 0.1

    def rotate(self, left=False, right=False):
        if left:
            self.angle += self.rotation_vel
        elif right:
            self.angle -= self.rotation_vel

    def draw(self, win):
        blit_rotate_center(win, self.img, (self.x, self.y), self.angle)

    def move_forward(self):
        self.vel = min(self.vel + self.acceleration, self.max_vel)
        self.move()

    def move_backward(self):
        self.vel = max(self.vel - self.acceleration, -self.max_vel/2)
        self.move()

    def move(self):
        radians = math.radians(self.angle)
        vertical = math.cos(radians) * self.vel
        horizontal = math.sin(radians) * self.vel

        self.y -= vertical
        self.x -= horizontal

    def collide(self, mask, x=0, y=0):
        car_mask = pygame.mask.from_surface(self.img)
        offset = (int(self.x - x), int(self.y - y))
        poi = mask.overlap(car_mask, offset)
        return poi

    def reset(self):
        self.x, self.y = self.START_POS
        self.angle = 0
        self.vel = 0

class PlayerCar(AbstractCar):
    IMG = RED_CAR
    START_POS = (180, 200)

    def reduce_speed(self):
        self.vel = max(self.vel - self.acceleration / 2, 0)
        self.move()

    def bounce(self):
        self.vel = -self.vel
        self.move()

def draw(win, images, player_car):
    for img, pos in images:
        win.blit(img, pos)

    player_car.draw(win)
    pygame.display.update()

def move_player(player_car):
    keys = pygame.key.get_pressed()
    moved = False

    if keys[pygame.K_a]:
        player_car.rotate(left=True)
    if keys[pygame.K_d]:
        player_car.rotate(right=True)
    if keys[pygame.K_w]:
        moved = True
        player_car.move_forward()
    if keys[pygame.K_s]:
        moved = True
        player_car.move_backward()

    if not moved:
        player_car.reduce_speed()

run = True
clock = pygame.time.Clock()
images = [(GRASS, (0, 0)), (TRACK, (0, 0)),
          (FINISH, FINISH_POSITION), (TRACK_BORDER, (0, 0))]
player_car = PlayerCar(8, 8)

while run:
    clock.tick(FPS)

    draw(WIN, images, player_car)

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            run = False
            break

    move_player(player_car)

    if player_car.collide(TRACK_BORDER_MASK) != None:
        player_car.bounce()

    finish_poi_collide = player_car.collide(FINISH_MASK, *FINISH_POSITION)
    if finish_poi_collide != None:
        if finish_poi_collide[1] == 0:
            player_car.bounce()
        else:
            player_car.reset()
            print("finish")

pygame.quit()


#!/usr/bin/env python3
# coding: utf-8

"""Loop detector

Detect loops in Hi-C contact maps with pattern matching.

"""

from setuptools import setup, find_packages

CLASSIFIERS = [
    "Development Status :: 1 - Planning",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: Artistic License",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3.6",
    "Topic :: Scientific/Engineering",
    "Topic :: Scientific/Engineering :: Bio-Informatics",
    "Topic :: Scientific/Engineering :: Visualization",
    "Operating System :: Microsoft :: Windows",
    "Operating System :: POSIX",
    "Operating System :: Unix",
    "Operating System :: MacOS",
]

name = "declooptor"

MAJOR = 0
MINOR = 1
MAINTENANCE = 0
VERSION = f"{MAJOR}.{MINOR}.{MAINTENANCE}"

LICENSE = "Artistic"

with open("requirements.txt", "r") as f:
    REQUIREMENTS = f.read().splitlines()

with open("serpentine/version.py", "w") as f:
    f.write("__version__ = '{}'\n".format(VERSION))

setup(
    name=name,
    author="axel.cournac@pasteur.fr",
    description=__doc__,
    version=VERSION,
    license=LICENSE,
    classifiers=CLASSIFIERS,
    packages=find_packages(exclude=["demos"]),
    install_requires=REQUIREMENTS,
    include_package_data=True,
    entry_points={"console_scripts": ["declooptor=declooptor.detector:_main"]},
)


import numpy as np

import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import warnings
warnings.filterwarnings('ignore')

import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')

from sklearn.preprocessing import Imputer

import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc, roc_auc_score
from sklearn.ensemble import RandomForestClassifier
def show_kde_and_bins(df, col):
    global fig_num
    plt.figure(fig_num)
    sns.kdeplot(df.loc[df['TARGET'] == 0, col], label = 'target == 0')
    sns.kdeplot(df.loc[df['TARGET'] == 1, col], label = 'target == 1')
    plt.xlabel(col)
    plt.ylabel('Density')

    plt.figure(fig_num + 1)
    axes = plt.gca()
    axes.set_xlim([df[col].min(), df[col].max()])
    staging = df[[col,'TARGET']]
    staging['COL_BINNED'] = pd.cut(staging[col], bins = 10)
    final = staging.groupby('COL_BINNED').mean()
    plt.bar(final[col], final['TARGET'])
    plt.xlabel(col)
    plt.ylabel('AVG TARGET')

    fig_num += 2
train = pd.read_csv('../input/application_train.csv')
test = pd.read_csv('../input/application_test.csv')

print(train.shape)

# columns with outliers:
# DAYS_EMPLOYED

outlier = train[train['DAYS_EMPLOYED'] > 30000]
non_outlier = train[train['DAYS_EMPLOYED'] < 30000]

# new column for outlier
train['DAYS_EMPLOYED_OUTLIER'] = 0
train.loc[train['DAYS_EMPLOYED'] > 30000, 'DAYS_EMPLOYED_OUTLIER'] = 1

# fill in outliers with mean
train.loc[train['DAYS_EMPLOYED'] > 30000, 'DAYS_EMPLOYED'] = non_outlier['DAYS_EMPLOYED'].mean()

# clean up DAYS_BIRTH and DAYS_EMPLOYED to be easy to understand
train['YEARS_BIRTH'] = train['DAYS_BIRTH'] / -365
train['YEARS_EMPLOYED'] = train['DAYS_EMPLOYED'] / -365

train = pd.get_dummies(train)

# columns with outliers:
# DAYS_EMPLOYED

outlier_te = test[test['DAYS_EMPLOYED'] > 30000]
non_outlier_te = test[test['DAYS_EMPLOYED'] < 30000]

# new column for outlier
test['DAYS_EMPLOYED_OUTLIER'] = 0
test.loc[test['DAYS_EMPLOYED'] > 30000, 'DAYS_EMPLOYED_OUTLIER'] = 1

# fill in outliers with mean
test.loc[test['DAYS_EMPLOYED'] > 30000, 'DAYS_EMPLOYED'] = non_outlier_te['DAYS_EMPLOYED'].mean()

# clean up DAYS_BIRTH and DAYS_EMPLOYED to be easy to understand
test['YEARS_BIRTH'] = test['DAYS_BIRTH'] / -365
test['YEARS_EMPLOYED'] = test['DAYS_EMPLOYED'] / -365

test = pd.get_dummies(test)

target = train['TARGET']

train, test = train.align(test, join = 'inner', axis = 1)

train['TARGET'] = target

#corr = train.corr()
#print(corr['TARGET'].sort_values(ascending = False))

# strong positive corr -> DAYS_BIRTH, DAYS_EMPLOYED,REGION_RATING_CLIENT_W_CITY
# strong negative corr -> EXT_SOURCE 1,2,3
# Let's explore!

# KDE / pd.cut()

fig_num = 1

show_kde_and_bins(train, 'EXT_SOURCE_1')
show_kde_and_bins(train, 'EXT_SOURCE_2')
show_kde_and_bins(train, 'EXT_SOURCE_3')
show_kde_and_bins(train, 'REGION_RATING_CLIENT_W_CITY')
show_kde_and_bins(train, 'YEARS_BIRTH')
show_kde_and_bins(train, 'YEARS_EMPLOYED')
bureau = pd.read_csv('../input/bureau.csv')
bureau = pd.get_dummies(bureau)

# odd columns
# amt_credit_max_overdue
# AMT_CREDIT_SUM_LIMIT
# AMT_CREDIT_SUM_OVERDUE

def outliers(df, col):
    mean = df[col].mean()
    std_dev = df[col].std()
    df['Z_SCORE'] = (df[col] - mean) / std_dev
    print(df[col].describe())
    print('mean: ', mean,'. std dev: ', std_dev)
    print(df.loc[df['Z_SCORE'] > 5, [col, 'Z_SCORE']])

#outliers(bureau, 'AMT_CREDIT_MAX_OVERDUE')
#outliers(bureau, 'AMT_CREDIT_SUM_LIMIT')
#outliers(bureau, 'AMT_CREDIT_SUM_OVERDUE')
# .. definitely outliers but leaving there since it seems legit (not one value or anything mysterious)

# metrics on bureau
bureau_staging = bureau \
    .drop(columns = ['SK_ID_BUREAU']) \
    .groupby('SK_ID_CURR') \
    .agg(['count','mean','min','max', 'sum']) \
    .reset_index()

columns = ['SK_ID_CURR']

# Convert multi-level index from .agg() into clean columns
# borrowing from: https://www.kaggle.com/willkoehrsen/introduction-to-manual-feature-engineering
for var in bureau_staging.columns.levels[0]:
    if var != 'SK_ID_CURR':
        for stat in bureau_staging.columns.levels[1][:-1]:
            columns.append('bureau_%s_%s' % (var, stat))

bureau_staging.columns = columns

train = pd.merge(
    train,
    bureau_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)

test = pd.merge(
    test,
    bureau_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)

# columns.append('TARGET')

# bureau_corr = tr[columns]
#tr = imputer.fit_transform(tr)
#print(tr)

# corr = bureau_corr.corr()
# print(corr['TARGET'].sort_values(ascending = False))

# strong-ish corrs:
# bureau_DAYS_CREDIT_mean
# bureau_CREDIT_ACTIVE_Active_mean
# bureau_DAYS_CREDIT_min

show_kde_and_bins(train, 'bureau_DAYS_CREDIT_mean')
show_kde_and_bins(train, 'bureau_CREDIT_ACTIVE_Active_mean')
show_kde_and_bins(train, 'bureau_DAYS_CREDIT_min')
previous_application = pd.read_csv('../input/previous_application.csv')

previous_application = previous_application[
    [
        'SK_ID_CURR',
        'CODE_REJECT_REASON',
        'NAME_CONTRACT_STATUS',
        'NAME_PRODUCT_TYPE'
    ]
]
previous_application = pd.get_dummies(previous_application)

previous_application_staging = previous_application \
    .groupby('SK_ID_CURR') \
    .agg(['count','mean','min','max', 'sum']) \
    .reset_index()

columns = ['SK_ID_CURR']

# Convert multi-level index from .agg() into clean columns
# borrowing from: https://www.kaggle.com/willkoehrsen/introduction-to-manual-feature-engineering
for var in previous_application_staging.columns.levels[0]:
    if var != 'SK_ID_CURR':
        for stat in previous_application_staging.columns.levels[1][:-1]:
            columns.append('previous_application_%s_%s' % (var, stat))

previous_application_staging.columns = columns

previous_application = previous_application_staging[
    [
        'SK_ID_CURR',
        'previous_application_CODE_REJECT_REASON_XAP_mean',
        'previous_application_NAME_CONTRACT_STATUS_Approved_mean',
        'previous_application_NAME_CONTRACT_STATUS_Refused_mean',
        'previous_application_NAME_CONTRACT_STATUS_Refused_sum',
        'previous_application_CODE_REJECT_REASON_SCOFR_max',
        'previous_application_NAME_PRODUCT_TYPE_walk-in_sum'
    ]
]

train = pd.merge(
    train,
    previous_application_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)

test = pd.merge(
    test,
    previous_application_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)
POS_CASH_balance = pd.read_csv('../input/POS_CASH_balance.csv')
POS_CASH_balance = POS_CASH_balance[['SK_ID_CURR', 'MONTHS_BALANCE']]

POS_CASH_balance_staging = POS_CASH_balance \
    .groupby('SK_ID_CURR') \
    .agg(['count','mean','min','max', 'sum']) \
    .reset_index()

columns = ['SK_ID_CURR']

# Convert multi-level index from .agg() into clean columns
# borrowing from: https://www.kaggle.com/willkoehrsen/introduction-to-manual-feature-engineering
for var in POS_CASH_balance_staging.columns.levels[0]:
    if var != 'SK_ID_CURR':
        for stat in POS_CASH_balance_staging.columns.levels[1][:-1]:
            columns.append('POS_CASH_balance_%s_%s' % (var, stat))

POS_CASH_balance_staging.columns = columns

train = pd.merge(
    train,
    POS_CASH_balance_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)

test = pd.merge(
    test,
    POS_CASH_balance_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)
credit_card_balance = pd.read_csv('../input/credit_card_balance.csv')

credit_card_balance_staging = credit_card_balance \
    .groupby('SK_ID_CURR') \
    .agg(['count','mean','min','max', 'sum']) \
    .reset_index()

columns = ['SK_ID_CURR']

# Convert multi-level index from .agg() into clean columns
# borrowing from: https://www.kaggle.com/willkoehrsen/introduction-to-manual-feature-engineering
for var in credit_card_balance_staging.columns.levels[0]:
    if var != 'SK_ID_CURR':
        for stat in credit_card_balance_staging.columns.levels[1][:-1]:
            columns.append('credit_card_balance_%s_%s' % (var, stat))

credit_card_balance_staging.columns = columns

train = pd.merge(
    train,
    credit_card_balance_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)

test = pd.merge(
    test,
    credit_card_balance_staging,
    how = 'left',
    on = 'SK_ID_CURR'
)
y = train['TARGET']
imputer = Imputer(strategy = 'mean')
x_cols_app = [
    'EXT_SOURCE_1',
    'EXT_SOURCE_2',
    'EXT_SOURCE_3',
    'REGION_RATING_CLIENT_W_CITY',
    'YEARS_BIRTH',
    'YEARS_EMPLOYED'
]

x_app = train[x_cols_app]
x_app = imputer.fit_transform(x_app)

x_app_train, x_app_test, y_app_train, y_app_test = train_test_split(x_app, y, test_size = 0.33, random_state = 0)

model_app = RandomForestClassifier()
model_app.fit(x_app_train, y_app_train)
predictions_app = model_app.predict_proba(x_app_test)

auc_app = roc_auc_score(y_app_test, predictions_app[:,1])
print('AUC: app only:', auc_app)
x_cols_app_bureau = [
    'EXT_SOURCE_1',
    'EXT_SOURCE_2',
    'EXT_SOURCE_3',
    'REGION_RATING_CLIENT_W_CITY',
    'YEARS_BIRTH',
    'YEARS_EMPLOYED',
    'bureau_DAYS_CREDIT_mean',
    'bureau_CREDIT_ACTIVE_Active_mean',
    'bureau_DAYS_CREDIT_min'
]

x_app_bureau = train[x_cols_app_bureau]
x_app_bureau = imputer.fit_transform(x_app_bureau)

x_app_bureau_train, x_app_bureau_test, y_app_bureau_train, y_app_bureau_test = train_test_split(x_app_bureau, y, test_size = 0.33, random_state = 0)

model_app_bureau = RandomForestClassifier()
model_app_bureau.fit(x_app_bureau_train, y_app_bureau_train)
predictions_app_bureau = model_app_bureau.predict_proba(x_app_bureau_test)

auc_app_bureau = roc_auc_score(y_app_bureau_test, predictions_app_bureau[:,1])
print('AUC: app + bureau:', auc_app_bureau)
x_cols_app_bureau_prev = [
    'EXT_SOURCE_1',
    'EXT_SOURCE_2',
    'EXT_SOURCE_3',
    'REGION_RATING_CLIENT_W_CITY',
    'YEARS_BIRTH',
    'YEARS_EMPLOYED',
    'bureau_DAYS_CREDIT_mean',
    'bureau_CREDIT_ACTIVE_Active_mean',
    'bureau_DAYS_CREDIT_min',
    'previous_application_CODE_REJECT_REASON_XAP_mean',
    'previous_application_NAME_CONTRACT_STATUS_Approved_mean',
    'previous_application_NAME_CONTRACT_STATUS_Refused_mean',
    'previous_application_NAME_CONTRACT_STATUS_Refused_sum',
    'previous_application_CODE_REJECT_REASON_SCOFR_max',
    'previous_application_NAME_PRODUCT_TYPE_walk-in_sum'
]

x_app_bureau_prev = train[x_cols_app_bureau_prev]
x_app_bureau_prev = imputer.fit_transform(x_app_bureau_prev)

x_app_bureau_prev_train, x_app_bureau_prev_test, y_app_bureau_prev_train, y_app_bureau_prev_test = train_test_split(x_app_bureau_prev, y, test_size = 0.33, random_state = 0)

model_app_bureau_prev = RandomForestClassifier()
model_app_bureau_prev.fit(x_app_bureau_prev_train, y_app_bureau_prev_train)
predictions_app_bureau_prev = model_app_bureau_prev.predict_proba(x_app_bureau_prev_test)

auc_app_bureau_prev = roc_auc_score(y_app_bureau_prev_test, predictions_app_bureau_prev[:,1])
print('AUC: application_train.csv + bureau.csv + previous_application.csv:', auc_app_bureau_prev)
#previous_application_CODE_REJECT_REASON_XAP_mean                           -0.073930
#previous_application_NAME_CONTRACT_STATUS_Approved_mean                    -0.063521
#previous_application_NAME_CONTRACT_STATUS_Refused_mean                      0.077671
#previous_application_NAME_CONTRACT_STATUS_Refused_sum                       0.064469
#previous_application_CODE_REJECT_REASON_SCOFR_max                           0.063657
#previous_application_NAME_PRODUCT_TYPE_walk-in_sum                          0.062628
x_cols_app_bureau_prev_pos = [
    'EXT_SOURCE_1',
    'EXT_SOURCE_2',
    'EXT_SOURCE_3',
    'REGION_RATING_CLIENT_W_CITY',
    'YEARS_BIRTH',
    'YEARS_EMPLOYED',
    'bureau_DAYS_CREDIT_mean',
    'bureau_CREDIT_ACTIVE_Active_mean',
    'bureau_DAYS_CREDIT_min',
    'previous_application_CODE_REJECT_REASON_XAP_mean',
    'previous_application_NAME_CONTRACT_STATUS_Approved_mean',
    'previous_application_NAME_CONTRACT_STATUS_Refused_mean',
    'previous_application_NAME_CONTRACT_STATUS_Refused_sum',
    'previous_application_CODE_REJECT_REASON_SCOFR_max',
    'previous_application_NAME_PRODUCT_TYPE_walk-in_sum',
    'POS_CASH_balance_MONTHS_BALANCE_min'
]

x_app_bureau_prev_pos = train[x_cols_app_bureau_prev_pos]
x_app_bureau_prev_pos = imputer.fit_transform(x_app_bureau_prev_pos)

x_app_bureau_prev_pos_train, x_app_bureau_prev_pos_test, y_app_bureau_prev_pos_train, y_app_bureau_prev_pos_test = train_test_split(x_app_bureau_prev_pos, y, test_size = 0.33, random_state = 0)

model_app_bureau_prev_pos = RandomForestClassifier()
model_app_bureau_prev_pos.fit(x_app_bureau_prev_pos_train, y_app_bureau_prev_pos_train)
predictions_app_bureau_prev_pos = model_app_bureau_prev_pos.predict_proba(x_app_bureau_prev_pos_test)

auc_app_bureau_prev_pos = roc_auc_score(y_app_bureau_prev_pos_test, predictions_app_bureau_prev_pos[:,1])
print('AUC: application_train.csv + bureau.csv + previous_application.csv + POS_CASH_balance.csv:', auc_app_bureau_prev_pos)
#previous_application_CODE_REJECT_REASON_XAP_mean                           -0.073930
#previous_application_NAME_CONTRACT_STATUS_Approved_mean                    -0.063521
#previous_application_NAME_CONTRACT_STATUS_Refused_mean                      0.077671
#previous_application_NAME_CONTRACT_STATUS_Refused_sum                       0.064469
#previous_application_CODE_REJECT_REASON_SCOFR_max                           0.063657
#previous_application_NAME_PRODUCT_TYPE_walk-in_sum                          0.062628
x_cols_app_bureau_prev_pos_credit = [
    'EXT_SOURCE_1',
    'EXT_SOURCE_2',
    'EXT_SOURCE_3',
    'REGION_RATING_CLIENT_W_CITY',
    'YEARS_BIRTH',
    'YEARS_EMPLOYED',
    'bureau_DAYS_CREDIT_mean',
    'bureau_CREDIT_ACTIVE_Active_mean',
    'bureau_DAYS_CREDIT_min',
    'previous_application_CODE_REJECT_REASON_XAP_mean',
    'previous_application_NAME_CONTRACT_STATUS_Approved_mean',
    'previous_application_NAME_CONTRACT_STATUS_Refused_mean',
    'previous_application_NAME_CONTRACT_STATUS_Refused_sum',
    'previous_application_CODE_REJECT_REASON_SCOFR_max',
    'previous_application_NAME_PRODUCT_TYPE_walk-in_sum',
    'POS_CASH_balance_MONTHS_BALANCE_min',
    'credit_card_balance_CNT_DRAWINGS_ATM_CURRENT_mean',
    'credit_card_balance_CNT_DRAWINGS_CURRENT_max',
    'credit_card_balance_AMT_BALANCE_mean',
    'credit_card_balance_AMT_TOTAL_RECEIVABLE_mean'
]

x_app_bureau_prev_pos_credit = train[x_cols_app_bureau_prev_pos_credit]
x_app_bureau_prev_pos_credit = imputer.fit_transform(x_app_bureau_prev_pos_credit)

x_app_bureau_prev_pos_credit_train, x_app_bureau_prev_pos_credit_test, y_app_bureau_prev_pos_credit_train, y_app_bureau_prev_pos_credit_test = train_test_split(x_app_bureau_prev_pos_credit, y, test_size = 0.33, random_state = 0)

model_app_bureau_prev_pos_credit = RandomForestClassifier()
model_app_bureau_prev_pos_credit.fit(x_app_bureau_prev_pos_credit_train, y_app_bureau_prev_pos_credit_train)
predictions_app_bureau_prev_pos_credit = model_app_bureau_prev_pos_credit.predict_proba(x_app_bureau_prev_pos_credit_test)

auc_app_bureau_prev_pos_credit = roc_auc_score(y_app_bureau_prev_pos_credit_test, predictions_app_bureau_prev_pos_credit[:,1])
print('AUC: application_train.csv + bureau.csv + previous_application.csv + POS_CASH_balance.csv + credit_card_balance.csv:', auc_app_bureau_prev_pos_credit)
import lightgbm as lgb

x_app_bureau_prev_pos_credit_train, x_app_bureau_prev_pos_credit_test, y_app_bureau_prev_pos_credit_train, y_app_bureau_prev_pos_credit_test = train_test_split(x_app_bureau_prev_pos_credit, y, test_size = 0.33, random_state = 0)

model = lgb.LGBMClassifier(
    n_estimators=10000,
    objective = 'binary',
    class_weight = 'balanced',
    learning_rate = 0.05,
    reg_alpha = 0.1,
    reg_lambda = 0.1,
    subsample = 0.8,
    n_jobs = -1,
    random_state = 50
)

model.fit(
    x_app_bureau_prev_pos_credit_train,
    y_app_bureau_prev_pos_credit_train,
    eval_metric = 'auc'
)

p = model.predict_proba(x_app_bureau_prev_pos_credit_test)

lightGBM_guinea = roc_auc_score(y_app_bureau_prev_pos_credit_test, p[:,1])
print('LGBM:', lightGBM_guinea)
test_submit = test[x_cols_app_bureau_prev_pos_credit]
test_submit = imputer.fit_transform(test_submit)

predictions_final = model.predict_proba(test_submit)
submit = pd.DataFrame({
    "SK_ID_CURR": test['SK_ID_CURR'],
    "TARGET": predictions_final[:,1]
})

submit.to_csv('submit.csv', index = False)


#!/usr/bin/env python

# Author: Tom Parker <palfrey@tevp.net>
# Description: Docs rebuild script
# Requires: Docutils (http://docutils.sourceforge.net/)
# Hacked together from rst2html.py by David Goodger <goodger@users.sourceforge.net>

import locale
try:
    locale.setlocale(locale.LC_ALL, '')
except:
    pass

try:
	from docutils.core import publish_file
except:
	print("You need docutils (http://docutils.sourceforge.net/)\n")
	raise
from .html import Writer

def rebuild(rst,php):
	read_txt = open(rst,'r')
	read_html = open(php,'w')
	html = Writer()
	publish_file(writer=html, source=read_txt, destination=read_html)

	read_html = open(php,'r')
	data = read_html.readlines()
	read_html.close()
	data.insert(0,"<script language=\"php\">$title=\"Comics Grabber\";$TOPDIR=\"../../\";include $TOPDIR.\"top.php3\";</script>")
	data.append("<script language=\"php\">include $TOPDIR.\"bottom.php3\";</script>")
	read_html = open(php,'w')
	read_html.writelines(data)
	read_html.close()

rebuild('readme.txt','index.php')
rebuild('strips.def.txt','strips.php')


import pickle
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn.model_selection import ShuffleSplit
from feature_engineering import FeatureEngineering
from sklearn.ensemble import RandomForestClassifier
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.naive_bayes import MultinomialNB
import pandas as pd

with open("features_train.pickle", 'rb') as output :
	features_train = pickle.load(output)

with open("labels_train.pickle", 'rb') as output :
	labels_train = pickle.load(output)

with open("features_test.pickle", 'rb') as output :
	features_test = pickle.load(output)

with open("labels_test.pickle", 'rb') as output :
	labels_test = pickle.load(output)

class modellig :

	# Build base model using Random Forest Classifier
	# def RFC(self,features_train, labels_train,features_test, labels_test,clean_data_frame):
		# print(features_train.shape)
		# print(features_test.shape)

		# rf_0 = RandomForestClassifier(random_state = 8)
		# print('Parameters currently in use:\n')
		# print(rf_0.get_params())

		# # n_estimators
		# n_estimators = [int(x) for x in np.linspace(start = 200, stop = 1000, num = 5)]

		# # max_features
		# max_features = ['auto', 'sqrt']

		# # max_depth
		# max_depth = [int(x) for x in np.linspace(20, 100, num = 5)]
		# max_depth.append(None)

		# # min_samples_split
		# min_samples_split = [2, 5, 10]

		# # min_samples_leaf
		# min_samples_leaf = [1, 2, 4]

		# # bootstrap
		# bootstrap = [True, False]

		# # Create the random grid
		# random_grid = {'n_estimators': n_estimators,
		#                'max_features': max_features,
		#                'max_depth': max_depth,
		#                'min_samples_split': min_samples_split,
		#                'min_samples_leaf': min_samples_leaf,
		#                'bootstrap': bootstrap}

		# print(random_grid)

		# # First create the base model to tune
		# rfc = RandomForestClassifier(random_state=8)

		# # Definition of the random search
		# random_search = RandomizedSearchCV(estimator=rfc,
		#                                    param_distributions=random_grid,
		#                                    n_iter=50,
		#                                    scoring='accuracy',
		#                                    cv=3,
		#                                    verbose=1,
		#                                    random_state=8)

		# # Fit the random search model
		# random_search.fit(features_train, labels_train)

		# print("The best hyperparameters from Random Search are:")
		# print(random_search.best_params_)
		# print("")
		# print("The mean accuracy of a model with these hyperparameters is:")
		# print(random_search.best_score_)

		# Create the parameter grid based on the results of random search
		# bootstrap = [False]
		# max_depth = [30, 40, 50]
		# max_features = ['sqrt']
		# min_samples_leaf = [1, 2, 4]
		# min_samples_split = [5, 10, 15]
		# n_estimators = [800]

		# param_grid = {
		#     'bootstrap': bootstrap,
		#     'max_depth': max_depth,
		#     'max_features': max_features,
		#     'min_samples_leaf': min_samples_leaf,
		#     'min_samples_split': min_samples_split,
		#     'n_estimators': n_estimators
		# }

		# # Create a base model
		# rfc = RandomForestClassifier(random_state=8)

		# # Manually create the splits in CV in order to be able to fix a random_state (GridSearchCV doesn't have that argument)
		# cv_sets = ShuffleSplit(n_splits = 3, test_size = .33, random_state = 8)

		# # Instantiate the grid search model
		# grid_search = GridSearchCV(estimator=rfc,
		#                            param_grid=param_grid,
		#                            scoring='accuracy',
		#                            cv=cv_sets,
		#                            verbose=1)

		# # Fit the grid search to the data
		# grid_search.fit(features_train, labels_train)

		# print("The best hyperparameters from Grid Search are:")
		# print(grid_search.best_params_)
		# print("")
		# print("The mean accuracy of a model with these hyperparameters is:")
		# print(grid_search.best_score_)

		# best_rfc = grid_search.best_estimator_
		# best_rfc

		# best_rfc.fit(features_train, labels_train)
		# rfc_pred = best_rfc.predict(features_test)

		# # Training accuracy
		# print("The training accuracy is: ")
		# print(accuracy_score(labels_train, best_rfc.predict(features_train)))

		# # Test accuracy
		# print("The test accuracy is: ")
		# print(accuracy_score(labels_test, rfc_pred))

		# # Classification report
		# print("Classification report")
		# print(classification_report(labels_test,rfc_pred))

		# aux_df = clean_data_frame[['Sentiment Class', 'Category_Code']].drop_duplicates().sort_values('Category_Code')
		# conf_matrix = confusion_matrix(labels_test, rfc_pred)
		# plt.figure(figsize=(12.8,6))
		# sns.heatmap(conf_matrix,
		#             annot=True,
		#             xticklabels=aux_df['Sentiment Class'].values,
		#             yticklabels=aux_df['Sentiment Class'].values,
		#             cmap="Blues")
		# plt.ylabel('Predicted')
		# plt.xlabel('Actual')
		# plt.title('Confusion matrix')
		# plt.show()

		# base_model = RandomForestClassifier(random_state = 8)
		# base_model.fit(features_train, labels_train)
		# accuracy_score(labels_test, base_model.predict(features_test))

		# best_rfc.fit(features_train, labels_train)
		# accuracy_score(labels_test, best_rfc.predict(features_test))

		# d = {
		#      'Model': 'Random Forest',
		#      'Training Set Accuracy': accuracy_score(labels_train, best_rfc.predict(features_train)),
		#      'Test Set Accuracy': accuracy_score(labels_test, rfc_pred)
		# }

		# df_models_rfc = pd.DataFrame(d, index=[0])

		# df_models_rfc

		# with open('Models/best_rfc.pickle', 'wb') as output:
		#     pickle.dump(best_rfc, output)

		# with open('Models/df_models_rfc.pickle', 'wb') as output:
		#     pickle.dump(df_models_rfc, output)

		# using multinomial naive bayes classifier

	def MNB(self,features_train, labels_train,features_test, labels_test,clean_data_frame):

		print(features_train.shape)
		print(features_test.shape)

		mnbc = MultinomialNB()
		mnbc

		mnbc.fit(features_train, labels_train)
		mnbc_pred = mnbc.predict(features_test)
		# Training accuracy
		print("The training accuracy is: ")
		print(accuracy_score(labels_train, mnbc.predict(features_train)))

		# Test accuracy
		print("The test accuracy is: ")
		print(accuracy_score(labels_test, mnbc_pred))

		# Classification report
		print("Classification report")
		print(classification_report(labels_test,mnbc_pred))

		aux_df = clean_data_frame[['Sentiment Class', 'category_code']].drop_duplicates().sort_values('category_code')
		conf_matrix = confusion_matrix(labels_test, mnbc_pred)
		plt.figure(figsize=(12.8,6))
		sns.heatmap(conf_matrix/np.sum(conf_matrix),
		            annot=True,
		            fmt='.2%',
		            xticklabels=aux_df['Sentiment Class'].values,
		            yticklabels=aux_df['Sentiment Class'].values,
		            cmap="Blues")
		plt.ylabel('Predicted')
		plt.xlabel('Actual')
		plt.title('Confusion matrix')
		plt.show()

		d = {
		     'Model': 'Multinomial Naïve Bayes',
		     'Training Set Accuracy': accuracy_score(labels_train, mnbc.predict(features_train)),
		     'Test Set Accuracy': accuracy_score(labels_test, mnbc_pred)
		}

		df_models_mnbc = pd.DataFrame(d, index=[0])

		df_models_mnbc

		with open('df_models_mnbc.pickle', 'wb') as output:
			pickle.dump(df_models_mnbc, output)

	def main(self):
		FE=FeatureEngineering()
		features_train, labels_train,features_test, labels_test,clean_data_frame=FE.main()
		# RandomforestC=self.RFC(features_train, labels_train,features_test, labels_test,clean_data_frame)
		MultinomialNB = self.MNB(features_train, labels_train,features_test, labels_test,clean_data_frame)

if __name__ == '__main__':
	KNN = modellig()
	KNN.main()