# Ainsley Lawson.  ainsley.lawson@utoronto.ca
# August 2009

from genshi import HTML

def format_relations_for_flare(self, r_dict):
    """
    Takes a dictionary of relations and returns the HTML for the
    data in json format (for Flare to visualize)
    """

    #TODO:  Strength is always set to 1 at the moment, since I haven't
    #		figured out how to show edge weights yet in Flare.

    html = '['
    for file1 in r_dict:
        if r_dict[file1] == []:
            html += '{"name":"sn/' + file1 + '","strength":1,"links":[]},'
            continue
        html += '{"name":"sn/' + file1 + '","strength":1,"links":['
        for link in r_dict[file1]:
            html += '"sn/' + link + '",'
        html = html[0:len(html) - 1] + ']},'
    html = html[0:len(html) - 1] + ']'
    if html == ']':
        html = '[]'

    data = HTML(html)
    return data

def get_my_files_data(self, person, depth):
    """
    Returns the HTML for a data file that Flare uses to construct the graph
    of the my file relations.
    """
    db = self.env.get_db_cnx()
    cursor = db.cursor()

    q, relations, dist = [], [], {}

    # Get the files the person has worked on
    cursor.execute('SELECT DISTINCT path \
                    FROM node_change n, revision r \
                    WHERE n.rev = r.rev \
                    AND r.author = %s',
                   (person,))
    for row in cursor:
        q.append(row[0])
        dist[row[0]] = 0

    while len(q) != 0:

        # Grab a file from the queue and get its related files
        current = q[0]
        q.remove(current)
        cursor.execute('SELECT file1, file2 \
                        FROM code_relations \
                        WHERE file1 = %s \
                        OR file2 = %s',
                       (current, current,))

        # Examine each file neighbouring node 'current'
        for file1, file2 in cursor:
            if file1 == current:
                file1, file2 = file2, file1

            # Calculate distance from root node(s)
            if (file1 not in dist):
                dist[file1] = dist[current] + 1

                # Add file to queue if necessary
                if dist[file1] <= depth:
                    q.append(file1)

            # Record the relation
            if ((current, file1) not in relations) \
                and ((file1, current) not in relations) \
                and (dist[file1] <= depth):
                relations.append((current, file1))

                # end for
        # end while

    db.close()

    # Set up a dictionary of the relations
    r_dict = {}
    for file1, file2 in relations:
        if file2 not in r_dict:
            r_dict[file2] = []
        if file1 in r_dict:
            r_dict[file1].append(file2)
        else:
            r_dict[file1] = [file2]

    # Format data for Flare
    data = format_relations_for_flare(self, r_dict)

    return data

def get_overall_files_data(self):
    """
    Returns the HTML for a data file that Flare uses to construct the graph
    of the overall file relations.
    """

    db = self.env.get_db_cnx()
    cursor = db.cursor()

    # Get all the file relations
    cursor.execute('SELECT file1, file2 \
                    FROM code_relations')

    # Set up a dictionary of the relations
    r_dict = {}
    for file1, file2 in cursor:
        file1 = file1[0: file1.rfind('/')] + '/*'
        file2 = file2[0: file2.rfind('/')] + '/*'
        if file1 != file2:
            if file2 not in r_dict:
                r_dict[file2] = []
            if file1 in r_dict:
                if file2 not in r_dict[file1]:
                    r_dict[file1].append(file2)
            else:
                r_dict[file1] = [file2]

    db.close()

    # Format data for Flare
    data = format_relations_for_flare(self, r_dict)

    return data

def get_subdirs_files_data(self, subdirs):
    """
    Returns the HTML for a data file that Flare uses to construct the graph
    of the specified file relations.  subdirs is a list of directories.
    Only relations between files in the specified directories (and their
    child directories) are included.
    """

    relations = []
    subdirs.sort()

    db = self.env.get_db_cnx()
    cursor = db.cursor()

    # Get the relevant file relations
    for i in range(0, len(subdirs)):
        dir1 = subdirs[i]

        for j in range(i, len(subdirs)):
            dir2 = subdirs[j]
            cursor.execute('SELECT file1, file2 \
                            FROM code_relations \
                            WHERE (file1 LIKE "' + dir1 + '%" \
                            AND file2 LIKE "' + dir2 + '%")')
            for file1, file2 in cursor:
                if file1 != file2:
                    relations.append((file1, file2))

    # Set up a dictionary of the relations
    r_dict = {}
    for file1, file2 in relations:
        if file2 not in r_dict:
            r_dict[file2] = []
        if file1 in r_dict:
            r_dict[file1].append(file2)
        else:
            r_dict[file1] = [file2]

    # Format data for Flare
    data = format_relations_for_flare(self, r_dict)

    db.close()

    return data

def get_my_social_data(self, me, depth):
    """
    Returns the HTML for a data file that Flare uses to construct the graph
    of the my social relations.
    """
    db = self.env.get_db_cnx()
    cursor = db.cursor()

    q, relations, dist = [], [], {}

    q.append(me)
    dist[me] = 0

    while len(q) != 0:

        # Grab a person from the queue and get its related people
        current = q[0]
        q.remove(current)
        cursor.execute('SELECT person1, person2 \
                        FROM social_relations \
                        WHERE person1 = %s \
                        OR person2 = %s',
                       (current, current,))

        # Examine each person neighbouring node 'current'
        for person1, person2 in cursor:
            if person1 == current:
                person1, person2 = person2, person1

            # Calculate distance from root node(s)
            if (person1 not in dist):
                dist[person1] = dist[current] + 1

                # Add file to queue if necessary
                if dist[person1] <= depth:
                    q.append(person1)

            # Record the relation
            if ((current, person1) not in relations) \
                and ((person1, current) not in relations) \
                and (dist[person1] <= depth):
                relations.append((current, person1))

                # end for
        # end while

    db.close()

    # Set up a dictionary of the relations
    r_dict = {}
    for person1, person2 in relations:
        if person2 not in r_dict:
            r_dict[person2] = []
        if person1 in r_dict:
            r_dict[person1].append(person2)
        else:
            r_dict[person1] = [person2]

    # Format data for Flare
    data = format_relations_for_flare(self, r_dict)

    return data

def get_overall_social_data(self):
    """
    Returns the HTML for a data file that Flare uses to construct the graph
    of the overall social relations.
    """

    db = self.env.get_db_cnx()
    cursor = db.cursor()

    # Get all social relations
    cursor.execute('SELECT person1, person2, strength \
                    FROM social_relations')
    r_dict = {}

    # Set up a dictionary of the relations
    for person1, person2, strength in cursor:
        if person2 not in r_dict:
            r_dict[person2] = []
        if person1 in r_dict:
            r_dict[person1].append(person2)
        else:
            r_dict[person1] = [person2]

    # Format data for Flare
    data = format_relations_for_flare(self, r_dict)

    db.close()

    return (data)

def get_correction_dropdown(self, person):
    """
    Return the HTML for the options of a dropdown list of people
    that person does not talk to.
    """

    db = self.env.get_db_cnx()
    cursor = db.cursor()

    all_people, spoken_to = [], [person]

    # Get all the people
    cursor.execute("SELECT DISTINCT author FROM revision \
                    UNION \
                    SELECT DISTINCT reporter FROM ticket \
                    UNION \
                    SELECT DISTINCT author FROM ticket_change")
    for row in cursor:
        all_people.append(row[0])

    # Get the people that person talks to
    cursor.execute('SELECT DISTINCT person1 \
                    FROM social_relations \
                    WHERE person2 = %s',
                   (person,))
    for row in cursor:
        spoken_to.append(row[0])

    cursor.execute('SELECT DISTINCT person2 \
                    FROM social_relations \
                    WHERE person1 = %s',
                   (person,))
    for row in cursor:
        spoken_to.append(row[0])

    # Subtract to get the people that person doesn't talk to
    all_people = set(all_people)
    spoken_to = set(spoken_to)
    not_spoken_to = all_people.difference(spoken_to)

    # Format as HTML dropdown list options
    drop_down = ""
    for p in not_spoken_to:
        drop_down += '<option value="' + p + '">' + p + '</option>'

    return HTML(drop_down)

def get_my_linkto(self, me, them):
    '''
    Returns the HTML for a data file that Flare uses to construct a graph
    of the shortest path of social connections between person 'me' and
    person 'them'.  If there is no path, then the data file is empty.
    '''

    # Make sure that people have been specified
    if me == "" or them == "":
        return HTML("[]")

    db = self.env.get_db_cnx()
    cursor = db.cursor()

    # Get all the social relations
    cursor.execute("SELECT person1, person2 \
                    FROM social_relations")

    # Get a list of all people who have communicated
    all_nodes = []
    for person1, person2 in cursor:
        all_nodes.append(person1)
        all_nodes.append(person2)
    all_nodes = list(set(all_nodes))

    # Make sure that 'me' and 'them' have had social communication
    if (me not in all_nodes) or (them not in all_nodes):
        return HTML("[]")

    dist = {}    # dist[p] = distance in tree from 'me' to 'p'
    prev = {}    # prev[p] = parent of person in the path from 'me' to 'p'

    # Find shortest path from 'me' to 'them' using Dijkstra's algorithm
    # http://en.wikipedia.org/wiki/Dijkstra%27s_algorithm#Pseudocode

    # Initialize variables
    for node in all_nodes:
        dist[node] = None
        prev[node] = None
    dist[me] = 0
    q = all_nodes

    while len(q) != 0:

        # Get the node in q with smalled dist[]
        smallest = q[0]
        for node in q:
            if dist[smallest] == None and dist[node] != None:
                smallest = node
            elif (dist[smallest] != None and dist[node] != None
                  and dist[node] < dist[smallest]):
                smallest = node

        # Check if 'me' has no social connection to 'them'
        if dist[smallest] == None:
            break

        # Remove smallest from q
        q.remove(smallest)

        # Get the neighbours of smallest that are still in the q
        neighbours = []
        cursor.execute("SELECT person1, person2 \
                        FROM social_relations \
                        WHERE person1 = %s \
                        OR person2 = %s",
                       (smallest, smallest,))
        for person1, person2 in cursor:
            if person1 == smallest and person2 in q:
                neighbours.append(person2)
            elif person2 == smallest and person1 in q:
                neighbours.append(person1)

        # Update distances and parent nodes
        for node in neighbours:
            alt = dist[smallest] + 1
            if dist[node] == None or alt < dist[node]:
                dist[node] = alt
                prev[node] = smallest

    # end while

    db.close()

    # Get all the people in the path from 'me' to 'them'
    r_dict = {}
    current = them
    while current != me:
        r_dict[current] = [prev[current]]
        current = prev[current]
    r_dict[me] = []

    # Format data for Flare
    data = format_relations_for_flare(self, r_dict)

    return data

def get_recent_files(self, author, base_path):
    '''
    Get the HTML for a list of author's most recent files.
    base_path should be the link to the trac environment, so that
    the list can include links to the files.
    '''

    db = self.env.get_db_cnx()
    cursor = db.cursor()

    # Get author's recent files
    cursor.execute('SELECT nc.path, r.time \
                    FROM node_change nc, revision r\
                    WHERE r.author=%s \
                    AND nc.rev = r.rev \
                    AND nc.node_type="F" \
                    ORDER BY r.time DESC',
                   (author,))

    html = '
'        # HTML for the list
    num_recent = 0        # Number of files in the list

    # Construct the list
    for path, time in cursor:

        # Maximum of 10 files in the list
        num_recent += 1
        if num_recent > 10:
            break

        # Link to the file
        html += '
' + \
                path + '
'

    html += '
'

    return HTML(html)

def determine_directory_structure(self, repo):
    """
    Return a dictionary where each entry is the form
    ( directory : [list of sub-directories] ).  The directory is the
    full path of the directory, and each item in the value list is a name
    of a direct sub-directory. 	For example, if the directories are:
        d1, d2, d1/d1a, d1/d1b, d1/d1a/d1aa
    the dictionary would have the following entries:
        { "":["d1", "d2"], "d1":["d1a", "d1b"], "d2":[],
          "d1/d1a":["d1aa"], "d1/d1b":[], "d1/d1a/d1aa":[] }
    """

    db = self.env.get_db_cnx()
    cursor = db.cursor()

    # Get all the directories
    cursor.execute('SELECT DISTINCT path \
                    FROM node_change \
                    WHERE node_type = "D"')

    # Construct the directory dictionary
    # directory_dict[d] = [list of d's subdirectories]
    directory_dict = {}
    for (path,) in cursor:
        ind = path.rfind('/')
        directory = path[0:ind]

        if ind == -1:
            directory = ""
        if path not in directory_dict:
            directory_dict[path] = []
        if directory in directory_dict:
            directory_dict[directory].append(path)
        else:
            directory_dict[directory] = [path]
    db.close()

    return directory_dict

def process_structure(self, directory, directory_dict, indent):
    """
    Construct the HTML for the options of a drop-down list of the
    directory structure.
    - directory is the directory you want to start the structure from
      ("" if you want to start from the root)
    - directory_dict is as the output from determine_directory_structure().
    - indent is the string of blank characters that should preceed this
      option in the dropdown list (pass in "" to start)
    """

    # Get the last token in the path to the directory
    if directory == "":
        this_dir = ""
    else:
        ind = directory.rfind('/')
        this_dir = directory[ind + 1:]

    # Add this directory to the result
    result = ''
    if this_dir != "":
        result = '<option value="' + directory + '">' + indent + \
                 ' |- ' + this_dir + '</option>'

    # Process this directory's subdirectories
    sub_directories = directory_dict[directory]
    indent += '&nbsp&nbsp&nbsp'
    for sub in sub_directories:
        result += process_structure(self, sub, directory_dict, indent)

    return result

def get_directory_list(self, repo):
    """
    Returns the HTML for a form to select directories, and pass them as
    query args to sn_overall_files page.  repo is the svn repository.
    """

    # Get the directory dictionary
    dd = determine_directory_structure(self, repo)

    # Construct the HTML
    html = '\
    <form method="GET" action="sn_overall_files"> \
    \

 \

 \
           \
            Select directories you wish to examine \
            (hold ctrl to select more than one). \
          	 \
           \
            <select name="subdirs" multiple size=6>' + \
           process_structure(self, "", dd, "") + \
           '</select> \
         	 \
          \
           &nbsp&nbsp<input type="submit" value="Submit" /> \
         	 \

 \

 \
    \
   </form>'

    return HTML(html)


""" Module that handles some static map constructing logic helper. """
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import attr
import cv2
import numpy as np

from bc_gym_planning_env.utilities.costmap_2d import CostMap2D
from bc_gym_planning_env.utilities.map_drawing_utils import add_wall_to_static_map
from bc_gym_planning_env.utilities.path_tools import refine_path, orient_path

@attr.s
class MapConfig(object):
    """ Configuration that can be turned into
    (path to follow, costmap) pair """
    trajectory = attr.ib()
    obstacles = attr.ib(type=list)
    size = attr.ib(type=tuple)
    origin = attr.ib(type=tuple)
    resolution = attr.ib(type=float)

@attr.s
class Wall(object):
    """ The most basic type of obstacle - a wall between two points. """
    from_pt = attr.ib(type=np.ndarray)
    to_pt = attr.ib(type=np.ndarray)
    width = attr.ib(type=float, default=0.05)
    cost = attr.ib(default=CostMap2D.LETHAL_OBSTACLE)

    def render(self, costmap):
        """
        Render this wall onto a costmap.
        :param costmap Costmap2D: Costmap to draw the wall on.
        :return Costmap2D: the costmap with the wall on it
        """
        add_wall_to_static_map(costmap, self.from_pt, self.to_pt, width=self.width, cost=self.cost)
        return costmap

def generate_zigzag_walls(trajectory, corridor_y_span, corridor_x_span):
    """Generates list of objects corresponding to zig-zagging walls.
    :param trajectory np.ndarray(N, 3): a zig-zagging trajectory around which
                                        the parallel walls should be drawn
    :param corridor_y_span float: how high should the corridor be on the costmap
    :param corridor_x_span float: how wide should the corridor be?
    :return List[Wall]: list of obstacles to draw on the costmap
    """
    obstacles = []

    stuff = trajectory + np.array([-corridor_y_span, corridor_x_span])
    for from_pt, to_pt in zip(stuff[:-1], stuff[1:]):
        wall = Wall(from_pt=from_pt, to_pt=to_pt)
        obstacles.append(wall)

    stuff = trajectory + np.array([corridor_y_span, -corridor_x_span])
    for from_pt, to_pt in zip(stuff[:-1], stuff[1:]):
        wall = Wall(from_pt=from_pt, to_pt=to_pt)
        obstacles.append(wall)

    return obstacles

def generate_trajectory_and_map_from_config(config):
    """ Based on given MapConfig,
    generate (trajectory to follow, CostMap2D) pair.
    :param config MapConfig: map config based on which we will make a map.
    :return (path_to_follow, CostMap2D) pair: path following task specification
    """

    static_map = CostMap2D.create_empty(
        world_size=config.size,
        resolution=config.resolution,
        world_origin=config.origin
    )

    for obs in config.obstacles:
        static_map = obs.render(static_map)

    return config.trajectory, static_map

def example_config():
    """
    Generates an example config.
    :return MapConfig: an example config
    """
    traj = np.array([
        [2., 2.],
        [2., 4.],
        [4., 4.],
        [6., 4.],
        [6., 8.],
    ])

    obs = generate_zigzag_walls(
        trajectory=traj,
        corridor_y_span=0.65,
        corridor_x_span=0.975
    )

    traj = refine_path(orient_path(traj), 0.05)

    config = MapConfig(
        trajectory=traj,
        obstacles=obs,
        size=(10, 10),
        resolution=0.03,
        origin=(0, 0)
    )

    return config

def _neighbours():
    """
    What to add to my [x,y] coordinates to get
    coordinates of all my neighbours?
    :return List[List[int]]: What to add to my [x,y] coordinates to get
    coordinates of all my neighbours?
    """

    return [
        [0, 1],
        [1, 1],
        [1, 0],
        [1, -1],
        [0, -1],
        [-1, -1],
        [-1, 0],
        [-1, 1],
    ]

def load_costmap_from_img(img_fname):
    """
    Make a costmap from an image.
    :param img_fname str: fname of image to load.
    :return (path_to_follow, CostMap2D) pair: path following task specification
    """
    resolution = 0.03
    world_size = 10

    static_map = CostMap2D.create_empty(
        world_size=(world_size, world_size),
        resolution=resolution,
        world_origin=(0, 0)
    )

    img = cv2.imread(img_fname)
    # opencv loads bgr
    b, g, _ = img[..., 0], img[..., 1], img[..., 2]

    start_img = g
    # walls_img = r
    path_img = b

    start_y, start_x = np.where(start_img == 255)
    start_x = start_x[0]
    start_y = start_y[0]

    current_node = start_x, start_y
    path = []

    found_neighbour = True

    while found_neighbour:
        path.append(np.array(current_node))
        cur_x, cur_y = current_node

        found_neighbour = False
        for v_x, v_y in _neighbours():
            n_x = cur_x + v_x
            n_y = cur_y + v_y
            val = path_img[n_y, n_x]

            if val == 255:
                current_node = n_x, n_y
                found_neighbour = True
                path_img[n_y, n_x] = 0
                break

    prepath = np.array(path)
    path = prepath * resolution

    # Should actually do this, but not allowed for now
    # static_map._data = np.clip(walls_img, 0, 254)

    path = orient_path(path)
    return path, static_map


import copy
import sys
from typing import List, Tuple

import numpy as np

from RBM import RBM

class DNN:
    def __init__(self, network_shape: np.ndarray):
        """
        Initialize the DNN

        :param network_shape: the shape of the network starting from layer 0 to layer P.
        :type network_shape: np.ndarray
        """
        self._DNN: List[RBM] = [None] * (network_shape.size - 1)
        for i in range(network_shape.size - 1):
            self._DNN[i] = RBM(network_shape[i], network_shape[i + 1])

    def pretrain(self, X: np.ndarray, nb_iter: int, learning_rate: float, minibatch_size: int, verbose: bool = False):
        """
        Pretrain the DNN.

        :param X: Data to pretrain the DNN.
        :type X: np.ndarray
        :param nb_iter: Number of iterations of gradient descent.
        :type nb_iter: int
        :param learning_rate: Learning rate of the gradient descent.
        :type learning_rate: float
        :param minibatch_size: Size of the minibatch.
        :type minibatch_size: int
        """

        x = X.copy()
        for i, rbm in enumerate(self._DNN):
            if verbose:
                print(f"Pretraining layer {i}")
            rbm.train(x, nb_iter, minibatch_size, learning_rate, verbose=verbose)
            x = rbm.entree_sortie(x)

    def generer_image(self, nb_iter: int, nb_img: int) -> np.ndarray:
        """
        Generate image with Gibbs procedure.

        :param nb_iter: Number of iteration of Gibbs.
        :type nb_iter: int
        :param nb_img: Number of images to be generated
        :type nb_img: int
        :return: The images generated
        :rtype: np.ndarray
        """

        h = 0.5
        for rbm in self._DNN[::-1]:
            v = (np.random.rand(nb_img, rbm.q) < h) * 1
            for j in range(nb_iter):
                h = (np.random.rand(nb_img, rbm.p) < rbm.sortie_entree(v)) * 1
                v = (np.random.rand(nb_img, rbm.q) < rbm.entree_sortie(h)) * 1

        return h

    def entree_sortie_reseau(self, X: np.ndarray) -> Tuple[List, np.ndarray]:
        sorties = []
        h = X
        for rbm in self._DNN[:-1]:
            h = rbm.entree_sortie(h)
            sorties.append(h)
        probs = self._DNN[-1].calcul_softmax(h)
        return sorties, probs

    def retropropagation(self, X: np.ndarray, labels: np.ndarray, nb_iter: int, learning_rate: float,
                         minibatch_size: int, verbose: bool = False):

        def cost(true_labels, predicted_labels):
            return -np.sum(true_labels * np.log(predicted_labels)) / X.shape[0]

        N = X.shape[0]

        for epoch in range(nb_iter):
            seed = np.random.randint(0, 10000)
            X_copy = X.copy()
            labels_copy = labels.copy()
            np.random.seed(seed)
            np.random.shuffle(X_copy)
            np.random.seed(seed)
            np.random.shuffle(labels_copy)

            DNN_copy = copy.deepcopy(self._DNN)

            # Mini-batch (Should I copy the RBMs?)
            for j in range(0, N, minibatch_size):
                # Taking only the minibatch
                X_minibatch = X_copy[j: min(j + minibatch_size, N)]
                labels_minibatch = labels_copy[j: min(j + minibatch_size, N)]

                # Getting batch size
                batch_size = X_minibatch.shape[0]

                # Forward pass
                sorties, probs = self.entree_sortie_reseau(X_minibatch)
                estimated_labels = probs

                # Computing the gradient
                C = estimated_labels - labels_minibatch
                for k in range(len(DNN_copy) - 1, -1, -1):
                    # Updating the weights
                    if k == 0:
                        dw = X_minibatch.T @ C / batch_size
                    else:
                        dw = sorties[k - 1].T @ C / batch_size

                    # Updating the biaises
                    db = np.sum(C, axis=0) / batch_size

                    # Updating the weights and the biases of the DNN
                    self._DNN[k].w -= learning_rate * dw
                    self._DNN[k].b -= learning_rate * db

                    # Updating the constant derivation
                    if k == 0:
                        C = (C @ DNN_copy[k].w.T) * X_minibatch * (1 - X_minibatch)
                    else:
                        C = (C @ DNN_copy[k].w.T) * sorties[k - 1] * (1 - sorties[k - 1])

            if verbose:
                temp, probs = self.entree_sortie_reseau(X)
                sys.stdout.write(f"\rEpoch {epoch} : {cost(labels, probs)}")
                sys.stdout.flush()

    def test(self, X: np.ndarray, labels: np.ndarray) -> float:
        sorties, probs = self.entree_sortie_reseau(X)
        estimated_labels = np.argmax(probs, axis=1)
        error_rate = np.sum(estimated_labels != labels) / X.shape[0]
        return error_rate


"""
calculate the mAP (mean average precision) for object detection task
Normally we calculate mAP@iou_threshold
"""
import numpy as np
from sklearn.metrics import average_precision_score

from ..vision import iou

def calc_map(iou_threshold, pred, truth):
    """
    Calculate the mAP@iou_threshold, all the coordinates are in the original image space
    Parameters:
        iou_threshold: only greater than this value it can count
        pred: prediction for one image
            [roi1, roi2, roi3, ...]
            roi: {"x1": 1, "x2": 2, "y1": 1, "y2": 2, "class": "cat", "prob": 0.8}
        truth: ground truth for one image
            [bbox1, bbox2, bbox3, ...]
            bbox: {"x1": 1, "x2": 2, "y1": 1, "y2": 2, "class": "cat"}
    Returns:
        mAP@iou_threshold
    """
    result_trues, result_probs = {}, {}

    # we use this flag to identify whether there is a roi ever matched for each bounding box
    for bbox in truth:
        bbox["bbox_matched"] = False

    pred_probs = np.array([item["prob"] for item in pred])
    preds_sorted_by_prob = np.argsort(pred_probs)[::-1]

    for pred_index in preds_sorted_by_prob:
        pred_item = pred[pred_index]
        pred_class = pred_item["class"]
        pred_prob = pred_item["prob"]
        pred_x1, pred_x2, pred_y1, pred_y2 = pred_item["x1"], pred_item["x2"], pred_item["y1"], pred_item["y2"]

        if pred_class not in result_probs:
            result_probs[pred_class] = []
            result_trues[pred_class] = []

        result_probs[pred_class].append(pred_prob)
        # identify whether True Positive or False Positive
        found_match = False

        for bbox in truth:
            truth_class = bbox["class"]
            truth_x1, truth_x2, truth_y1, truth_y2 = bbox["x1"], bbox["x2"], bbox["y1"], bbox["y2"]

            # if pred and truth are not the same
            if truth_class != pred_class:
                continue
            # if the current bounding box is occupied by anthoer roi
            # this makes us not to repeat to count true positives
            if bbox["bbox_matched"] is True:
                continue

            current_iou = iou.iou((pred_x1, pred_y1, pred_x2, pred_y2),
                                  (truth_x1, truth_y1, truth_x2, truth_y2))
            if current_ious >= iou_threshold:
                found_match = True
                bbox["bbox_matched"] = True
                break
            else:
                continue

        result_truth[pred_class].append(int(found_match))

    # we also can't miss False Negative
    for bbox in truth:
        if bbox["bbox_matched"] is False:
            if bbox["class"] not in result_probs:
                result_probs[bbox["class"]] = []
                result_truth[bbox["class"]] = []
            result_probs[bbox["class"]].append(0)
            result_truth[bbox["class"]].append(1)

    # ok now we start to calculate the metrics
    all_aps = []
    for class_item in result_truth:
        ap = average_precision_score(result_truth[class_item], result_probs[class_item])
        all_aps.append(ap)
    return np.mean(all_aps)


import streamlit as st
import plotly_express as px
import pandas as pd
import numpy as np
import datetime

data = pd.read_csv('cleaned_data.csv', index_col=0)

# Sidebar
st.sidebar.title("Sidebar Search")
st.sidebar.info("This is an application for our data analysis")

facility = st.sidebar.multiselect('What is the facility?', data['facilityID'].unique())
apt_title = st.sidebar.multiselect('Select Appointment?', data['appointmentTitle'].unique())

# Page

'''
# Observations:
- We can see that majority of our data is duplicates.
- Our memory usage has comedown from ***183 MB*** to ***4 MB*** .
- eventDate must be changed to a DateTime type
- We do not know what **pid** , **providerID** and **appointmentStatus** actually represent
- our memory usage is 183.1 MB, which will slow things down when using complex transformations.
- We need a different data slice from out database. This dataset is very imbalances.
- If we had access to the database scema, it would help in feature engineering

--------------------------------------------------------------------------------------------------------------------------------------

# ML model

--------------------------------------------------------------------------------------------------------------------------------------

## Dataset and Historical Analysis
'''
st.dataframe(data)
'''
### Unique counts
'''

st.dataframe(data.nunique())
'''
### Discription of Numerical
'''
st.dataframe(data.describe())

'''
### Appointment Titles

'''
st.dataframe(data.appointmentTitle.value_counts())
'''
### Facility
'''
st.dataframe(data.facilityID.value_counts())

'''
### Appointment Status
'''
st.dataframe(data.appointmentStatus.value_counts())

'''
### Datewise
'''
st.dataframe(data.eventDate.value_counts())

'''
### Grouped
'''
st.dataframe(data.groupby(['eventDate','lastName', 'firstName','facilityID']).nunique().sort_values(by=['facilityID'], ascending=False).head(20))

st.dataframe(data.groupby(['eventDate', 'facilityID','appointmentTitle']).nunique())

'''
---------------------------------------------------------------------------------------------------------------
'''
'''
## Running the model
'''

t = st.time_input("Run the model at:", datetime.time(8, 45))
st.write("The model will run at:", t)

option = st.selectbox( "How would you like to be contacted?", ("Email", "Home phone", "Mobile phone"))

'''
---------------------------------------------------------------------------------------------------------------
'''

'''
## Input from streamlit
'''

Patitent = st.text_input("Name of Patient ")
sex = st.selectbox("Sex",options=['Male' , 'Female'])
age = st.slider("Age", 1, 100,1)
p_class = st.selectbox("Patient Class",options=['A Class' , 'B Class' , 'C Class'])

# Filter dataframe
#new_df = data[(data['facilityID'].isin(facility)) & (data['appointmentTitle'].isin(apt_title))]
# write dataframe to screen
#st.dataframe(new_df)


import warnings

import torch

from vp_suite.measure import METRIC_CLASSES

class PredictionMetricProvider:
    r"""
        This class provides bundled access to multiple metrics. With this class's :meth:`get_metrics()` method,
        all specified metric scores are calculated on the same input prediction and target tensor.

        Attributes:
            device (str): A string specifying whether to use the GPU for calculations (`cuda`) or the CPU (`cpu`).
            available_metrics (dict): A dictionary containing the string identifiers and corresponding metrics that the metric provider should use when provided with input tensors.
            metrics (dict): The concrete instantiated metrics that the metric provider uses when provided with input tensors.
    """
    def __init__(self, config: dict):
        r"""
        Initializes the provider by extracting device and metric IDs from the provided config dict
        and instantiating the metrics that shall be used.

        Args:
            config (dict): A dictionary containing the devices and metrics to use.
        """
        self.device = config["device"]
        self.available_metrics = METRIC_CLASSES if config["metrics"] == "all" \
            else {k: METRIC_CLASSES[k] for k in config["metrics"]}
        if config["img_c"] not in [2, 3]:
            warnings.warn("'FVD' measure won't be used since image channels needs to be in [2, 3]")
            self.available_metrics.pop("fvd")
        self.metrics = {k: metric(device=self.device) for k, metric in self.available_metrics.items()}

    def get_metrics(self, pred: torch.Tensor, target: torch.Tensor, frames: int = None, all_frame_cnts: bool = False):
        r"""
        Takes in tensors of predicted frames and the corresponding ground truth and calculates the metric scores for
        the metrics instantiated previously.

        Args:
            pred (torch.Tensor): The predicted frame sequence as a 5D float tensor (batch, frames, c, h, w).
            target (torch.Tensor): The ground truth frame sequence as a 5D float tensor (batch, frames, c, h, w)

            frames (int): If frames is specified, only considers the first 'frames' frames.
            all_frame_cnts (bool): If set to true, elicits metrics for all prediction horizons from 1 up to the maximum number of frames. Otherwise, just elicits metrics for the specified number of frames

        Returns:
            A list of dictionaries, where each dictionary contains the metric ids
            and the corresponding result value for a specific number of prediction frames.
        """
        if pred.ndim != 5 or target.ndim != 5:
            raise ValueError("Input tensors expected to be 5-dimensional!")
        if pred.shape != target.shape:
            raise ValueError("Output images and target images are of different shape!")
        pred = pred.contiguous()
        target = target.contiguous()
        frames = frames or pred.shape[1]

        metrics = []
        frames = [frames] if not all_frame_cnts else range(1, frames + 1)
        for frame_cnt in frames:
            pred_ = pred[:, :frame_cnt]
            target_ = target[:, :frame_cnt]
            frame_cnt_metrics = dict()
            for key, metric in self.metrics.items():
                metric_val = metric(pred_, target_)
                if metric_val is not None:
                    frame_cnt_metrics[f"{key} ({'↑' if metric.BIGGER_IS_BETTER else '↓'})"] \
                        = metric.to_display(metric_val.item())
            # remove metrics that returned 'None' (e.g. because they don't support the current frame cnt
            frame_cnt_metrics = {k: v for k, v in frame_cnt_metrics.items() if v is not None}
            metrics.append(frame_cnt_metrics)

        return metrics


"""Utilities for augmenting the data and generating batches of it.

Author: Lucy Tan

This module exports the Augmenter class for augmenting the training data with
image augmentation techniques, balancing the classes of the training data, and
generating sequences that yield batches of data. The sequences are designed
for use with the Keras APIs for model training, evaluating, and prediction.
"""
import glob
import math
import os
import shelve
from typing import Sequence, NamedTuple, Tuple

import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow_io as tfio

from utils import common

# Default parameters to use for augmenting if none are provided.
_DEFAULT_PARAMS = {
    'AUGMENTATION_ARGS': dict(width_shift_range=0.3,
            horizontal_flip=True,
            brightness_range=[0.3, 0.9],
            fill_mode='nearest'),
    'BASE_DATA_DIR': 'dzanga-bai-20210816T230919Z-001/dzanga-bai',
    'BASE_ELP_WAV_DATA_DIR': 'elp_data/wav_files',
    'OUTPUT_PATH': 'dzanga-bai-20210816T230919Z-001/dzanga-bai',
    'SHELF_PATH': 'backups/shelves/shelf-%s-%s-%d',
    'BATCH_SIZE': 16,
    'IMAGE_SIZE': 512,
    'SEED': 100,
    'TARGET_COL': 'agecat',
    'USE_WAV_FILES': False,
    'AUGMENT_WITH_BALANCED_CLASSES': False,
}

# Adapted from https://stackoverflow.com/questions/66410340/filter-tensorflow-dataset-by-id
def filter_elp_dataset(dataset, indices):
    keys_tensor = tf.constant(indices)
    vals_tensor = tf.ones_like(keys_tensor)  # Ones will be casted to True.

    table = tf.lookup.StaticHashTable(
        tf.lookup.KeyValueTensorInitializer(keys_tensor, vals_tensor),
        default_value=0)  # If index not in table, return 0.

    @tf.function
    def hash_table_filter(index, value):
        table_value = table.lookup(index)  # 1 if index in arr, else 0.
        index_in_arr =  tf.cast(table_value, tf.bool) # 1 -> True, 0 -> False
        return index_in_arr

    return dataset.enumerate().filter(hash_table_filter).map(lambda idx, value: value)

# Copied from https://www.tensorflow.org/tutorials/audio/transfer_learning_audio
@tf.function
def load_wav_16k_mono(filename):
    """ Load a WAV file, convert it to a float tensor, resample to 16 kHz single-channel audio. """
    file_contents = tf.io.read_file(filename)
    wav, sample_rate = tf.audio.decode_wav(
          file_contents,
          desired_channels=1)
    wav = tf.squeeze(wav, axis=-1)
    sample_rate = tf.cast(sample_rate, dtype=tf.int64)
    wav = tfio.audio.resample(wav, rate_in=sample_rate, rate_out=16000)
    return tf.reshape(wav, [-1])

class CrossValFoldSequence(NamedTuple):
    """Contains the sequences for a single cross validation fold.

    Each batch is a tuple of x_data, y_data, where x_data is spectrogram pixel
    values and y_data is 1-hot encoded target labels.

    train: The sequence used for the train data of this fold.
    val: The sequence used for the validation data of this fold.
    """
    train: tf.keras.utils.Sequence
    val: tf.keras.utils.Sequence

class DatasetSequence(tf.keras.utils.Sequence):

    def __init__(self, dataset, labels, label_mapping, shelf_path, length=None):
        self._dataset = dataset
        self.classes = labels
        self.class_indices = label_mapping
        if length is None:
            self._length = len(dataset)
        else:
            self._length = length
        self._shelf_path = shelf_path
        self._data = CompressedShelf(shelf_path)

    def __len__(self):
        return self._length

    def _prefetch_data(self):
        dataset_iterator = self._dataset.as_numpy_iterator()
        for i in range(self._length):
            self._data[str(i)] = next(dataset_iterator)
        self._data.sync()

    def __getitem__(self, index):
        str_index = str(index)
        if str_index not in self._data:
            print('Prefetching data')
            self._prefetch_data()
        return self._data[str_index]

    def delete_data(self):
        self._data.close()
        try:
            for f in glob.glob(f'{self._shelf_path}*'):
                os.remove(f)
                print('Deleted', f)
        except Exception as e:
            print('Failed to delete shelf')
            print(e)

    def on_epoch_end(self):
        pass

class CombinedSequence(tf.keras.utils.Sequence):
    """Combines sequences to produce data from all the given sequences.

    It is not guaranteed that each batch will have the same amount of data
    from each given sequence. Instead, if there are N sequences, then across
    each group of N batches, there will be batch_size elements from each
    sequence.

    It is designed to be used with the Keras APIs for model training,
    evaluating, and prediction. As a result, it needs to implement the
    following methods:
        __len__: indicates how many total elements are in the sequence
        __iter__: iterates through the sequence, generating each batch of data
        __next__: yields the next batch of data
    """
    def __init__(self, seed, batch_size, labels, label_mapping, *sequences):
        """Create a CombinedSequence with the seed, batch size, and sequences.

        The sequence is created to use in the __next__ method.
        """
        self._seed = seed
        self._batch_size = batch_size
        self._sequences = sequences
        assert batch_size % len(sequences) == 0, 'Batch size must be a multiple of the number of sequences to make each batch even'
        self._rng = np.random.default_rng(seed=self._seed)
        self._num_batches_per_sequence = self._get_num_batches_per_sequence()
        self._len = max(self._num_batches_per_sequence) * len(sequences)
        self.classes = []
        for _ in range(self._len):
            for i in range(len(sequences)):
                self.classes.extend([labels[i]] * (batch_size // len(sequences)))
        self.class_indices = label_mapping

    def __len__(self):
        return self._len

    def _get_num_batches_per_sequence(self):
        full_sequence_lengths = []
        for seq in self._sequences:
            i = len(seq) - 1
            while i > -1:
                if len(seq[i][0]) >= self._batch_size:
                    full_sequence_lengths.append(i + 1)
                    break
                i -= 1
            if i == -1:
                full_sequence_lengths.append(1)
        return full_sequence_lengths

    def _batches_per_sequence(self):
        shortest_length = len(min(self._sequences, key=len))
        while shortest_length > 0:
            all_are_valid = True
            for sequence in self._sequences:
                if len(sequence[shortest_length - 1][0]) < self._batch_size:
                    all_are_valid = False
                    break
            if all_are_valid:
                break
            shortest_length -= 1
        return shortest_length

    def _get_data_for_epoch(self):
        x_data = []
        y_data = []
        for sequence in self._sequences:
            for i in range(len(sequence)):
                cur_x_data, cur_y_data = sequence[i]
                # Don't use batch groups where a sequence had less than a full
                # batch. This would allow the training data to be unbalanced.
                if len(cur_x_data) != self._batch_size:
                    continue
                x_data.append(cur_x_data)
                y_data.append(cur_y_data)
        merged_data = np.concatenate(x_data), np.concatenate(y_data)
        data_len = len(merged_data[0])
        indices = np.arange(data_len)
        self._rng.shuffle(indices)
        return merged_data[0][indices], merged_data[1][indices]

    def __getitem__(self, index):
        sequence_batch_index = index // len(self._sequences)
        sequence_batches = [self._sequences[i][sequence_batch_index % self._num_batches_per_sequence[i]] for i in range(len(self._sequences))]
        batch_start_index = self._batch_size * (index % len(self._sequences))
        batch_end_index = batch_start_index + self._batch_size
        batch_indices = np.arange(batch_start_index, batch_end_index)
        x_data = []
        y_data = []
        for i in range(len(self._sequences)):
            cur_indices = batch_indices[batch_indices % len(self._sequences) == i] // len(self._sequences)
            x_data.extend(sequence_batches[i][0][cur_indices])
            y_data.extend(sequence_batches[i][1][cur_indices])
        indices = np.arange(len(x_data))
        return np.array(x_data)[indices], np.array(y_data)[indices]

    def on_epoch_end(self):
        for sequence in self._sequences:
            sequence.on_epoch_end()
        self._rng = np.random.default_rng(seed=self._seed)

class Augmenter:
    """A class to augment the training data and create sequences of batches.

    The public methods are:
        get_sequences: Returns the cross validation train and val sequences
            for the Dzanga Bai data given the indices (output by the
            CrossValidator).
        get_test_sequence: Returns the test sequence for the Dzanga Bai data
            given the indices (output by the TestSplitter).
    """

    def __init__(self, params=None):
        """Create a Augmenter with the given params.

        The params should include all of the following:
            AUGMENTATION_ARGS
            BASE_DATA_DIR: The path to the directory with the Dzanga Bai data.
            BATCH_SIZE: The number of data points to use in each batch.
            IMAGE_SIZE: The size in pixels for both height and width of the
                spectrogram when input to the model.
            SEED: The random seed to use for consistent image augmentation and
                data shuffling.
            TARGET_COL: The name of the column containing the target label.
        """
        self._params = _DEFAULT_PARAMS.copy()
        if params is not None:
            self._params.update(params)

    def _load_wav_for_map(self, filename, one_hot_label, label):
        return load_wav_16k_mono(filename), one_hot_label, label

    def _get_sequence_audio(self, df, filtered_df, shuffle, fold_index, is_train_fold):
        target_col = self._params['TARGET_COL']
        one_hot_labels = pd.get_dummies(filtered_df[target_col].astype(pd.CategoricalDtype(categories=sorted(list(df[target_col].unique())))))
        dataset = tf.data.Dataset.from_tensor_slices((filtered_df['wav_path'], one_hot_labels, np.argmax(one_hot_labels.to_numpy(), axis=1)))
        dataset = dataset.map(self._load_wav_for_map)
        max_length = dataset.reduce(np.int32(0), lambda cur_max, new_x: tf.math.maximum(cur_max, tf.shape(new_x[0])[0])).numpy()
        if shuffle:
            dataset = dataset.shuffle(buffer_size=len(filtered_df), seed=self._params['SEED'], reshuffle_each_iteration=False)
        dataset_labels = list(dataset.map(lambda x,y,label: label).as_numpy_iterator())
        label_mapping = {value: index for index, value in enumerate(one_hot_labels.columns)}
        dataset = dataset.map(lambda x,y,label: (tf.reshape(tf.pad(x, [[0, max_length - tf.shape(x)[0]]]), [max_length]),y))
        shelf_path = self._params['SHELF_PATH'] % ('dz', 'train' if is_train_fold else ('val' if fold_index is not None else 'test'), fold_index if fold_index is not None else 0)
        return DatasetSequence(dataset.batch(self._params['BATCH_SIZE']).prefetch(tf.data.AUTOTUNE), dataset_labels, label_mapping, shelf_path)

    def _get_sequence_image(self, df, filtered_df, shuffle, augment):
        target_col = self._params['TARGET_COL']
        image_size = self._params['IMAGE_SIZE']
        augmentation_args = self._params['AUGMENTATION_ARGS'] if augment else {}
        return tf.keras.preprocessing.image.ImageDataGenerator(
            **augmentation_args).flow_from_dataframe(
            dataframe=filtered_df,
            x_col='path',
            y_col=target_col,
            target_size=(image_size, image_size),
            color_mode='rgb',
            classes=sorted(list(df[target_col].unique())),
            class_mode='categorical',
            batch_size=self._params['BATCH_SIZE'],
            shuffle=shuffle,
            seed=self._params['SEED'],
            save_to_dir=None,
            save_prefix='',
            save_format='png',
            subset=None,
            interpolation='nearest',
            validate_filenames=True,)

    def _get_sequence(self, df, shuffle, augment, fold_index, is_train_fold, indices=None):
        filtered_df = df.iloc[indices] if indices is not None else df
        if self._params['USE_WAV_FILES']:
            return self._get_sequence_audio(df, filtered_df, shuffle, fold_index, is_train_fold)
        return self._get_sequence_image(df, filtered_df, augment, shuffle)

    def _get_elp_sequence(self, dataset, shuffle, indices, fold_index, is_train_fold):
        dataset = filter_elp_dataset(dataset, indices)
        max_length = dataset.reduce(np.int32(0), lambda cur_max, new_x: tf.math.maximum(cur_max, tf.shape(new_x)[0])).numpy()
        if shuffle:
            dataset = dataset.shuffle(buffer_size=len(indices), seed=self._params['SEED'], reshuffle_each_iteration=False)
        dataset = dataset.map(lambda x: tf.reshape(tf.pad(x, [[0, max_length - tf.shape(x)[0]]]), [max_length]))
        dataset = dataset.map(lambda x: (x, np.zeros(())))
        shelf_path = self._params['SHELF_PATH'] % ('elp', 'train' if is_train_fold else 'val', fold_index)
        return DatasetSequence(dataset.batch(self._params['BATCH_SIZE']).prefetch(tf.data.AUTOTUNE), None, None, shelf_path, math.ceil(len(indices) / self._params['BATCH_SIZE']))

    def _filter_indices(self, df, indices, value_to_accept):
        new_indices = [
            idx for idx in indices
            if df[self._params['TARGET_COL']].iloc[idx] == value_to_accept
        ]
        return np.array(new_indices)

    def _get_balanced_sequence(self, df, indices, fold_index):
        seed = self._params['SEED']
        target_col = self._params['TARGET_COL']
        sequences = []
        for target_val in sorted(df[target_col].unique()):
            filtered_indices = self._filter_indices(
                df, indices, target_val)
            sequences.append(self._get_sequence(
                df, shuffle=True, augment=True, fold_index=fold_index, is_train_fold=True, indices=filtered_indices))
        label_mapping = {value: index for index, value in enumerate(sorted(list(df[target_col].unique())))}
        return CombinedSequence(seed, self._params['BATCH_SIZE'], list(range(len(sequences))), label_mapping, *sequences)

    def get_test_sequence(self, test_indices):
        df = common.load_dz_data(self._params['BASE_DATA_DIR'])
        return self._get_sequence(df, shuffle=False, augment=False, fold_index=None, is_train_fold=False, indices=test_indices)

    def get_sequences(self, cross_val_indices, augment=True):
        """Get the sequences for the given indices.

        If augment is True, use image augmentation and class balancing for the
        training data.

        For each cross-validation fold, generate a sequence for the train and
        validation data.
        """
        df = common.load_dz_data(self._params['BASE_DATA_DIR'])
        target_col = self._params['TARGET_COL']
        sequences = []
        for i, fold_indices in enumerate(cross_val_indices):
            if augment and self._params['AUGMENT_WITH_BALANCED_CLASSES'] and not self._params['USE_WAV_FILES']:
                train_sequence = self._get_balanced_sequence(
                    df, fold_indices[0], i)
            else:
                train_sequence = self._get_sequence(
                    df, shuffle=augment, augment=augment, fold_index=i, is_train_fold=True, indices=fold_indices[0])
            val_sequence = self._get_sequence(
                df, shuffle=False, augment=False, fold_index=i, is_train_fold=False, indices=fold_indices[1])
            sequences.append(CrossValFoldSequence(
                train_sequence, val_sequence))
        return sequences

    def get_elp_sequences(self, cross_val_indices, shuffle=True):
        """Get the ELP sequences for the given indices.

        For each cross-validation fold, generate a sequence for the train and
        validation data.
        """
        dataset = common.load_elp_data(self._params['BASE_ELP_WAV_DATA_DIR'])
        sequences = []
        for i, fold_indices in enumerate(cross_val_indices):
            train_sequence = self._get_elp_sequence(
                    dataset, shuffle=shuffle, fold_index=i, is_train_fold=True, indices=fold_indices[0])
            val_sequence = self._get_elp_sequence(
                dataset, shuffle=False, fold_index=i, is_train_fold=False, indices=fold_indices[1])
            sequences.append(CrossValFoldSequence(
                train_sequence, val_sequence))
        return sequences

if __name__ == '__main__':
    # train_indices_filename_pattern = os.path.join(
    #     _DEFAULT_PARAMS['OUTPUT_PATH'], 'train_indices_%s.csv')
    # val_indices_filename_pattern = os.path.join(
    #     _DEFAULT_PARAMS['OUTPUT_PATH'], 'val_indices_%s.csv')
    # cross_val_indices = []
    # for i in range(5):
    #     with open(train_indices_filename_pattern % i, 'rt') as f:
    #         train_indices = np.array([int(index) for index in f.readlines()])
    #     with open(val_indices_filename_pattern % i, 'rt') as f:
    #         val_indices = np.array([int(index) for index in f.readlines()])
    #     cross_val_indices.append((train_indices,val_indices))
    # augmenter = Augmenter()
    # kfold_sequences = augmenter.get_sequences(cross_val_indices)
    # test_indices_filename = os.path.join(
    #     _DEFAULT_PARAMS['BASE_DATA_DIR'], 'test_indices.csv')
    # with open(test_indices_filename, 'rt') as f:
    #     test_indices = np.array([int(index) for index in f.readlines()])
    # test_sequence = augmenter.get_test_sequence(test_indices)
    # cross_val_data = [
    #     (next(iter(sequence.train)), next(iter(sequence.val)))
    #     for sequence in kfold_sequences
    # ]
    # #print(cross_val_data, next(iter(test_sequence)))
    # #print(test_sequence.classes, test_sequence.class_indices)
    # print(kfold_sequences[0].train.classes)
    # print(kfold_sequences[0].train.class_indices)
    # for i in range(len(kfold_sequences[0].train)):
    #     print(kfold_sequences[0].train[i][1])

    train_indices_filename_pattern = os.path.join(
        _DEFAULT_PARAMS['OUTPUT_PATH'], 'elp_train_indices_%s.csv')
    val_indices_filename_pattern = os.path.join(
        _DEFAULT_PARAMS['OUTPUT_PATH'], 'elp_val_indices_%s.csv')
    cross_val_indices = []
    for i in range(5):
        with open(train_indices_filename_pattern % i, 'rt') as f:
            train_indices = np.array([int(index) for index in f.readlines()])
        with open(val_indices_filename_pattern % i, 'rt') as f:
            val_indices = np.array([int(index) for index in f.readlines()])
        cross_val_indices.append((train_indices,val_indices))
    augmenter = Augmenter()
    kfold_sequences = augmenter.get_elp_sequences(cross_val_indices)
    cross_val_data = [
        (sequence.train[0], sequence.val[0])
        for sequence in kfold_sequences
    ]
    cross_val_len = [
        (len(sequence.train), len(sequence.val))
        for sequence in kfold_sequences
    ]
    print(cross_val_data, cross_val_len)


"""
* 이진 검색
정렬된 nums를 입력받아 이진 검색으로 target에 해당하는 인덱스를 찾아라.
Input: nums = [-1, 0, 3, 5, 9, 12], target = 9
Output: 4
"""
import bisect
from typing import List

class Solution:
    # 이진 검색을 사용하지 않는 index 풀이
    def search(self, nums: List[int], target: int) -> int:
        try:
            return nums.index(target)
        except ValueError:
            return -1

if __name__ == '__main__':
    nums = [-1, 0, 3, 5, 9, 12]
    target = 9
    print(Solution().search(nums, target))


#!/bin/python

import sys
import os
import requests
import time

from pprint import pprint as pp

from destiny.exceptions import *

class DestinyClient:

    HEADERS = {"X-API-Key": "PUT_API_KEY_HERE"}
    PLATFORM = {'XBOX': 1, 'PSN': 2}
    MODE = {'TOO' : 14}

    URL = 'http://www.bungie.net/Platform/Destiny/{0}'
    ENDPOINT = {
        # m_type, m_id, c_id, mode, page, count
        'ActivityHistory' : '/Stats/ActivityHistory/{0}/{1}/{2}/?mode={3}&page={4}&count={5}',
        # i_id
        'PostGameCarnageReport' : '/Stats/PostGameCarnageReport/{0}/',
        # m_type, d_name
        'GetMembershipIdByDisplayName' : '/{0}/Stats/GetMembershipIdByDisplayName/{1}/',
        # m_type, displayName
        'SearchDestinyPlayer': '/SearchDestinyPlayer/{0}/{1}/',
        # m_type, m_id
        'GetAccount' : '/{0}/Account/{1}/',
    }

    def __getattr__(self, method):
        def f(*args):
            url = DestinyClient.URL.format(DestinyClient.ENDPOINT[method].format(*args))
            st = time.time()
            tries = 0
            while tries < 5:
              tries += 1

              resp = requests.get(url, headers=DestinyClient.HEADERS).json()

              if resp['ErrorStatus'] == 'PerEndpointRequestThrottleExceeded':
                  time.sleep(1)
                  continue

              if resp['ErrorStatus'] == 'Success':
                  break

              raise_destiny_exception(resp['ErrorCode'])

            if tries == 5:
                raise Exception("Too many retries")

            end = time.time()

            # print('method: %s %s %0.2f %d %d %s %s' % (
            #       method, args, end-st,
            #       resp['ThrottleSeconds'], resp['ErrorCode'],
            #       resp['ErrorStatus'], resp['MessageData']))

            if resp['ErrorStatus'] == 'Success':
                return resp['Response']

            return None
        return f


"""
Neilson IQ Find avg, max time spend of charging per charging id and save to parquet.
"""
from pyspark.sql import SparkSession
import pyspark.sql.functions as F
from pyspark.sql.functions import round

class ChargePointsETLJob:
    input_path = 'data/input/electric-chargepoints-2017.csv'
    output_path = 'data/output/chargepoints-2017-analysis'
    dt_ts_format = 'yyyy-MM-dd HH:mm:ss'

    def __init__(self):
        self.spark_session = (SparkSession.builder
                                          .master("local[*]")
                                          .appName("ElectricChargePointsETLJob")
                                          .getOrCreate())

    def extract(self):
        raw_df = self.spark_session.read.option("header", True).option("inferSchema", True).csv(self.input_path).withColumn(
            "startTs", F.unix_timestamp(F.concat(F.col("StartDate"), F.lit(" "), F.col("StartTime")), self.dt_ts_format)).withColumn(
                "endTs", F.unix_timestamp(F.concat(F.col("EndDate"), F.lit(" "), F.col("EndTime")), self.dt_ts_format))
        return raw_df

    def transform(self, df):
        """
        df.withColumn('test', F.concat_ws(" ", "StartDate", "StartTime")).withColumn("test", F.unix_timestamp(F.col("test"), "yyyy-MM-dd HH:mm:ss")).show()
        """
        time_spent = df.withColumn("time_spent", F.round((F.col("endTs") - F.col("startTs"))/3600,2))
        ans = time_spent.groupBy("CPID").agg(
            F.round(F.max(F.col("time_spent")),2).alias("max_duration"),
            F.round(F.avg(F.col("time_spent")),2).alias("avg_duration")
        ).withColumnRenamed("CPID", "chargepoint_id")
        return ans
    def load(self, df):
        df.write.parquet(f"{self.output_path}")

    def run(self):
        self.load(self.transform(self.extract()))


"""
70592kb , 1052ms

"""

import sys

# sys.stdin = open('input_11660.txt')
from itertools import accumulate

input = sys.stdin.readline

N, M = map(int, input().split())

data = [list(map(int, input().split())) for _ in range(N)]

# 누적합 가로
for i in range(N):
    data[i] = list(accumulate(data[i]))
# 누적합 세로
for i in range(1, N):
    for j in range(N):
        data[i][j] += data[i - 1][j]
for _ in range(M):
    x1, y1, x2, y2 = map(lambda x: x - 1, map(int, input().split()))
    answer = data[x2][y2]
    if y1 != 0:
        answer -= data[x2][y1 - 1]

    if x1 != 0:
        answer -= data[x1 - 1][y2]
    if y1 !=0 and x1 !=0:
        answer += data[x1 - 1][y1 - 1]
    print(answer)


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import copy
import os
from argparse import ArgumentParser, Namespace

import mozlog

here = os.path.abspath(os.path.dirname(__file__))
try:
    from mozbuild.base import MachCommandConditions as conditions
    from mozbuild.base import MozbuildObject

    build_obj = MozbuildObject.from_environment(cwd=here)
except Exception:
    build_obj = None
    conditions = None

from mozperftest.metrics import get_layers as metrics_layers  # noqa
from mozperftest.system import get_layers as system_layers  # noqa
from mozperftest.test import get_layers as test_layers  # noqa
from mozperftest.utils import convert_day  # noqa

FLAVORS = "desktop-browser", "mobile-browser", "doc", "xpcshell", "webpagetest"

class Options:

    general_args = {
        "--flavor": {
            "choices": FLAVORS,
            "metavar": "{{{}}}".format(", ".join(FLAVORS)),
            "default": "desktop-browser",
            "help": "Only run tests of this flavor.",
        },
        "tests": {
            "nargs": "*",
            "metavar": "TEST",
            "default": [],
            "help": "Test to run. Can be a single test file or URL or a directory"
            " of tests (to run recursively). If omitted, the entire suite is run.",
        },
        "--test-iterations": {
            "type": int,
            "default": 1,
            "help": "Number of times the whole test is executed",
        },
        "--output": {
            "type": str,
            "default": "artifacts",
            "help": "Path to where data will be stored, defaults to a top-level "
            "`artifacts` folder.",
        },
        "--hooks": {
            "type": str,
            "default": None,
            "help": "Script containing hooks. Can be a path or a URL.",
        },
        "--verbose": {"action": "store_true", "default": False, "help": "Verbose mode"},
        "--push-to-try": {
            "action": "store_true",
            "default": False,
            "help": "Pushin the test to try",
        },
        "--try-platform": {
            "nargs": "*",
            "type": str,
            "default": "linux",
            "help": "Platform to use on try",
            "choices": ["g5", "pixel2", "linux", "mac", "win"],
        },
        "--on-try": {
            "action": "store_true",
            "default": False,
            "help": "Running the test on try",
        },
        "--test-date": {
            "type": convert_day,
            "default": "today",
            "help": "Used in multi-commit testing, it specifies the day to get test builds from. "
            "Must follow the format `YYYY.MM.DD` or be `today` or `yesterday`.",
        },
    }

    args = copy.deepcopy(general_args)

for layer in system_layers() + test_layers() + metrics_layers():
    if layer.activated:
        # add an option to deactivate it
        option_name = "--no-%s" % layer.name
        option_help = "Deactivates the %s layer" % layer.name
    else:
        option_name = "--%s" % layer.name
        option_help = "Activates the %s layer" % layer.name

    Options.args[option_name] = {
        "action": "store_true",
        "default": False,
        "help": option_help,
    }

    for option, value in layer.arguments.items():
        option = "--%s-%s" % (layer.name, option.replace("_", "-"))
        if option in Options.args:
            raise KeyError("%s option already defined!" % option)
        Options.args[option] = value

class PerftestArgumentParser(ArgumentParser):
    """%(prog)s [options] [test paths]"""

    def __init__(self, app=None, **kwargs):
        ArgumentParser.__init__(
            self, usage=self.__doc__, conflict_handler="resolve", **kwargs
        )
        # XXX see if this list will vary depending on the flavor & app
        self.oldcwd = os.getcwd()
        self.app = app
        if not self.app and build_obj:
            if conditions.is_android(build_obj):
                self.app = "android"
        if not self.app:
            self.app = "generic"
        for name, options in Options.args.items():
            self.add_argument(name, **options)

        mozlog.commandline.add_logging_group(self)
        self.set_by_user = []

    def parse_helper(self, args):
        for arg in args:
            arg_part = arg.partition("--")[-1].partition("-")
            layer_name = f"--{arg_part[0]}"
            layer_exists = arg_part[1] and layer_name in Options.args
            if layer_exists:
                args.append(layer_name)

    def get_user_args(self, args):
        # suppress args that were not provided by the user.
        res = {}
        for key, value in args.items():
            if key not in self.set_by_user:
                continue
            res[key] = value
        return res

    def _parse_known_args(self, arg_strings, namespace):
        # at this point, the namespace is filled with default values
        # defined in the args

        # let's parse what the user really gave us in the CLI
        # in a new namespace
        user_namespace, extras = super()._parse_known_args(arg_strings, Namespace())

        self.set_by_user = list([name for name, value in user_namespace._get_kwargs()])

        # we can now merge both
        for key, value in user_namespace._get_kwargs():
            setattr(namespace, key, value)

        return namespace, extras

    def parse_args(self, args=None, namespace=None):
        self.parse_helper(args)
        return super().parse_args(args, namespace)

    def parse_known_args(self, args=None, namespace=None):
        self.parse_helper(args)
        return super().parse_known_args(args, namespace)

class SideBySideOptions:
    args = [
        [
            ["-t", "--test-name"],
            {
                "type": str,
                "required": True,
                "dest": "test_name",
                "help": "The name of the test task to get videos from.",
            },
        ],
        [
            ["--new-test-name"],
            {
                "type": str,
                "default": None,
                "help": "The name of the test task to get videos from in the new revision.",
            },
        ],
        [
            ["--base-revision"],
            {
                "type": str,
                "required": True,
                "help": "The base revision to compare a new revision to.",
            },
        ],
        [
            ["--new-revision"],
            {
                "type": str,
                "required": True,
                "help": "The base revision to compare a new revision to.",
            },
        ],
        [
            ["--base-branch"],
            {
                "type": str,
                "default": "autoland",
                "help": "Branch to search for the base revision.",
            },
        ],
        [
            ["--new-branch"],
            {
                "type": str,
                "default": "autoland",
                "help": "Branch to search for the new revision.",
            },
        ],
        [
            ["--base-platform"],
            {
                "type": str,
                "required": True,
                "dest": "platform",
                "help": "Platform to return results for.",
            },
        ],
        [
            ["--new-platform"],
            {
                "type": str,
                "default": None,
                "help": "Platform to return results for in the new revision.",
            },
        ],
        [
            ["-o", "--overwrite"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, the downloaded task group data will be deleted before "
                + "it gets re-downloaded.",
            },
        ],
        [
            ["--cold"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, we'll only look at cold pageload tests.",
            },
        ],
        [
            ["--warm"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, we'll only look at warm pageload tests.",
            },
        ],
        [
            ["--most-similar"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, we'll search for a video pairing that is the most similar.",
            },
        ],
        [
            ["--search-crons"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, we will search for the tasks within the cron jobs as well. ",
            },
        ],
        [
            ["--skip-download"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, we won't try to download artifacts again and we'll "
                + "try using what already exists in the output folder.",
            },
        ],
        [
            ["--output"],
            {
                "type": str,
                "default": None,
                "help": "This is where the data will be saved. Defaults to CWD. "
                + "You can include a name for the file here, otherwise it will "
                + "default to side-by-side.mp4.",
            },
        ],
        [
            ["--metric"],
            {
                "type": str,
                "default": "speedindex",
                "help": "Metric to use for side-by-side comparison.",
            },
        ],
        [
            ["--vismetPath"],
            {
                "type": str,
                "default": False,
                "help": "Paths to visualmetrics.py for step chart generation.",
            },
        ],
        [
            ["--original"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, use the original videos in the side-by-side instead "
                + "of the postprocessed videos.",
            },
        ],
        [
            ["--skip-slow-gif"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, the slow-motion GIFs won't be produced.",
            },
        ],
    ]

class ChangeDetectorOptions:
    args = [
        # TODO: Move the common tool arguments to a common
        # argument class.
        [
            ["--task-name"],
            {
                "type": str,
                "nargs": "*",
                "default": [],
                "dest": "task_names",
                "help": "The full name of the test task to get data from e.g. "
                "test-android-hw-a51-11-0-aarch64-shippable-qr/opt-"
                "browsertime-tp6m-geckoview-sina-nofis.",
            },
        ],
        [
            ["-t", "--test-name"],
            {
                "type": str,
                "default": None,
                "dest": "test_name",
                "help": "The name of the test task to get data from e.g. "
                "browsertime-tp6m-geckoview-sina-nofis.",
            },
        ],
        [
            ["--platform"],
            {
                "type": str,
                "default": None,
                "help": "Platform to analyze e.g. "
                "test-android-hw-a51-11-0-aarch64-shippable-qr/opt.",
            },
        ],
        [
            ["--new-test-name"],
            {
                "type": str,
                "help": "The name of the test task to get data from in the "
                "base revision e.g. browsertime-tp6m-geckoview-sina-nofis.",
            },
        ],
        [
            ["--new-platform"],
            {
                "type": str,
                "help": "Platform to analyze in base revision e.g. "
                "test-android-hw-a51-11-0-aarch64-shippable-qr/opt.",
            },
        ],
        [
            ["--depth"],
            {
                "type": int,
                "default": None,
                "help": "This sets how the change detector should run. "
                "Default is None, which is a direct comparison between the "
                "revisions. -1 will autocompute the number of revisions to "
                "look at between the base, and new. Any other positive integer "
                "acts as a maximum number to look at.",
            },
        ],
        [
            ["--base-revision"],
            {
                "type": str,
                "required": True,
                "help": "The base revision to compare a new revision to.",
            },
        ],
        [
            ["--new-revision"],
            {
                "type": str,
                "required": True,
                "help": "The new revision to compare a base revision to.",
            },
        ],
        [
            ["--base-branch"],
            {
                "type": str,
                "default": "try",
                "help": "Branch to search for the base revision.",
            },
        ],
        [
            ["--new-branch"],
            {
                "type": str,
                "default": "try",
                "help": "Branch to search for the new revision.",
            },
        ],
        [
            ["--skip-download"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, we won't try to download artifacts again and we'll "
                + "try using what already exists in the output folder.",
            },
        ],
        [
            ["-o", "--overwrite"],
            {
                "action": "store_true",
                "default": False,
                "help": "If set, the downloaded task group data will be deleted before "
                + "it gets re-downloaded.",
            },
        ],
    ]

class ToolingOptions:
    args = {
        "side-by-side": SideBySideOptions.args,
        "change-detector": ChangeDetectorOptions.args,
    }

class PerftestToolsArgumentParser(ArgumentParser):
    """%(prog)s [options] [test paths]"""

    tool = None

    def __init__(self, *args, **kwargs):
        ArgumentParser.__init__(
            self, usage=self.__doc__, conflict_handler="resolve", **kwargs
        )

        if PerftestToolsArgumentParser.tool is None:
            raise SystemExit("No tool specified, cannot continue parsing")
        else:
            for name, options in ToolingOptions.args[PerftestToolsArgumentParser.tool]:
                self.add_argument(*name, **options)


import requests
from bs4 import BeautifulSoup
import openpyxl

wb = openpyxl.Workbook()
ws = wb.create_sheet('주식정보')

codes = [
    '005930',
    '000660',
    '035720'
]
i = 1
for code in codes:
    i += 1
    url = f'https://finance.naver.com/item/sise.naver?code={code}'
    response = requests.get(url)
    html = response.text

    soup = BeautifulSoup(html, 'html.parser')
    price = soup.select_one('#_nowVal').text
    price = price.replace(',', '')
    ws[f'B{i}'] = int(price)
wb.save(r'C:\python_crawling\C3\stock_data.xlsx')


import os
import django

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mrnd_summer_hackday.settings')
django.setup()

import csv
from iplapp.models import *

def import_matches(filename):
    seasons_data = set()
    matches_data = dict()

    with open(filename) as match_file:

        csv_reader = csv.DictReader(match_file)

        # reading the data
        for row in csv_reader:
            row['season'] = int(row['season'])
            seasons_data.add(row['season'])
            try:
                matches_data[row['season']].append(list(row.values()))
            except KeyError:
                matches_data[row['season']] = list()
                matches_data[row['season']].append(list(row.values()))

        # saving the data
        for season in seasons_data:
            Season(season=season).save()

        for season_year in matches_data:
            season = Season.objects.get(season=season_year)

            for id, _, city, date, team1, team2, toss_winner, toss_decision, result, dl_applied, \
                winner, win_by_runs, win_by_wickets, player_of_match, venue,\
                umpire1, umpire2, umpire3 in matches_data[season_year]:

                match = Match()

                match.id = id
                match.city = city
                match.date = date
                match.team1 = team1
                match.team2 = team2
                match.toss_winner = toss_winner
                match.toss_decision = toss_decision
                match.result = result
                match.dl_applied = dl_applied
                match.winner = winner
                match.win_by_runs = win_by_runs
                match.win_by_wickets = win_by_wickets
                match.player_of_match = player_of_match
                match.venue = venue
                match.umpire1 = umpire1
                match.umpire2 = umpire2
                match.umpire3 = umpire3

                match.season = season

                match.save()
                print('Matches success')

def import_deliveries(filename):

    with open(filename) as delivery_file:

        csv_reader = csv.DictReader(delivery_file)

        # reading data
        for row in csv_reader:
            delivery = Delivery()

            delivery.inning = int(row['inning'])
            delivery.batting_team = row['batting_team']
            delivery.bowling_team = row['bowling_team']
            delivery.over = row['over']
            delivery.ball = row['ball']
            delivery.batsman = row['batsman']
            delivery.non_striker = row['non_striker']
            delivery.bowler = row['bowler']
            delivery.is_super_over = row['is_super_over']
            delivery.wide_runs = row['wide_runs']
            delivery.bye_runs = row['bye_runs']
            delivery.legbye_runs = row['legbye_runs']
            delivery.noball_runs = row['noball_runs']
            delivery.penalty_runs = row['penalty_runs']
            delivery.batsman_runs = row['batsman_runs']
            delivery.extra_runs = row['extra_runs']
            delivery.total_runs = row['total_runs']
            delivery.player_dismissed = row['player_dismissed']
            delivery.dismissal_kind = row['dismissal_kind']
            delivery.fielder = row['fielder']

            delivery.match = Match.objects.get(id=row['match_id'])

            delivery.save()
            print('Deliveries success')

if __name__ == '__main__':
    import_matches('matches.csv')
    import_deliveries('deliveries.csv')
    pass


import pickle
import json
from .utils import *
from sklearn.linear_model import LogisticRegression

def run_experiments(input_path, few_shot, use_dist, balanced, output_path):
    # initiate.
    with open(input_path, "rb") as fr:
        all_data = pickle.load(fr)

    lr = LogisticRegression(C=0.1, max_iter=200000)
    accuracies_all_domains = {}
    data_size = 2000

    for source in all_data:
        for target in all_data:
            train_name = source[2] + "_to_" + target[2]
            print("\n", train_name)

            accuracies_all_domains[train_name] = run_gradual_train_groups(
                x_source_raw=source[0], y_source_raw=source[1],
                x_target_raw=target[0], y_target_raw=target[1],
                base_model=lr, data_size=data_size, top_n=100,
                balanced=balanced, use_dist=use_dist, few_shot=few_shot,
            )

    print(accuracies_all_domains)
    with open(output_path, "w") as outfile:
        json.dump(accuracies_all_domains, outfile, indent=4)

def run_check_version():
    # initiate.
    with open("../data/amazon_reviews/amazon_4.pickle", "rb") as fr:
        all_data = pickle.load(fr)

    lr = LogisticRegression(C=0.1, max_iter=200000)
    accuracies_all_domains = {}
    data_size = 2000

    for source in all_data:
        for target in all_data:
            train_name = source[2] + "_to_" + target[2]
            print("\n", train_name)

            accuracies_all_domains[train_name] = S2T_p4_adj_blc(
                train_features=source[0], train_labels=source[1],
                test_features=target[0], test_labels=target[1],
                model=lr, num_i=100,
            )

    print(accuracies_all_domains)
    with open("../outputs/accuracies_ti_amazon_S2T_p4_adj_blc_c0.1.json", "w") as outfile:
        json.dump(accuracies_all_domains, outfile, indent=4)

if __name__ == '__main__':
    pass

    # run balanced conf and few labels.
    amazon_data_path = "../data/amazon_reviews/amazon_4.pickle"

    # Distance unbalanced.
    run_experiments(input_path=amazon_data_path,
                    few_shot=None,
                    use_dist=True,
                    balanced=False,
                    output_path="../outputs/accuracies_ti_amazon_dist_unblc_c0.1.json")

    # Confidence unbalanced.
    run_experiments(input_path=amazon_data_path,
                    few_shot=None,
                    use_dist=False,
                    balanced=False,
                    output_path="../outputs/accuracies_ti_amazon_conf_unblc_c0.1.json")

    # Distance balanced.
    run_experiments(input_path=amazon_data_path,
                    few_shot=None,
                    use_dist=True,
                    balanced=True,
                    output_path="../outputs/accuracies_ti_amazon_dist_blc_c0.1.json")

    # Confidence balanced.
    run_experiments(input_path=amazon_data_path,
                    few_shot=None,
                    use_dist=False,
                    balanced=True,
                    output_path="../outputs/accuracies_ti_amazon_conf_blc_c0.1.json")

    # Few shots on confidence balanced (least).
    run_experiments(input_path=amazon_data_path,
                    few_shot="least",
                    use_dist=False,
                    balanced=True,
                    output_path="../outputs/accuracies_ti_amazon_conf_blc_c0.1_fs_least.json")

    # Few shots on confidence balanced (random).
    run_experiments(input_path=amazon_data_path,
                    few_shot="random",
                    use_dist=False,
                    balanced=True,
                    output_path="../outputs/accuracies_ti_amazon_conf_blc_c0.1_fs_random.json")


from django.shortcuts import render

# Create your views here.
from django.contrib.auth.decorators import login_required
from datetime import datetime
from django.shortcuts import render, redirect
from django.contrib.auth.models import User
from app.models import PurchaseOrder, PurchaseOrderProduct, Staff
from django.http import HttpRequest

@login_required
def arviewPo(request):
    po_list = PurchaseOrder.objects.filter(selected=True).only('purchaseOrderID', 'staffID', 'poStatus')

    context = {
        'title':'View Purchase Order',
        'year':datetime.now().year,
        'po_list': po_list,
    }
    return render(request, 'arPurchaseOrder/viewPo.html', context)

def arselectPo(request):
    data = PurchaseOrder.objects.filter()
    po = None

    for i in data:
        if i.purchaseOrderID in request.POST:
            po = PurchaseOrder.objects.filter(purchaseOrderID=i.purchaseOrderID)
            products = PurchaseOrderProduct.objects.filter(purchaseOrderID=i.purchaseOrderID, productPurchased=True)
            qtyProvided = po.get().qtyProvided
            staffID = po.get().staffID
            vendorID = po.get().vendorID
            poStatus = po.get().poStatus
            totalPrice = po.get().totalPrice
            print(poStatus)

    quo = po.get().quotationID

    context = {
        'po': po.get(), 'quo': quo,
        'products': products,
        'qtyProvided': qtyProvided,
        'staffID': staffID, 'vendorID':vendorID,
        'poStatus':poStatus, 'totalPrice': totalPrice,
    }

    return render(request,'arPurchaseOrder/selectPo.html', context)

def arapprovePo(request):
    #DATA BEFORE
    # dataP = PurchaseOrder.objects.filter(poStatus__in=['Pending']).values()
    # dataAp = PurchaseOrder.objects.filter(poStatus__in=['Approved']).values()
    # print(dataP)
    # print(dataAp)

    #GET SELECTED PO
    currentPo = PurchaseOrder.objects.filter(purchaseOrderID=request.POST.get("purchaseOrder"))
    print(currentPo)
    currentPo.update(poStatus='Approved')

    #DATA AFTER
    # print(dataP)
    # print(dataAp)

    context = {
        'currentPo':currentPo
    }
    return render(request, 'arPurchaseOrder/messagePo.html', context)

def arrejectPo(request):
    #DATA BEFORE
    # dataP = PurchaseOrder.objects.filter(poStatus__in=['Pending']).values()
    # dataAp = PurchaseOrder.objects.filter(poStatus__in=['Approved']).values()
    # print(dataP)
    # print(dataAp)

    #GET SELECTED PO
    currentPo = PurchaseOrder.objects.filter(purchaseOrderID=request.POST.get("purchaseOrder"))
    print(currentPo)
    currentPo.update(poStatus='Rejected')

    #DATA AFTER
    # print(dataP)
    # print(dataAp)

    context = {
        'currentPo':currentPo
    }
    return render(request, 'arPurchaseOrder/messagePo.html', context)

##############################################################################################
def arsearchPo(request):
    if request.method == "POST":
        searched = request.POST['searched']
        purchaseOrderID = PurchaseOrder.objects.filter(purchaseOrderID__contains=searched)

        context = {
                'searched': searched,
                'purchaseOrderID' : purchaseOrderID,
            }

        return render(request,'arPurchaseOrder/searchPo.html',context)
    else:
        return render(request,'arPurchaseOrder/searchPo.html',context)

def arbacktohome(request):
    """Renders the home page."""
    assert isinstance(request, HttpRequest)
    if request.user.is_authenticated:
        return(redirect('/menu'))
    else:
        return render(
            request,
            'app/menu.html',
            {
                'title':'View Item',
                'year': datetime.now().year,
            }
        )
def arbacktolist(request):
    """Renders the home page."""
    assert isinstance(request, HttpRequest)
    if request.user.is_authenticated:
        return(redirect('/arviewPo/'))
    return render(
            request,
            'arPurchaseOrder/viewPo.html',
            {
                'title':'Purchase Order List',
                'year': datetime.now().year,
            }
        )


import random
import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.metrics import rand_score
from sklearn.metrics import pairwise_distances
from sklearn.metrics.cluster import adjusted_rand_score

class KmeansPlus:
    def __init__(self, numOfClusters):
        self.__vectors = None
        self.__centroids = None
        self.__distances = None
        self.__K = numOfClusters
        self.__weightedDistances = None

    def __reFit(self, data):
        """ set the class variables to the new data """

        self.__centroids = []
        self.__vectors = data
        self.__distances = [[np.inf, 0] for i in range(len(self.__vectors))]
        self.__weightedDistances = [1 for i in enumerate(self.__vectors)]

    def __placeClusterCentroids(self):
        """ create a randome center using weights from all the data points """

        return random.choices(self.__vectors, weights=(self.__weightedDistances), k = 1)

    def __computeDistance(self, dataPoint, cent):
        """ check the distance between all data points to the last center added
            and insert the distance squared into the weights """

        distances = pairwise_distances(dataPoint, self.__centroids[-1], metric = 'euclidean')

        for i, dis in enumerate(distances):
            if self.__distances[i][0] > dis:
                self.__distances[i] = [dis, cent]
                self.__weightedDistances[i] = dis ** 2

    def accuracy(self, trueLabels):
        """ return the accuracy of the model in the form of (ri score, ari score) """

        labels = [item[1] for item in self.__distances]
        return rand_score(trueLabels, labels), adjusted_rand_score(trueLabels, labels)

    def plot(self):
        """ plot the centeres and all the data points given """

        cent = np.array(self.__centroids)
        labels = [item[1] for item in self.__distances]
        sklearn_pca = PCA(n_components = 2)
        t_labels = sklearn_pca.fit_transform(self.__vectors)
        plt.scatter(t_labels[:, 0], t_labels[:, 1], c=labels, s=50, cmap='viridis')
        plt.scatter(cent[0][:, 0], cent[0][:, 1],c='black', s=300, alpha=0.6)
        plt.show()

    def fit(self, data):
        """ create all centeres over k iterations and improve the centeres locations """

        self.__reFit(data)

        for i in range(self.__K):
            self.__centroids.append(self.__placeClusterCentroids())
            self.__computeDistance(self.__vectors, i)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019-06-04 14:55
# @Author  : zhangzhen
# @Site    :
# @File    : svm_text.py
# @Software: PyCharm
import logging
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn import neighbors
from matplotlib import pyplot as plt
import numpy as np
from data.news_classification import read_news_corpus
from text_classification import corpus_filter

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

train_corpus, train_labels, test_corpus, test_labels, dev_corpus, dev_labels = read_news_corpus()

labels = ['discovery', 'story', 'essay']
train_corpus, train_labels, test_corpus, test_labels, dev_corpus, dev_labels = corpus_filter(train_corpus, train_labels,
                                                                                test_corpus, test_labels,
                                                                                dev_corpus, dev_labels,
                                                                                labels=labels)

X_train = [" ".join(dat) for dat in train_corpus]
X_test = [" ".join(dat) for dat in test_corpus]
X_dev = [" ".join(dat) for dat in dev_corpus]

tfidf_model = TfidfVectorizer(min_df=0.002, max_df=0.5, token_pattern=r"(?u)\b\w\w+\b").fit(X_train)

# 得到tf-idf矩阵，稀疏矩阵表示法
X_train = tfidf_model.transform(X_train).todense()
X_test = tfidf_model.transform(X_test).todense()
X_dev = tfidf_model.transform(X_dev).todense()

y_train = train_labels
y_test = test_labels
y_dev = dev_labels

print("Train shape:", X_train.shape, "\nTest shape:", X_test.shape, "\nDev shape:", X_dev.shape)
# 词语与列的对应关系
print(tfidf_model.vocabulary_)

print("Train shape:", X_train.shape, "\nTest shape:", X_test.shape, "\nDev shape:", X_dev.shape)
# 词语与列的对应关系
print(tfidf_model.vocabulary_)

def train():
    clf = neighbors.KNeighborsClassifier()
    clf = clf.fit(X_train, y_train)
    y_predict = clf.predict(X_test)  # 对参数进行预测
    y_dev_predict = clf.predict(X_dev)  # 对参数进行预测

    # 获取结果报告
    print()
    print('The Test Accuracy of DT is:', clf.score(X_test, y_test))
    print(classification_report(y_test, y_predict, target_names=labels))
    print()
    print('The Dev Accuracy of DT is:', clf.score(X_dev, y_dev))
    print(classification_report(y_dev, y_dev_predict, target_names=labels))

def neighbors_search(nns=40):
    ns = np.arange(1, nns)
    training_scores = []
    testing_scores = []
    for n in ns:
        clf = neighbors.KNeighborsClassifier(n_neighbors=n, algorithm="ball_tree")
        clf = clf.fit(X_train, y_train)
        train_score = clf.score(X_train, y_train)
        test_score = clf.score(X_test, y_test)

        logging.info("neighbors: {} , train score: {}, test score: {}".format(n, train_score, test_score))
        training_scores.append(train_score)
        testing_scores.append(test_score)

    # 绘图
    fig = plt.figure()
    ax = fig.add_subplot(1, 1, 1)
    ax.plot(ns, training_scores, label="traing score", marker='o')
    ax.plot(ns, testing_scores, label="testing score", marker='*')
    ax.set_xlabel("neighbors")
    ax.set_ylabel("score")
    ax.set_title("KNN Classification")
    ax.legend(framealpha=0.5, loc='best')
    plt.show()

if __name__ == '__main__':
    # train()
    neighbors_search()