import numpy as np
import matplotlib.pyplot as plt
from matplotlib import gridspec
import matplotlib.lines as mlines
import matplotlib.patches as mpatches
import networkx as nx

# intra-package python imports
from .ratelaws import RateLaws

class Graph:
    """
    Class provides topological view of an individual network.

    Attributes:

        output_node (int) - index of output node

        nodes (np.ndarray) - vector of node indices

        reactions (list) - list of reaction objects

        stoichiometry (np.ndarray) - stoichiometric coefficients, (N, M)

        N (int) - number of nodes

        M (int) - number of reactions

        I (int) - number of input channels

        node_key (dict) - maps state dimension (key) to unique node id (value)

        edge_list (list) - edges defined as a (from, to, edge_dict) tuple

        up_edges (dict) - up-regulating edges in which keys are (from, to) tuples, values are edge weights

        down_edges (dict) - down-regulating edges in which keys are (from, to) tuples, values are edge weights

        graph (Networkx MultiDiGraph)

    """

    def __init__(self, network):
        """
        Inherits a mutable network then compiles an edge list and creates a networkx graph object.

        Args:

            network (MutableNetwork)

        """

        # assign poperties
        self.output_node = network.output_node
        self.nodes = network.nodes
        self.reactions = network.reactions
        self.stoichiometry = network.stoichiometry
        self.node_key = network.node_key

        # define system size
        self.N = self.nodes.size
        self.M = len(self.reactions)
        self.I = network.input_size

        # compile edge lists
        self.edge_list = self.get_edges()
        self.up_edges, self.down_edges = self.parse_edges()

        # create graph object
        self.graph = self.create_graph()

        # create node color dictionary
        self.node_colors = {
            'input': (206/256, 111/256, 111/256), # red
            'output': (138/256, 201/256, 228/256), # blue
            'reactive': (138/256, 201/256, 228/256)} # blue

    def get_direct_dependents(self, parent=None, input_dim=0):
        """
        Find all nodes directly dependent upon a single node. Node B is dependent upon node A if B appears within the stoichiometry
        vector of a reaction whose propensity includes A.

        Parameters:
            parent (int or None) - node whose dependents are desired, if None return nodes dependent on input(s)
            input_dim (int) - dimension of input vector used if parent is None

        Return:
            upregulated (set) - dependent nodes whose levels are increased by the parent node
            downregulated (set) - dependent nodes whose levels are decreased by the parent node
        """

        # initialize up/down regulated node lists
        upregulated, downregulated = set(), set()

        # iterate across all reactions
        for rxn in self.reactions:

            # initialize as non-dependent
            rxn_is_proportionally_dependent, rxn_is_inversely_dependent = False, False

            # check if reaction is input dependent
            if parent is None and rxn.input_dependence[input_dim] != 0:
                rxn_is_proportionally_dependent = True

            # check if parent node appears within reaction's propensity function
            elif parent is not None and parent in [self.node_key[node] for node, coeff in enumerate(rxn.propensity) if coeff != 0]:
                rxn_is_proportionally_dependent = True

            # if reaction is proportionally dependent upon the parent node, append all dependents to up/down regulated node lists
            if rxn_is_proportionally_dependent is True:
                upregulated = upregulated.union(set([self.node_key[dependent] for dependent, coeff in enumerate(rxn.stoichiometry) if coeff > 0]))
                downregulated = downregulated.union(set([self.node_key[dependent] for dependent, coeff in enumerate(rxn.stoichiometry) if coeff < 0]))

            # for enzymatic reactions, check all repressors
            if type(rxn) == Hill:
                for repressor in rxn.repressors:

                    # check if repressor is input dependent
                    if parent is None and repressor.input_dependence[input_dim] != 0:
                        rxn_is_inversely_dependent = True

                    # check if parent node appears within repressor's propensity function
                    elif parent is not None and parent in [self.node_key[node] for node, coeff in enumerate(repressor.propensity) if coeff != 0]:
                        rxn_is_inversely_dependent = True

            # if reaction is proportionally dependent upon the parent node, append all dependents to up/down regulated node lists
            if rxn_is_inversely_dependent is True:
                upregulated = upregulated.union(set([self.node_key[dependent] for dependent, coeff in enumerate(rxn.stoichiometry) if coeff < 0]))
                downregulated = downregulated.union(set([self.node_key[dependent] for dependent, coeff in enumerate(rxn.stoichiometry) if coeff > 0]))

        return upregulated, downregulated

    def get_edges(self):
        """
        Generates upregulating and downregulating edge sets for network visualization.

        Returns:
            edges (list) - each entry is a tuple containing (from, to, {edge_prop})
        """
        # initialize edge list
        edges = []

        # add input dependencies to edge list
        for input_dim in range(self.I):
            up, down = self.get_direct_dependents(parent=None, input_dim=input_dim)
            input_node = 'IN_' + str(input_dim)
            for node in up:
                edges.append((input_node, node, {'weight': 1}))
            for node in down:
                edges.append((input_node, node, {'weight': -1}))

        # add edges for all node-node dependencies
        for parent in self.nodes:
            up, down = self.get_direct_dependents(parent)
            for node in up:
                edges.append((parent, node, {'weight': 1}))
            for node in down:
                edges.append((parent, node, {'weight': -1}))

        return edges

    def parse_edges(self):
        """
        Sorts edge_list into up-regulating and down-regulating interactions.

        Returns:
            up_edges (dict) - keys are (from, to) tuples, values are edge weights
            down_edges (dict) - keys are (from, to) tuples, values are edge weights
        """

        up_edges, down_edges = {}, {}
        for edge in self.edge_list:
            if edge[2]['weight'] > 0:
                up_edges[(edge[0], edge[1])] = edge[2]['weight']
            else:
                down_edges[(edge[0], edge[1])] = edge[2]['weight']

        return up_edges, down_edges

    def get_degree_distributions(self):
        """
        Obtains in- and out- degree distributions for network excluding input node.

        Returns:
            in_degrees (np.ndarray) - distribution of in-degrees
            out_degrees (np.ndarray) - distribution of out-degrees
        """

        in_degrees = np.array([degree for node, degree in self.graph.in_degree_iter() if node != None])
        out_degrees = np.array([degree for node, degree in self.graph.out_degree_iter() if node != None])
        return in_degrees, out_degrees

    def create_graph(self):
        """
        Generates Networkx object of network topology.

        Returns:
            graph (Networkx MultiDiGraph)
        """

        # if network has no edges, abort
        if len(self.edge_list) == 0:
            print('Network has no edges.')

        # create directed graph with multiple parallel edges
        graph = nx.MultiDiGraph()

        # add nodes
        graph.add_nodes_from(['IN_'+str(input_dim) for input_dim in range(self.I)], node_type='input')
        if self.output_node is not None:
            graph.add_node(self.output_node, node_type='output')
        graph.add_nodes_from([node for node in self.nodes if node != self.output_node], node_type='reactive')

        # add edges
        for edge in self.edge_list:
            graph.add_edge(edge[0], edge[1], weight=edge[2]['weight'])

        return graph

    def add_legend_to_network(self, ax, edge_alpha=0.5):
        """
        Adds legend to network visualization.

        Parameters:
            ax (axes object) - axis for legend
        """

        # create patch artists for legend
        legend_patches = []
        for node_type, color in sorted(self.node_colors.items()):
            if node_type != 'input':
                legend_patches.append(mpatches.Patch(color=color, label=node_type))

        # add legend
        upreg_line = mlines.Line2D([], [], color='g', linewidth=5, label='upregulation', alpha=edge_alpha)
        downreg_line = mlines.Line2D([], [], color='r', linewidth=5, label='downregulation', alpha=edge_alpha)
        _ = ax.legend(loc=10, handles=legend_patches+[upreg_line, downreg_line], ncol=2, prop={'size': 16})

    def visualize_graph(self, graph_layout='shell', ax=None, fig_size=(5, 4),
                        label_edges=False, show_legend=False, title=None, label_nodes=True,
                        node_size=2000, node_text_size=10, edge_width=3,
                        ):
        """
        Visualize network topology.

        Parameters:
            graph_layout (string) - method used to arrange network nodes in space
            label_edges (bool) - if True, add edge weight labels
            fig_size (tuple) - figure dimensions in inches
            show_legend (bool) - if True, add legend
            ax (axes object) - axis on which network is drawn
            title (str) - title to add to plot
        """

        if self.graph is None:
            print('No network object has been created.')
            return

        # display options
        node_alpha = 1
        edge_alpha = 0.5
        edge_text_pos = 0.4

        # create figure for visualization
        if ax is None:
            fig = plt.figure()
            fig.set_size_inches(*fig_size)
            ax = plt.gca()
        _ = ax.axis('off')

        # select graph layout scheme
        if graph_layout == 'spring':
            pos = nx.spring_layout(self.graph)
        elif graph_layout == 'spectral':
            pos = nx.spectral_layout(self.graph)
        elif graph_layout == 'random':
            pos = nx.random_layout(self.graph)
        elif graph_layout == 'circular':
            pos = nx.circular_layout(self.graph)
        elif graph_layout == 'FR':
            pos = nx.fruchterman_reingold_layout(self.graph)
        else:
            pos = nx.shell_layout(self.graph)

        # draw nodes)
        for node_type, color in self.node_colors.items():

            # get all nodes of corresponding type
            node_list = [node for node, attr in self.graph.node.items() if attr['node_type']==node_type]

            # networkx interprets 3 nodes as color mappable so may need to duplicate nodes if count=3
            if len(node_list) == 3:
                node_list *= 2

            # add nodes to graph
            nx.draw_networkx_nodes(self.graph, pos, ax=ax, nodelist=node_list, node_color=color, node_size=node_size, alpha=node_alpha)

        # add node labels
        node_labels = {'IN_'+str(input_dim): 'IN_'+str(input_dim) for input_dim in range(self.I)}
        for node in self.nodes:
            if node == self.output_node:
                node_labels[node] = 'output'
            else:
                node_labels[node] = str(node)
        if label_nodes is True:
            nx.draw_networkx_labels(self.graph, pos, ax=ax, labels=node_labels, font_size=node_text_size, fontweight='bold', color='k', ha='center')

        # add edges (green for upregulating, red for downregulating)
        nx.draw_networkx_edges(self.graph, pos, ax=ax, edgelist=self.up_edges.keys(), width=edge_width, alpha=edge_alpha, edge_color='g')
        nx.draw_networkx_edges(self.graph, pos, ax=ax, edgelist=self.down_edges.keys(), width=edge_width, alpha=edge_alpha, edge_color='r')

        # add edge labels
        if label_edges is True:
            nx.draw_networkx_edge_labels(self.graph, pos, ax=ax, edge_labels=self.up_edges, label_pos=edge_text_pos, font_size=10, fontweight='bold')
            nx.draw_networkx_edge_labels(self.graph, pos, ax=ax, edge_labels=self.down_edges, label_pos=edge_text_pos, font_size=10, fontweight='bold')

        # add legend
        if show_legend:
            fig = plt.gcf()
            fig.axes[0].change_geometry(2, 1, 1)
            gs = gridspec.GridSpec(2, 1, height_ratios=[5, 1])
            ax.set_position(gs[0].get_position(fig))
            ax.set_subplotspec(gs[0])
            ax_legend = fig.add_subplot(gs[1])
            _, _ = ax.axis('off'), ax_legend.axis('off')
            self.add_legend_to_network(ax_legend)

        if title is not None:
            ax.set_title(title, fontsize=18)

        plt.tight_layout()

        return None


"""Utilities used in the communication in the transport layer."""

import hashlib
import logging
import math
import os

from osp.core.session.transport.transport_utils import check_hash

logger = logging.getLogger(__name__)
BLOCK_SIZE = 4096
LEN_FILES_HEADER = [5]  # num_blocks

def decode_header(bytestring, lengths):
    """Decode the header given as a string of bytes.

    Args:
        bytestring (bytes): The encoded header
        lengths (List[int]): The number of bytes for the individual components

    Yields:
        Union[int, str]: Interpret all elements that have a corresponding
                         length as int. If there leftover bytes decode them
                         using utf-8.
    """
    i = 0
    for length in lengths:
        if i + length > len(bytestring):
            raise IndexError("Length mismatch in header")
        yield int.from_bytes(bytestring[i : i + length], byteorder="big")
        i += length
    if len(bytestring) > i:
        yield bytestring[i:].decode("utf-8")

def encode_header(elements, lengths):
    """Encode the header to single array of bytes.

    Args:
        elements (Union[int, str]): The elements to encode. All but the last
                                    must be int. The last one can be str.
        lengths ([type]): The number of bytes in the encoded header.

    Raises:
        ValueError: elements and lengths mismatch in number of elements
        NotImplementedError: Invalid datatype in elements.

    Returns:
        bytes: The encoded header
    """
    r = b""
    if len(lengths) > len(elements) or len(elements) > len(lengths) + 1:
        raise ValueError
    for element, length in zip(elements, lengths):
        if not isinstance(element, int):
            raise NotImplementedError
        r += element.to_bytes(length=length, byteorder="big")
    if len(elements) > len(lengths):
        if not isinstance(elements[-1], str):
            raise NotImplementedError("Invalid type of %s" % elements[-1])
        r += elements[-1].encode("utf-8")
    return r

def split_message(msg, block_size=BLOCK_SIZE):
    """Split the message to send in small blocks.

    Args:
        msg (str): The message to send.
        block_size (int, optional): The size of the blocks.
                                    Defaults to BLOCK_SIZE.

    Returns:
        int, Generator: Number of blocks, Generator over blocks.
    """
    msg = msg.encode("utf-8")
    num_blocks = int(math.ceil(len(msg) / block_size))

    def gen(msg, num_blocks, block_size):
        for i in range(num_blocks):
            logger.debug(
                "Sending message block %s of %s" % (i + 1, num_blocks)
            )
            yield msg[i * block_size : (i + 1) * block_size]
        logger.debug("Done")

    return num_blocks, gen(msg, num_blocks, block_size)

async def join_message(websocket, num_blocks):
    """Get the message that was decomposed in different blocks.

    Args:
        websocket (websocket): wecksocket object to receive the objects.
        num_blocks (int): The number of blocks that belong to the message.

    Returns:
        str: The data, decoded using utf-8.
    """
    data = b""
    for i in range(num_blocks):
        logger.debug("Receiving message block %s of %s" % (i + 1, num_blocks))
        data += await websocket.recv()
    logger.debug("Done")
    data = data.decode("utf-8")
    return data

def filter_files(files, file_hashes):
    """Remove the files the receiver already has.

    Args:
        files (List[path]): A list of paths to send.

    Yields:
        List[str]: The files to send
    """
    result = list()
    for file in files:
        if not os.path.exists(file):
            logger.warning("Cannot send %s, because it does not exist" % file)
            continue
        if check_hash(file, file_hashes):
            logger.debug(
                "Skip sending file %s, "
                "receiver already has a copy of it." % file
            )
            continue
        result.append(file)
    return result

def encode_files(files):
    """Encode the files to be sent to over the networks.

    Will send file in several blocks.

    Args:
        files (List[path]): A list of paths to send.

    Yields:
        bytes: The bytes of the file
    """
    logger.debug("Will send %s files" % len(files))
    for i, file in enumerate(files):
        filename = os.path.basename(file)
        num_blocks = int(math.ceil(os.path.getsize(file) / BLOCK_SIZE))
        logger.debug(
            "Send file %s (%s of %s) with %s block(s) of %s bytes"
            % (file, i + 1, len(files), num_blocks, BLOCK_SIZE)
        )
        yield encode_header([num_blocks, filename], LEN_FILES_HEADER)

        # send the file contents
        with open(file, "rb") as f:
            for i, block in enumerate(iter(lambda: f.read(BLOCK_SIZE), b"")):
                logger.debug("Send file block %s of %s" % (i + 1, num_blocks))
                yield block
            logger.debug("Done")

async def receive_files(num_files, websocket, directory, file_hashes=None):
    """Will receive and store the files sent to the websocket.

    Args:
        num_files (int): The number of files to load.
        websocket (websocket): The websocket to load the files from.
        directory (path): The location to store the files.
        file_hashes(dict[str, str]): Hashes of files of already received files.
    """
    if file_hashes is None:
        file_hashes = dict()
    for i in range(num_files):
        logger.debug("Load file %s of %s" % (i + 1, num_files))
        num_blocks, filename = decode_header(
            await websocket.recv(), LEN_FILES_HEADER
        )
        filename = os.path.basename(filename)
        file_hashes[filename] = hashlib.sha256()
        file_path = os.path.join(directory, filename)
        logger.debug(
            "Storing file %s with %s blocks." % (file_path, num_blocks)
        )
        with open(file_path, "wb") as f:
            for j in range(num_blocks):
                logger.debug("Receive block %s of %s" % (j + 1, num_blocks))
                data = await websocket.recv()
                file_hashes[filename].update(data)
                f.write(data)
            logger.debug("Done")
        file_hashes[filename] = file_hashes[filename].hexdigest()


# TODO: Implement temporary status changes.
# TODO: Power up the items as the level goes up...
from random import choice
from random import uniform
from functools import reduce
from getch import _Getch
from maze_object import MazeObject

poison_hp_reduction_rate = 200
getch = _Getch()

starving_ep_reduction_rate = 4
status_effect_decay_rate_in_map = 1
status_effect_decay_rate_in_fight = 10
status_effect_decay_rate_in_fight_pararalyze = 1
status_effect_decay_rate_in_map_paralyze = 1

# class Item(object):
#     def __init__(self, item_name,json_data = {}, level = 1):
strength_parameters =["strength", "agility", "vitality", "dexterity",
                      "smartness", "magic_power", "mental_strength", "luckiness"]

item_change_parameters = ["strength_change","agility_change",
                          "vitality_change","dexterity_change",
                          "smartness_change","magic_power_change",
                          "mental_strength_change","luckiness_change",
                          "hp_change", "mp_change",
                          "ep_change","sp_change"]

# Take level into consideration when creating item
def extract_item_names(item_data):
    if item_data != []:
        return list(map(lambda x: list(x.keys())[0],item_data))
    else:
        return []

def reduce_item_data(one_item, item_types):
    if len(item_types) > 1:
        return reduce(lambda x, y: one_item[list(one_item.keys())[0]]["is_" + x]
                    or one_item[list(one_item.keys())[0]]["is_" + y], item_types)
    else:
        return one_item[list(one_item.keys())[0]]["is_" + item_types[0]]

# Find the items to list based on item_type.
def find_item_type(item_data, item_types):

    # If the item type is a string, then it will become [string]
    if isinstance(item_types, str):
        item_types = [item_types]

    tmp_items = []
    other_items = []

    for i,item_type in enumerate(map(lambda x: reduce_item_data(x, item_types), item_data)):
        if item_type:
            tmp_items.append(item_data[i])
        else:
            other_items.append(item_data[i])

    return tmp_items, other_items

# Change the effects of items (equipment and expendable)

def alter_item_effects_by_level(tmp_key, tmp_item_json, level = 1):

    tmp = {}
    # All item parameters are changed based on the floor level
    # Generally, it is often to be positive!
    for i in item_change_parameters:
        tmp_item_json[i] = tmp_item_json[i] * (uniform(1,1 + 0.1*level))

    # The item is followed by level.
    tmp_key += f"_{level}"
    tmp[tmp_key] = tmp_item_json

    return tmp

# The skill book gives player to create the
# Randomly select the skills from skill list.
# Skills are randomly selected based on the book level.
# When a skill book is used, he/she can obtain one skill randomly.
# Player cannot choose his/her skills as he/she wishes from skill books.
def use_skill_book(skill_data, book_level = 1):
    # Filter the skill based on the depth of the floor.
    itemized_skill_data = skill_data.items()
    filtered_skill_data = list(filter(lambda x: x[1]["level"] <= book_level, itemized_skill_data))
    return dict((choice(filtered_skill_data),))

def filter_and_choose_monster_by_level(monster_data, floor_level = 1):
    filtered_monster_data = list(filter(lambda x: x[list(x.keys())[0]]["level"] <= floor_level, monster_data))
    return dict((choice(filtered_monster_data),))

def filter_and_choose_item_data_by_level(item_data, floor_level = 1):
    filtered_item_data = list(filter(lambda x: x[1]["level"] <= floor_level, item_data.items()))
    return dict((choice(filtered_item_data),))

# This function is mainly used for sorting purpose.
def filter_item_data_by_type(item_data, item_type):
    filtered_item_data = list(filter(lambda x: x[1][item_type], item_data.items()))
    return filtered_item_data

# Use item for player.
def use_item(player: MazeObject,item_name,item_data):

    # TODO: Do not use item when a current value is the same as a maximum value.
    hp_changed = mp_changed = sp_changed = ep_changed =\
    poison_changed = curse_changed = seal_changed = paralyze_changed = starving_changed = False

    if player.object_data["current_hp"] != player.object_data["current_max_hp"] and item_data[item_name]["hp_change"] != 0:
        player.object_data["current_hp"]  = \
            min(player.object_data["current_hp"]  + item_data[item_name]["hp_change"], player.object_data["current_max_hp"])
        print("hp is recovered by {}".format(item_data[item_name]["hp_change"]))
        hp_changed = True

    if player.object_data["current_mp"] != player.object_data["current_max_mp"] and item_data[item_name]["mp_change"] != 0:
        player.object_data["current_mp"]  = \
            min(player.object_data["current_mp"]  + item_data[item_name]["mp_change"], player.object_data["current_max_mp"])
        print("mp is recovered by {}".format(item_data[item_name]["mp_change"]))
        mp_changed = True

    if player.object_data["current_sp"] != player.object_data["current_max_sp"] and item_data[item_name]["sp_change"] != 0:
        player.object_data["current_sp"]  = \
            min(player.object_data["current_sp"]  + item_data[item_name]["sp_change"], player.object_data["current_max_sp"])
        print("sp is recovered by {}".format(item_data[item_name]["sp_change"]))
        sp_changed = True

    if player.object_data["current_ep"] != player.object_data["current_max_ep"] and item_data[item_name]["ep_change"] != 0:
        player.object_data["current_ep"]  = \
            min(player.object_data["current_ep"]  + item_data[item_name]["ep_change"], player.object_data["current_max_ep"])
        print("ep is recovered by {}".format(item_data[item_name]["ep_change"]))
        ep_changed = True

    if "poison" in player.object_data["status_effects"] and item_data[item_name]["cure_poison"]:
        player.object_data["status_effects"].remove("poison")
        player.object_data["poison_count"] = 0
        print("Cured poison!")
        poison_changed = True

    if "curse" in player.object_data["status_effects"] and item_data[item_name]["cure_curse"]:
        player.object_data["status_effects"].remove("curse")
        player.object_data["curse_count"] = 0
        print("Cured curse!")
        curse_changed = True

    if "seal" in player.object_data["status_effects"] and item_data[item_name]["cure_seal"]:
        player.object_data["status_effects"].remove("seal")
        player.object_data["seal_count"] = 0
        print("Cured seal!")
        seal_changed = True

    if "paralyze" in player.object_data["status_effects"] and item_data[item_name]["cure_paralyze"]:
        player.object_data["status_effects"].remove("paralyze")
        player.object_data["paralyze_count"] = 0
        print("Cured paralyze!")
        paralyze_changed = True

    if "starving" in player.object_data["status_effects"] and item_data[item_name]["cure_starving"]:
        player.object_data["status_effects"].remove("starving")
        player.object_data["starving_count"] = 0
        print("Cured starving!")
        starving_changed = True

    return hp_changed or mp_changed or sp_changed or ep_changed or\
        poison_changed or curse_changed or seal_changed or paralyze_changed or\
            starving_changed

def remove_status_effects(user:MazeObject, data, name):

    is_removed_status_effect = False
    cured_list = []

    if "poison" in user.object_data["status_effects"] and data[name]["cure_poison"]:
        user.object_data["status_effects"].remove("poison")
        is_removed_status_effect = True
        cured_list.append("poison")
        user.object_data["poison_count"] = 0

    if "curse" in user.object_data["status_effects"] and data[name]["cure_curse"]:
        user.object_data["status_effects"].remove("curse")
        is_removed_status_effect = True
        cured_list.append("curse")
        user.object_data["curse_count"] = 0

    if "seal" in user.object_data["status_effects"] and data[name]["cure_seal"]:
        user.object_data["status_effects"].remove("seal")
        is_removed_status_effect = True
        cured_list.append("seal")
        user.object_data["seal_count"] = 0

    if "paralyze" in user.object_data["status_effects"] and data[name]["cure_paralyze"]:
        user.object_data["status_effects"].remove("paralyze")
        is_removed_status_effect = True
        cured_list.append("paralyze")
        user.object_data["paralyze_count"] = 0

    if "starving" in user.object_data["status_effects"] and data[name]["cure_starving"]:
        user.object_data["status_effects"].remove("starving")
        is_removed_status_effect = True
        cured_list.append("starving")
        user.object_data["starving_count"] = 0

    return is_removed_status_effect, cured_list

# The infliction of status effects on either player or enemy.
def add_target_status_effect(target:MazeObject, skill_data, skill_name):

    # TODO: Calculate the possibility of status infliction.

    if skill_data[skill_name]["poison_possibility"] > uniform(0, 1.0):
        if not "poison" in target.object_data["status_effects"]:
            target.object_data["status_effects"].append("poison")
        target.object_data["poison_count"] = 100

    if skill_data[skill_name]["curse_possibility"] > uniform(0, 1.0):
        if not "curse" in target.object_data["status_effects"]:
            target.object_data["status_effects"].append("curse")
        target.object_data["curse_count"] = 100

    if skill_data[skill_name]["seal_possibility"] > uniform(0, 1.0):
        if not "seal" in target.object_data["status_effects"]:
            target.object_data["status_effects"].append("seal")
        target.object_data["seal_count"] = 200

    if skill_data[skill_name]["paralyze_possibility"] > uniform(0, 1.0):
        if not "paralyze" in target.object_data["status_effects"]:
            target.object_data["status_effects"].append("paralyze")
        target.object_data["paralyze_count"] = 5

    if skill_data[skill_name]["starving_possibility"] > uniform(0, 1.0):
        if not "starving" in target.object_data["status_effects"]:
            target.object_data["status_effects"].append("starving")
        target.object_data["starving_count"] = 200

def use_skill(skill_user, skill_data, target = None, is_in_menu = True):

    # TODO: Do not use item when a current value is the same as a maximum value
    skill_name = list(skill_data.keys())[0]

    # If "seal" effect is active, then the
    if "seal" in skill_user.object_data["status_effects"]:
        print("Cannot use skill in seal condition!")
        getch()
        return False

    if is_in_menu and skill_data[skill_name]["is_in_fight"]:
        print("This skill cannot be used in the player's menu.")
        getch()
        return False

    # Heal or alter player's status.
    # When enemy us`ing the skill, whether skill is used in main menu must be considered
    # Status change is applied to
    elif not skill_data[skill_name]["is_in_fight"]:
        if not is_in_menu or (skill_user.object_data["current_hp"] - skill_data[skill_name]["hp_spent"] >= 0
        or skill_data[skill_name]["hp_spent"] == 0)\
        and (skill_user.object_data["current_mp"] - skill_data[skill_name]["mp_spent"] >= 0
        or skill_data[skill_name]["mp_spent"] == 0)\
        and (skill_user.object_data["current_sp"] - skill_data[skill_name]["sp_spent"] >= 0
        or skill_data[skill_name]["sp_spent"] == 0)\
        and (skill_user.object_data["current_ep"] - skill_data[skill_name]["ep_spent"] >= 0
        or skill_data[skill_name]["ep_spent"] == 0):

            is_status_removed, removed_list = remove_status_effects(skill_user, skill_data, skill_name)

            if not is_in_menu or (skill_user.object_data["current_hp"] != skill_user.object_data["current_max_hp"]
            and skill_data[skill_name]["hp_change"] != 0)\
            or (skill_user.object_data["current_mp"] != skill_user.object_data["current_max_mp"]
            and skill_data[skill_name]["mp_change"] != 0)\
            or (skill_user.object_data["current_sp"] != skill_user.object_data["current_max_sp"]
            and skill_data[skill_name]["sp_change"] != 0)\
            or (skill_user.object_data["current_ep"] != skill_user.object_data["current_max_ep"]
            and skill_data[skill_name]["ep_change"] != 0)\
            or is_status_removed:

                hp_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["hp_change"])
                skill_user.object_data["current_hp"] = min(skill_user.object_data["current_hp"] +
                hp_change,
                skill_user.object_data["current_max_hp"])

                mp_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["mp_change"])
                skill_user.object_data["current_mp"] = min(skill_user.object_data["current_mp"] +
                mp_change,
                skill_user.object_data["current_max_mp"])

                sp_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["sp_change"])
                skill_user.object_data["current_sp"] = min(skill_user.object_data["current_sp"] +
                sp_change,
                skill_user.object_data["current_max_sp"])

                ep_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["ep_change"])
                skill_user.object_data["current_ep"] = min(skill_user.object_data["current_ep"] +
                ep_change,
                skill_user.object_data["current_max_ep"])

                if removed_list != []:
                    print("Cured {0}!".format(", ".join(removed_list)))

                return hp_change, mp_change, sp_change, ep_change, True

    # Player can be the person who receive the damage from enemy.
    # TODO: Add the status effects on the enemy.
    # If player is in fight, then it will perform the attack against enemy.
    elif skill_data[skill_name]["is_in_fight"] and target != None:
        if not is_in_menu or (skill_user.object_data["current_hp"] - skill_data[skill_name]["hp_spent"] >= 0
        or skill_data[skill_name]["hp_spent"] == 0)\
        and (skill_user.object_data["current_mp"] - skill_data[skill_name]["mp_spent"] >= 0
        or skill_data[skill_name]["mp_spent"] == 0)\
        and (skill_user.object_data["current_sp"] - skill_data[skill_name]["sp_spent"] >= 0
        or skill_data[skill_name]["sp_spent"] == 0)\
        and (skill_user.object_data["current_ep"] - skill_data[skill_name]["ep_spent"] >= 0
        or skill_data[skill_name]["ep_spent"] == 0):

            hp_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["hp_change"])
            target.object_data["current_hp"] = min(target.object_data["current_hp"] +
            hp_change,
            target.object_data["current_max_hp"])

            mp_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["mp_change"])
            target.object_data["current_mp"] = min(target.object_data["current_mp"] +
            mp_change,
            target.object_data["current_max_mp"])

            sp_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["sp_change"])
            target.object_data["current_sp"] = min(target.object_data["current_sp"] +
            sp_change,
            target.object_data["current_max_sp"])

            ep_change = _use_skill_sub(skill_user, skill_data, skill_data[skill_name]["ep_change"])
            target.object_data["current_ep"] = min(target.object_data["current_ep"] +
            ep_change,
            target.object_data["current_max_ep"])

            skill_user.object_data["current_hp"]  -= skill_data[skill_name]["hp_spent"]
            skill_user.object_data["current_mp"]  -= skill_data[skill_name]["mp_spent"]
            skill_user.object_data["current_sp"]  -= skill_data[skill_name]["sp_spent"]
            skill_user.object_data["current_ep"]  -= skill_data[skill_name]["ep_spent"]

            # Remove status effects from users.
            remove_status_effects(skill_user, skill_data, skill_name)

            # Inflict status effects on player or enemy.
            add_target_status_effect(target, skill_data,skill_name)

            # Inflict the status effect on user.
            return hp_change, mp_change, sp_change, ep_change, False

# Return the value in accordance with the changes of the values.
# TODO: Add the multiplier to the skills.
def _use_skill_sub(player,skill_data,changed_value):

    # Initialise the values
    tmp = 0
    tmp_name = list(skill_data.keys())[0]
    for i in strength_parameters:
        tmp += changed_value * (uniform(0.8, 1.0) * (player.object_data["current_" + i] * skill_data[tmp_name][i + "_multiplier"]))

    return int(round(tmp,0))

# NOTE: The following methods are only for debugging purposes.
# Shows all the contents of data...
def display_debug_status(obj):
    # Print all json-formatted player's data:
    print(obj.object_data)


from collections import Counter
from timeit import default_timer as timer

def moves(x, y):
    dx, dy = ((-1, 0, 1, 1, 1, 0, -1, -1), (-1, -1, -1, 0, 1, 1, 1, 0))
    for d in range(8):
        yield (x + dx[d], y + dy[d])

def show(acres, max_x, max_y):
    for y in range(max_y):
        for x in range(max_x):
            c = '.' if acres[(x, y)] == 0 else '|' if acres[(x, y)] == 1 else "#"
            print(c, sep='', end='')
        print()

start = timer()
lines = [line.strip() for line in open("input/18.txt")]

# 0 = open, 1 = trees, 2 = lumber
acres = dict()
for y, line in enumerate(lines):
    for x, c in enumerate(line):
        acres[(x, y)] = 0 if c == "." else 1 if c == "|" else 2
max_x, max_y = (x, y)

seen = {}
minute = 0
target_minute = 1_000_000_000
while minute < target_minute:
    minute += 1

    # Simulate new states of all acres
    new_acres = acres.copy()
    for (x, y), state in acres.items():
        adj_count = Counter(acres[(xx, yy)] for (xx, yy) in moves(x, y) if (xx, yy) in acres)
        if state == 0 and adj_count[1] >= 3:
            state = 1
        elif state == 1 and adj_count[2] >= 3:
            state = 2
        elif state == 2 and (adj_count[1] < 1 or adj_count[2] < 1):
            state = 0
        new_acres[(x, y)] = state
    acres = new_acres

    # Part A
    if minute == 10:
        count = Counter(acres.values())
        print(count[1] * count[2])

    # Part B
    hash = "".join(str(s) for s in acres.values())
    if hash in seen.keys():
        diff = minute - seen[hash]
        minute = minute + diff * ((target_minute - minute) // diff)
    else:
        seen[hash] = minute

count = Counter(acres.values())
print(count[1] * count[2])
end = timer()
print(f'Took {end - start} seconds.')

# show(acres, max_x, max_y)


#!/usr/bin/env python
import pygame
import sys
from pygame.locals import *

CLOCK_TICK = pygame.USEREVENT

class CApp:
    def __init__(self):
        self._running = True
        self._surf_display = None
        self.MoveBullet = False

    def OnInit(self):
        self.size = self.width, self.height = 640, 400
        self.speed_down =[0, 2]
        self.speed_right = [2, 0]
        self.speed_up = [0, -2]
        self.speed_left = [-2, 0]
        self.black = 0, 0, 0

        pygame.init()
        self._display_surf = pygame.display.set_mode((640,400),\
                               pygame.HWSURFACE | pygame.DOUBLEBUF)
        pygame.display.set_caption('this is the first programme!')
        self._running = True

        #Fill background
        self.background = pygame.Surface(self._display_surf.get_size())
        self.background = self.background.convert()
        self.background.fill((255, 235, 205))
        self._display_surf.blit(self.background, (0, 0))

        # add a plane
        self.plane = pygame.image.load("plane.png")
        self.planerect = self.plane.get_rect()

        # add a bullet image
        self.bullet = pygame.image.load("apple.png")

        # add a bullet list
        self.bullets = []

        pygame.time.set_timer(CLOCK_TICK, 20)

    def OnEvent(self, event):
        if event.type == QUIT:
            self._running = False
            sys.exit()
        elif event.type == CLOCK_TICK:
           for rect in self.bullets:
                self.bullets.remove(rect)
                rect = rect.move(self.speed_down)
                if rect.left < 0 or rect.right > self.width or \
                     rect.top < 0 or rect.bottom > self.height:
                    pass
                else:
                    self.bullets.append(rect)
        elif event.type == KEYUP:
            if event.key == pygame.K_SPACE:
                self.bulletrect = self.bullet.get_rect()
                self.bulletrect = self.bulletrect.move(self.planerect.center)
                self.bullets.append(self.bulletrect)

    def OnLoop(self):
        pass
    def OnRender(self):
        pass

    def BallMoveBykey(self):
        """
        move the plane by the way
        """
        pressed_keys = pygame.key.get_pressed()

        if pressed_keys[pygame.K_UP]:
            self.planerect = self.planerect.move(self.speed_up)
            self.BorderHandle()
        elif pressed_keys[pygame.K_DOWN]:
            self.planerect = self.planerect.move(self.speed_down)
            self.BorderHandle()
        elif pressed_keys[pygame.K_LEFT]:
            self.planerect = self.planerect.move(self.speed_left)
            self.BorderHandle()
        elif pressed_keys[pygame.K_RIGHT]:
            self.planerect = self.planerect.move(self.speed_right)
            self.BorderHandle()

    def BorderHandle(self):
        """
        prevent the plane out of the border
        """
        if self.planerect.left < 0:
            self.planerect = self.planerect.move(self.speed_right)
        elif self.planerect.right > self.width:
            self.planerect = self.planerect.move(self.speed_left)
        elif self.planerect.top < 0:
            self.planerect = self.planerect.move(self.speed_down)
        elif self.planerect.bottom > self.height:
            self.planerect = self.planerect.move(self.speed_up)

    def DrawBullet(self):
        """
        make the plane move auto
        """
        self._display_surf.blit(self.bullet, self.bulletrect)
        self._display_surf.blit(self.plane, self.planerect)
        self.bulletrect = self.bulletrect.move(self.speed_down)

    def BulletMove(self):
        """
        the moving bullet launched by flighter
        """

    def Blit(self):
        """
        draw the picture
        """
        pygame.display.flip()
        self._display_surf.blit(self.background, (0, 0))
        self._display_surf.blit(self.plane, self.planerect)
        for rect in self.bullets:
            self._display_surf.blit(self.bullet, rect)

    def OnCleanup(self):
        pygame.quit()

    def OnExecute(self):
        if self.OnInit() == False:
            self._running = False

        while( self._running ):
            for event in pygame.event.get():
                self.OnEvent(event)
            self.OnLoop()
            self.OnRender()
            self.BallMoveBykey()
            self.Blit()

if __name__ == "__main__" :
    theApp = CApp()
    theApp.OnExecute()


import numbers
from typing import List, Optional

# Definition for singly-linked list.
class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:
    def parseIntFromReversedLinkedList(self, reversedLinkedList:ListNode) -> int:
        values_in_reversed_linked_list = []
        current_element = reversedLinkedList
        result = 0
        while True:
            values_in_reversed_linked_list.append(current_element.val)

            if current_element.next == None:
                break
            current_element = current_element.next

        for i in range(len(values_in_reversed_linked_list) -1, -1, -1):
            result = result + (values_in_reversed_linked_list[i] * pow(10, i))

        return result

    def buildRevesedLinkedListFromInt(self, value) -> ListNode:
        value_as_string = str(value)
        result = ListNode()
        current_node = result
        for i in range(len(value_as_string) -1, -1, -1):
            current_node.val = value_as_string[i]

            if(i == 0):
                break

            current_node.next = ListNode()
            current_node =  current_node.next

        return result

    def addTwoNumbers(self, l1: Optional[ListNode], l2: Optional[ListNode]) -> Optional[ListNode]:
        l1_as_int = self.parseIntFromReversedLinkedList(l1)
        l2_as_int = self.parseIntFromReversedLinkedList(l2)
        result_as_int = l1_as_int + l2_as_int
        return self.buildRevesedLinkedListFromInt(result_as_int)

list_a = ListNode(9, ListNode(9, ListNode(9, ListNode(5))))
list_b = ListNode(5, ListNode(6, ListNode(7)))
solution = Solution()
print(solution.addTwoNumbers(list_a, list_b))


import numpy as np
import pdb

from modules.base import ModuleBase

class BinaryCrossEntropyLoss(ModuleBase):
    def __init__(self):
        super(BinaryCrossEntropyLoss, self).__init__()
    def forward(self, x, label):
        # label: batch_size
        # x: batch_size x 1
        batch_size = x.shape[0]
        blabel = label.reshape([batch_size, 1])
        bce = -(blabel * np.log(x) + (1 - blabel) * log(1 - x)) # batch_size  x 1
        self.results['blabel'] = blabel
        self.results['x'] = x
        return np.sum(bce) / batch_size
    def backward(self):
        blabel = self.results['blabel']
        batch_size = blabel.shape[0]
        x = self.results['x']
        grad_input = -(blabel / x - (1 - blabel) / (1 -  x))
        self.results = {}
        return grad_input / batch_size

class CrossEntropyLoss(ModuleBase):
    def __init__(self):
        super(CrossEntropyLoss, self).__init__()
    def forward(self, x, label):
        # label: batch_size
        # x:  batch_size  x num_classes
        batch_size = x.shape[0]
        num_classes = x.shape[1]
        onehot = np.eye(num_classes)[label]
        loss = -np.sum(np.log(x) * onehot)
        self.results['x'] = x
        self.results['onehot'] = onehot
        return loss / batch_size
    def backward(self):
        revx = 1 / self.results['x']
        batch_size = revx.shape[0]
        grad_input = -revx * self.results['onehot']
        self.results = {}
        return grad_input / batch_size

class L2Loss(ModuleBase):
    def __init__(self):
        super(L2Loss, self).__init__()
    def forward(self, x, groud_truth):
        # x: batch_size x dim
        batch_size = x.shape[0]
        diff = x - groud_truth # batch_size x dim
        self.result['diff'] = diff
        return np.sum(diff ** 2) / batch_size
    def backward(self):
        batch_size = self.results['diff'].shape[0]
        grad_input = 2 * self.results['diff']
        self.results = {}
        return grad_input / batch_size


'''
Created on 2022-11-22

@author: wf
'''
from tests.basetest import Basetest
from skg.doi import DOI
from skg.dblp import Dblp
from dataclasses import dataclass
from unittest import IsolatedAsyncioTestCase
import json

@dataclass
class DOIExample:
    doi: str
    isDoi: bool

class TestDOI(Basetest):
    """
    test DOI access
    """

    def testDOICheck(self):
        """
        check DOI regex
        """
        doi_examples=[
            DOIExample("10.1007/11581116_19",True),
            DOIExample("10.1109/TBDATA.2022.3224749",True),
            DOIExample("xyz",False)]
        for doi_example in doi_examples:
            self.assertTrue(DOI.isDOI(doi_example.doi)==doi_example.isDoi,doi_example.doi)

    def test_dblp_dois(self):
        """
        test dblp dois
        """
        dblp=Dblp()
        limit=5
        debug=True
        paper_records=dblp.get_paper_records("CEUR Workshop Proceedings","publishedin",limit=limit,debug=debug)
        for paper_record in paper_records:
            print(paper_record)

    def testDOILookup(self):
        """
        test DOI lookup
        """
        debug=True
        dois=["10.1109/TBDATA.2022.3224749"]
        expected=["@article{Li_2022,","@inproceedings{Faruqui_2015,"]
        for i,doi in enumerate(dois):
            doi_obj=DOI(doi)
            result=doi_obj.doi2bibTex()
            if debug:
                print(result)
            self.assertTrue(result.startswith(expected[i]))

    def testCiteproc(self):
        """
        cite proc lookup
        """
        dois=["10.3115/v1/N15-1184",
              "10.1007/s13042-022-01686-5",
              "10.3390/info13120562",
              "10.48550/arXiv.2211.16865"]
        debug=True
        for doi in dois:
            doi_obj=DOI(doi)
            json_data=doi_obj.doi2Citeproc()
            if debug:
                print(json.dumps(json_data,indent=2))
            self.assertTrue("DOI" in json_data)
            self.assertEqual(doi.lower(),json_data["DOI"].lower())

    def testDataCiteLookup(self):
        """
        test the dataCite Lookup api
        """
        debug=True
        dois=["10.5438/0012"]
        for doi in dois:
            doi_obj=DOI(doi)
            json_data=doi_obj.dataCiteLookup()
            if debug:
                print(json.dumps(json_data,indent=2))
            self.assertTrue("data" in json_data)
            data=json_data["data"]
            self.assertTrue("id" in data)
            self.assertEquals(doi,data["id"])
            pass

    def testScite(self):
        """
        test Semantic Cite conversion
        """
        dois=["10.1007/978-3-031-21595-7_17","10.48550/ARXIV.2211.16865"]
        debug=self.debug
        for doi in dois:
            doi_obj=DOI(doi)
            markup=doi_obj.asScite()
            if debug:
                print(markup)
            self.assertTrue(f"|doi={doi}" in markup)

#class TestDOILookup(IsolatedAsyncioTestCase):
#    """
#    test DOI lookup
#    """


from __future__ import division
from __future__ import print_function
from sentence_transformers import SentenceTransformer

from nltk import tokenize
import sys
import numpy as np

from pyod.models.copod import COPOD
from pyod.utils.data import evaluate_print

# 提取dataset中的评论并且
import csv

import nltk
nltk.download('punkt')

data = []
# with open('./paper.json', 'r') as file:
#     papers = json.load(file)
#     data = [paper["title"] for paper in papers]

with open('./papers.csv', 'rt',encoding='utf-8') as file:
    csv_reader = csv.reader(file)
    for idx, row in enumerate(csv_reader):
        if idx > 601:
            break
        if idx > 101:
            data.append(row[4])

print("end csv reading")

data.append("Recurrent topic-transition gan for visual paragraph generation.")
data.append("Modelling one‐and two‐dimensional solid‐state NMR spectra ")
data.append("Storygan: A sequential conditional gan for story visualization")
data.append("Attngan: Fine-grained text to image generation with attentional generative adversarial networks")
data.append("A perpendicular-anisotropy CoFeB–MgO magnetic tunnel junction")
data.append("Tidal ventilation at low airway pressures can augment lung injury.")

data_vecs = []
model = SentenceTransformer('sentence-transformers/multi-qa-mpnet-base-dot-v1')
for i, s in enumerate(data):
    sys.stdout.write("\r%d/%d" % (i, len(data)))
    sys.stdout.flush()
    r = model.encode(tokenize.sent_tokenize(s)).mean(axis=0)
    data_vecs.append(r.tolist())

print("end model encoding")

data_vecs = np.array(data_vecs)
print(np.shape(data_vecs))

clf = COPOD()
clf.fit(data_vecs)

print(np.shape(clf.labels_))

res = np.c_[clf.labels_, clf.decision_scores_].tolist()
for i, r in enumerate(res):
    r.append(data[i])

with open("new.csv", "w+",encoding = 'utf-8') as my_csv:
    csvWriter = csv.writer(my_csv, delimiter=',')
    csvWriter.writerows(res)


'''
@author: Marco Bonvini
'''
import numpy
import pandas as pd

from estimationpy.fmu_utils.csv_reader import CsvReader
from estimationpy.fmu_utils import strings
import pyfmi

import logging
logger = logging.getLogger(__name__)

class InOutVar():
    """
    This class represents a generic input or output variable of a
    dynamic system.
    In case of state and parameter estimation both input and outputs
    of a system have the characteristic of having measurements associated
    to them. These measurements, together with other informations that can
    quantify their unsertanty, are used by the estimation algorithm.

    This class can be seen as a wrapper around the class **pyfmi.fmi.ScalarVariable**
    with the addition of an object that represent a measurement time series.
    The time series can be directly defined as a pandas.Series object or with the
    convenience class :class:`estimationpy.fmu_utils.csv_reader.CsvReader`.

    **Note**

        When dealing with real systems not all the output measurements can be measured.
        For example a translational mechanic system can have as outputs the position of a mass,
        it velocity, and its acceleration. However, in a real application one may only have
        access to one of these measurements.
        The class uses a boolean flag to define whether an output is actually being measured and
        thus can be used by a state estimation algorithm.

    """

    def __init__(self, pyfmi_var = None):
        """
        Consttuctor for the class :class:`InOutVar`. The constructor takes as input
        parameter an object of type **pyfmi.fmi.ScalarVariable** that contains the
        information about the variable.
        The method initializes an empty **CsvReader** object and an empty **pandas.Series**
        object that can be futher defined.
        The method also initializes the covariance :math:`\sigma^2` associated to this variable to be 1.0,
        and the flag ``measOut = False``.

        The class provide, the CsvReader class associated to the input (that contains the data),
        a dictionary called dataSeries = {"time": [], "data": []} that contains the two arrays that represent
        the data series (that are read from the csv file).

        The integer index is used when the data values are read using the function ReadFromDataSeries, while
        cov is the covariance associated to the data series.

        :param pyfmi.fmi.ScalarVariable pyfmi_var: the pyfmi object representing a variable.

        """
        self.pyfmi_var = pyfmi_var
        self.csvReader = CsvReader()
        self.dataSeries = pd.Series()

        self.index = 0
        self.cov = 1.0
        self.measOut = False

    def read_value_in_fmu(self, fmu):
        """
        This method reads the value of a variable/parameter
        assumes in a specific FMU object.

        :param FmuModel fmu: an object representing an FMU model in PyFMI.

        :return: The value of the variable represented by an instance of this class.
          The method returns `None` is the type of the variable is not recognized as one
          of the available ones (Real, Integer, Boolean, Enumeration, String).

        :rtype: float, None

        """
        t = self.pyfmi_var.type
        if t == pyfmi.fmi.FMI_REAL:
            val = fmu.get_real(self.pyfmi_var.value_reference)
        elif t == pyfmi.fmi.FMI_INTEGER:
            val = fmu.get_integer(self.pyfmi_var.value_reference)
        elif t == pyfmi.fmi.FMI_BOOLEAN:
            val = fmu.get_boolean(self.pyfmi_var.value_reference)
        elif t == pyfmi.fmi.FMI_ENUMERATION:
            val = fmu.get_int(self.pyfmi_var.value_reference)
        elif t == pyfmi.fmi.FMI_STRING:
            val = fmu.get_string(self.pyfmi_var.value_reference)
        else:
            msg = "FMU-EXCEPTION, The type {0} is not known".format(t)
            logger.error(msg)
            return None
        return val[0]

    def set_measured_output(self, flag = True):
        """
        This method set the flag that indicates if the variable represents a measured output.

        :param bool flag: flag that indicates whether this variable is a measured output
          or not.
        """
        self.measOut = flag

    def is_measured_output(self):
        """
        This method returns the value of the boolean flag that describe
        if the variable is a measured output.

        :return: a boolean value that indicates if this variable is a measured output.

        :rtype: bool

        """
        return self.measOut

    def set_covariance(self, cov):
        """
        This method sets the covariance associated to
        an instance of the class :class:`InOutVar` that is used
        by the state and parameter estimation algorithm.

        :param float cov: The value to be used as initial value in the estimation
            algorithm. The value must be positive.

        :return: True if the value has been set corectly, False otherwise.

        :rtype: bool

        """
        if cov > 0.0:
            self.cov = cov
            return True
        else:
            msg = "The covariance must be positive"
            logger.error(msg)
            raise ValueError(msg)

    def get_covariance(self):
        """
        This method returns the covariance of the **InOutVar** object.

        :return: the covariance of the variable.
        :rtype: float

        """
        return self.cov

    def set_object(self, pyfmi_var):
        """
        This method sets the variable associated to the object of class
        **InOutVar**.

        :param pyfmi.fmi.ScalarVariable pyfmi_var: The variable associated to the input/output represented
          by the object.

        :raises TypeError: The method raises an exception if the parameter is not of the proper type.

        """
        if (isinstance(pyfmi_var, pyfmi.fmi.ScalarVariable)) or (isinstance(pyfmi_var, pyfmi.fmi.ScalarVariable2)):
            self.pyfmi_var = pyfmi_var
        else:
            raise TypeError("The object passed to the method InOutVar.set_object() is not of type pyfmi.fmi.ScalarVariable ")

    def get_object(self):
        """
        Get the object associated to the **pyfmi.ScalarVariable** associated to this input/output
        variable.

        :return: the pyfmi variable object reference by this variable.
        :rtype: pyfmi.fmi.ScalarVariable

        """
        return self.pyfmi_var

    def set_csv_reader(self, reader):
        """
        This method associates an object of type :class:`estimationpy.fmu_utils.csv_reader.CsvReader` to this
        input/output variable. The **CsvReder** will be used to read data by the state and parameter estimation
        algorithm.

        :param estimationpy.fmu_utils.csv_reader.CsvReader reader: The **CsvReader** object to associate to this variable.

        :raises TypeError: The method raises an exception if the type of the argument reader is not correct.

        """
        if isinstance(reader, CsvReader):
            self.csvReader = reader
        else:
            msg = "The object passed to the method InOutVar.SetCsvReader() is not of type FmuUtils.CsvReader.CsvReader"
            msg += "\n it is of type %s" % (str(type(reader)))
            raise TypeError(msg)

    def get_csv_reader(self):
        """
        This method returns a reference to the **CsvReader** object associated to the
        input/output variable.

        :return: the reference to the **CsvReader** object.

        :rtype: estimationpy.fmu_utils.csv_reader.CsvReader

        """
        return self.csvReader

    def read_data_series(self):
        """
        This method reads the data series associated to this input/output variable.
        The method checks if the **CsvReader** object has a valid file name
        associated to it. If not it looks if the data series has already been
        specified and is a not empty **pandas.Series**.
        The method returns True if the data is read, False otherwise.

        :return: True if the method is able to read the data from either the CsvReader obejct
          of the pandas.Series. False otherwise.
        :rtype: bool
        """
        # If the CsvReader has been specified the try to load the data from there
        if self.csvReader.filename == "" or self.csvReader.filename == None:

            # Check because the dataSeries may have bee specified using a pandas.Series
            if len(self.dataSeries) > 0:
                return True
            else:
                return False
        else:

            # Read the data from the CSV
            self.dataSeries = self.csvReader.get_data_series()
            if len(self.dataSeries) > 0:
                return True
            else:
                return False

    def get_data_series(self):
        """
        This method returns the data series associated to this input/output variable.
        The dat aseries can be either specified by a CSV file by means of a
        **Csvreader** object, or directly from a **pandas.Series** object.

        :return: the pandas.Series associated to this variable.
        :rtype: pandas.Series
        """
        return self.dataSeries

    def set_data_series(self, series):
        """
        This function sets a data series instead of reading it from the CSV file.
        The data series has to be a **pandas.Series** object that is indexed with
        a **pandas.tseries.index.DatetimeIndex**.

        :param pandas.Series series: the time series to be associated to this input/output
          variable.

        :raises TypeError: The method raise an error if the parameter ``series`` is
          not of the right type or it's a pandas.Series not indexed with a
          **pandas.tseries.index.DatetimeIndex**.

        """
        if isinstance(series, pd.Series):
            if isinstance(series.index, pd.DatetimeIndex):
                self.dataSeries = series
            else:
                raise TypeError("The index of the Series passed to the method InOutVar.SetDataSeries() is not of type pandas.DatetimeIndex")
        else:
            raise TypeError("The object passed to the method InOutVar.SetDataSeries() is not of type pandas.Series ")

    def read_from_data_series(self, ix):
        """
        This method reads and return the value associated to the input/output variable
        at the time specified by the parameter ``ix``. The parameter ``ix`` needs
        to be a vaid index that belongs to a **pandas.tseries.index.DatetimeIndex**.
        If the index ``ix`` is not one of the indexes of the pandas.Series the
        method performs a linear interpolation between the two closest values to
        compute the value.

        :param ix: the time stamp for which providing the value.

        :return: the value that is read from the pandas.Series associated to the variable.
          if the index is out of range the method returns False.

        :rtype: float, bool

        """
        # Identify start and end date of the period covered by the time series
        from_start = (ix - self.dataSeries.index[0]).total_seconds()
        to_end = (self.dataSeries.index[-1] - ix).total_seconds()

        if from_start < 0.0 or to_end < 0.0:
            # The index ix is not contained in the array, it's either
            # before the start or after the end
            return False

        try:
            # try to read directly the index, if it exists it's done
            value = self.dataSeries.loc[ix]
            return value

        except KeyError:

            # The index ix is not present, an interpolation is needed.
            # Since it is a sequential access, store the last position to reduce the access time for the next iteration
            index = self.index
            N = len(self.dataSeries.index)

            # Start the identification of the position of the closest time step

            # If len(time) = 10 and index was 2, indexes is [2, 3, 4, 5, 6, 7, 8, 9, 0, 1]
            indexes = numpy.concatenate((numpy.arange(index,N), numpy.arange(index+1)))

            # Start the iteration
            logger.debug("Indexes = {0}".format(indexes))

            for i in range(N):

                j = indexes[i]

                logger.debug("j = {0}".format(j))

                # Get the time values (of type Timestamp)
                T_a = self.dataSeries.index[indexes[i]]
                T_b = self.dataSeries.index[indexes[i+1]]

                # Since the array is circular it may be necessary to sweep the values
                # This guarantees that T_0 is always the minimum and T_1 the maximum of
                # the considered interval
                T_0 = min(T_a, T_b)
                T_1 = max(T_a, T_b)

                msg = "Time {0} and [{1}, {2}]".format(ix, T_0, T_1)
                logger.debug(msg)

                # Measure the difference in seconds between the desired index
                # and the two points
                t_0 = (ix - T_0).total_seconds()
                t_1 = (ix - T_1).total_seconds()

                # Skip transition when restarting from the beginning (j == N-1)
                # If T_0 <= ix <= T_1 the exit
                if j != N-1 and t_0 >= 0 and t_1 <= 0:
                    break
                else:
                    # Otherwise go to the next couple of points
                    j += 1

            # This takes into account that the array is circular and the algorithm may
            # finish in a zone where the order
            if j < N-1:
                index_0 = indexes[i]
                index_1 = indexes[i+1]
            else:
                index_1 = indexes[i]
                index_0 = indexes[i-1]

            msg = "Picked values are {0} : {1}".format(self.dataSeries.values[index_0], self.dataSeries.values[index_1])
            logger.debug(msg)

            # Get distances in seconds to compute the linear interpolation
            deltaT = (self.dataSeries.index[index_1] - self.dataSeries.index[index_0]).total_seconds()
            dT0 = (ix - self.dataSeries.index[index_0]).total_seconds()
            dT1 = (self.dataSeries.index[index_1] - ix).total_seconds()
            interpData = (dT0*self.dataSeries.values[index_1] + dT1*self.dataSeries.values[index_0])/deltaT

            # Save the index
            self.index = j

            return interpData


import bw_conv_cb.python.breitWigner as breitWigner
import bw_conv_cb.python.crystalBall as crystalBall
import numpy as np
from scipy.optimize import minimize
from scipy.special import xlogy
from scipy.signal import fftconvolve
import matplotlib.pyplot as plt

def NLL(a,b):

    nll = xlogy(a,b)
    nll[nll==-np.inf] = 0
    nll = np.sum(nll)/len(nll)

    penalty = xlogy(np.sum(a) - a, 1 - b)
    penalty[penalty==-np.inf] = 0
    penalty = np.sum(penalty)/len(penalty)

    b = np.sum(a)*b/np.sum(b)
    chi_sqr = np.sum(np.divide(np.multiply(a-b,a-b),a))/(len(a)-1)

    return -2*(nll+penalty)*chi_sqr

def target(guess):
    global thisBW
    global thisCB
    global _DATA_

    thisCB.update(guess)
    y_vals = np.convolve(thisBW.y,thisCB.getY(),mode='same')
    y_vals = y_vals/np.sum(y_vals)

    return NLL(_DATA_,y_vals)

def scan(guess):
    global thisBW
    global thisCB
    global _DATA_

    ret = guess

    scan_alpha = np.arange(0.1,100,0.5)
    scan_n = np.arange(0.1,100,0.5)
    scan_mean = np.arange(-5,5,0.025)
    scan_width = np.arange(0.1,10,0.1)

    guesses_mean = [[ret[0],ret[1],x,ret[3]] for x in scan_mean]
    nll_mean = np.array([target(x) for x in guesses_mean])
    ret[2] = scan_mean[np.argmin(nll_mean)]

    guesses_width = [[ret[0],ret[1],ret[2],x] for x in scan_width]
    nll_width = np.array([target(x) for x in guesses_width])
    ret[3] = scan_width[np.argmin(nll_width)]

    guesses_alpha = [[x,ret[1],ret[2],ret[3]] for x in scan_alpha]
    nll_alpha = np.array([target(x) for x in guesses_alpha])
    ret[0] = scan_alpha[np.argmin(nll_alpha)]

    guesses_n = [[ret[0],x,ret[2],ret[3]] for x in scan_n]
    nll_n = np.array([target(x) for x in guesses_n])
    ret[1] = scan_n[np.argmin(nll_n)]

    return ret

def errors(result,min_chi):

    ret = []

    scan_mean = np.arange(-(2*abs(result[2])),2*abs(result[2]),0.0001)
    scan_width = np.arange(0.0001,2*result[3],0.0001)

    par_set_mean = [[result[0],result[1],x,result[3]] for x in scan_mean]
    par_set_width = [[result[0],result[1],result[2],x] for x in scan_width]

    chi_mean = np.array([target(x) for x in par_set_mean])
    chi_width = np.array([target(x) for x in par_set_width])

    sigma_mask_mean = chi_mean >= np.e * min_chi
    sigma_mask_width = chi_width >= np.e * min_chi

    val = []
    for i in range(len(sigma_mask_mean)-1):
        if sigma_mask_mean[i] and not sigma_mask_mean[i+1]:
            val.append(result[0]-scan_mean[i])
        if not sigma_mask_mean[i] and sigma_mask_mean[i+1]:
            val.append(scan_mean[i]-result[0])
    ret.append(val)

    val = []
    for i in range(len(sigma_mask_width)-1):
        if sigma_mask_width[i] and not sigma_mask_width[i+1]:
            val.append(result[0]-scan_width[i])
        if not sigma_mask_width[i] and sigma_mask_width[i+1]:
            val.append(scan_width[i]-result[0])
    ret.append(val)

    return ret

def fit(x,y):
    global thisBW
    global thisCB
    global _DATA_

    #make bw distribution
    _DATA_ = y
    mean = np.average(x,weights=y)-91.188
    width = np.sqrt(np.average(np.multiply(x-np.average(x,weights=y),x-np.average(x,weights=y)), weights=y))
    guess = [1.424,1.86,mean,width]

    bounds = [(0.01,100), (0.01,100), (-5,5), (0.1,10)]

    thisBW = breitWigner.bw(x)
    thisCB = crystalBall.cb(x, guess)
    guess = scan(guess)

    print(guess)
    result = minimize(target,
            np.array(guess),
            method="L-BFGS-B",
            bounds=bounds,
            options={"eps":0.0001}
            )

    print(result)
    thisCB.update(result.x)
    y_vals = np.convolve(thisBW.getY(),thisCB.getY(),mode='same')
    y_vals = np.sum(y)*y_vals/np.sum(y_vals)
    chi_sqr = np.sum(np.divide(np.multiply(y-y_vals,y-y_vals),y))/len(y)
    #uncertainties = errors(result.x, result.fun)
    uncertainties = np.zeros((5,5))

    mu = 91.188+result.x[2]
    dmu =  min(np.abs(uncertainties[0][0]),np.abs(uncertainties[0][1]))

    sigma = result.x[3]
    dsigma = min(np.abs(uncertainties[1][0]),np.abs(uncertainties[1][1]))

    print("minimization complete:")
    print("mu:", 91.188+result.x[2], "+/-", min(np.abs(uncertainties[0][0]),np.abs(uncertainties[0][1]) ))
    print("sigma:", result.x[3], "+/-", min(np.abs(uncertainties[1][0]),np.abs(uncertainties[1][1])))
    print("reduced chi squared:", chi_sqr)
    print()

    return mu-91.188, dmu, sigma, dsigma, chi_sqr

'''
    fig,axs = plt.subplots(ncols=1,nrows=1)
    axs.scatter(x,
            y,
            label='data',
            color='black',
            marker='o')
    axs.plot(x,
            y_vals,
            label='bw conv cb fit',
            color='darkred',
            linestyle='dashed')
    axs.legend(loc='best')
    fig.savefig("fit.png")
    plt.close(fig)
    return
'''


import torch
from torch import nn
from functools import reduce
import operator
import time

MULTI_ADD = 1
# global variables
total_flops = 0
total_params = 0
total_time = 0
module_cnt = 0
verbose = False
# output format control
name_space = 95
param_space = 18
flop_space = 18
time_space = 18
# forward times
forward_num = 10

class Timer(object):
    def __init__(self, verbose=False):
        self.verbose = verbose
        self.start = None
        self.end = None

    def __enter__(self):
        self.start = time.time()
        return self

    def __exit__(self, *args):
        self.end = time.time()
        self.time = self.end - self.start
        if self.verbose: print('Elapsed time: %f ms.' % self.time)

def calc_layer_param(model):
    return sum([reduce(operator.mul, i.size(), 1) for i in model.parameters()])

def profile_forward(layer, input):
    with Timer() as t:
        for _ in range(forward_num):
            layer.old_forward(input)
            if torch.cuda.is_available(): torch.cuda.synchronize()
    return int(t.time * 1e9 / forward_num)

def profile_layer(layer, x):
    global total_flops, total_params, total_time, module_cnt, verbose, MULTI_ADD
    delta_ops, delta_params, delta_time = 0, 0, 0.

    # Conv2d
    if isinstance(layer, nn.Conv2d):
        out_h = int((x.size(2) + 2 * layer.padding[0] - layer.kernel_size[0]) /
                    layer.stride[0] + 1)
        out_w = int((x.size(3) + 2 * layer.padding[1] - layer.kernel_size[1]) /
                    layer.stride[1] + 1)
        delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] *  \
                layer.kernel_size[1] * out_h * out_w / layer.groups * MULTI_ADD
        delta_params = calc_layer_param(layer)
        delta_time = profile_forward(layer, x)
        module_cnt += 1

    # Linear
    elif isinstance(layer, nn.Linear):
        weight_ops = layer.weight.numel() * MULTI_ADD
        bias_ops = layer.bias.numel()
        delta_ops = (weight_ops + bias_ops)
        delta_params = calc_layer_param(layer)
        delta_time = profile_forward(layer, x)
        module_cnt += 1

    # ReLU can be omited
    elif isinstance(layer, nn.ReLU):
        delta_ops = reduce(operator.mul, x.size()[1:])
        delta_time = profile_forward(layer, x)
        module_cnt += 1

    # Pool2d
    elif type(layer) in [nn.AvgPool2d, nn.MaxPool2d]:
        in_w = x.size(2)
        kernel_ops = layer.kernel_size * layer.kernel_size
        out_w = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1)
        out_h = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1)
        delta_ops = x.size(1) * out_w * out_h * kernel_ops
        delta_params = calc_layer_param(layer)
        delta_time = profile_forward(layer, x)
        module_cnt += 1

    # AdaptiveAvgPool2d
    elif isinstance(layer, nn.AdaptiveAvgPool2d):
        delta_ops = x.size(1) * x.size(2) * x.size(3)
        delta_params = calc_layer_param(layer)
        delta_time = profile_forward(layer, x)

    # BatchNorm2d, GroupNorm
    elif type(layer) in [nn.BatchNorm2d, nn.GroupNorm]:
        delta_ops = x.size(1) * x.size(2) * x.size(3)
        delta_params = calc_layer_param(layer)
        delta_time = profile_forward(layer, x)

    # ops ignore flops
    elif type(layer) in [nn.Dropout2d, nn.Dropout]:
        delta_params = calc_layer_param(layer)
        delta_time = profile_forward(layer, x)

    # ignore layer type
    elif type(layer) in [nn.Sequential]: # nn.BatchNorm2d, nn.GroupNorm, nn.ReLU
        return
    else:
        raise TypeError('unknown layer type: %s' % str(layer))

    total_flops += delta_ops
    total_params += delta_params
    total_time += delta_time
    if verbose:
        print(str(layer).ljust(name_space, ' ') +
            '{:,}'.format(delta_params).rjust(param_space, ' ') +
            '{:,}'.format(delta_ops).rjust(flop_space, ' ') +
            '{:,}'.format(delta_time).rjust(time_space, ' '))
    return

def profiling(model, H, W, C=3, B=1, debug=False):
    global total_flops, total_params, total_time, module_cnt, verbose
    total_flops, total_params, module_cnt, verbose = 0, 0, 0, debug
    data = torch.zeros(B, C, H, W)

    def is_leaf(model):
        ''' measure all leaf nodes '''
        return len(list(model.children())) == 0

    def modify_forward(model):
        for child in model.children():
            if is_leaf(child):
                def new_forward(m):
                    def lambda_forward(x):
                        profile_layer(m, x)
                        return m.old_forward(x)
                    return lambda_forward
                child.old_forward = child.forward
                child.forward = new_forward(child)
            else:
                modify_forward(child)

    def restore_forward(model):
        for child in model.children():
            # leaf node
            if is_leaf(child) and hasattr(child, 'old_forward'):
                child.forward = child.old_forward
                child.old_forward = None
            else:
                restore_forward(child)

    def line_breaker():
        print(''.center(name_space + param_space +
                        flop_space + time_space, '-'))

    print('Item'.ljust(name_space, ' ') +
        'params'.rjust(param_space, ' ') +
        'flops'.rjust(flop_space, ' ') +
        'nanosecs'.rjust(time_space, ' '))
    if verbose: line_breaker()
    modify_forward(model)
    model.forward(data)
    restore_forward(model)
    if verbose:
        line_breaker()
        print('Total'.ljust(name_space, ' ') +
            '{:,}'.format(total_params).rjust(param_space, ' ') +
            '{:,}'.format(total_flops).rjust(flop_space, ' ') +
            '{:,}'.format(total_time).rjust(time_space, ' '))

    return total_params, total_flops, total_time


from __future__ import print_function, division

import re
import glob
import yaml
import sqlite3
import numpy as np
from collections import Counter
from nltk import word_tokenize, sent_tokenize

def process_text(body):
    d = Counter([w for s in map(word_tokenize, sent_tokenize(body))
                 for w in s])
    norm = np.sqrt(float(sum(v*v for v in d.values())))
    d = dict((k, c / norm) for k, c in d.iteritems())
    return d

def cosine_similarity(c1, c2):
    if len(c1) > len(c2):
        c1, c2 = c2, c1
    d2 = 0.0
    for w, c in c1.iteritems():
        d2 += c * c2.get(w, 0.0)
    return d2

# Load the license text.
licenses = []
for fn in glob.iglob("../choosealicense.com/_licenses/*.txt"):
    with open(fn, "r") as f:
        txt = f.read()
        _, head, body = txt.split("---\n")
    meta = yaml.load(head)
    meta["_CONTENTS"] = process_text(body)

    if meta["title"] == "No License":
        continue

    # Parse it to get the common name.
    nm = meta["title"].strip("v.0123456789").strip()
    if nm == "Public Domain (Unlicense)":
        nm = "Public Domain"
    elif nm.startswith("CC"):
        nm = "Creative Commons"
    elif nm.startswith("GNU "):
        nm = nm[4:]
    elif "BSD" in nm:
        nm = "BSD"
    meta["common_name"] = nm

    licenses.append(meta)

# Get the unique list of common names.
license_names = list(set(l["common_name"] for l in licenses))

# Pre-compile the regular expressions.
readme_search = re.compile(r"\blicense\b", re.I | re.S)
license_search = re.compile("|".join(map(r"(\b{0}\b)".format, license_names)),
                            re.I | re.S)

# # Add the extra columns.
# with sqlite3.connect("data.db") as conn:
#     c = conn.cursor()
#     c.execute("alter table github add column license_common")
#     c.execute("alter table github add column license_name")
#     c.execute("alter table github add column readme_license")
#     c.execute("alter table github add column readme_license_name")

with sqlite3.connect("data.db") as conn:
    c = conn.cursor()
    c.execute("""select rowid, license, readme from github
        where license is not NULL or readme is not NULL""")
    rows = c.fetchall()
    total = float(len(rows))
    print("{0:.0f} total repos".format(total))

    for j, (rowid, license, readme) in enumerate(rows):
        if j % 10000 == 0:
            print(j, j / total)
        nm = None
        license_name = None
        readme_license = 0
        readme_license_name = None
        if license is not None:
            v = process_text(license.strip())
            d = [cosine_similarity(v, li["_CONTENTS"]) for li in licenses]
            i = np.argmax(d)
            license_name = "unknown" if d[i] < 0.7 else licenses[i]["title"]
            nm = "unknown" if d[i] < 0.7 else licenses[i]["common_name"]
            c.execute("update github set license_name=? where rowid=?",
                      (license_name, rowid))

        if readme is not None:
            readme_license = len(readme_search.findall(readme))
            if readme_license > 0:
                base_counts = license_search.findall(readme)
                if len(base_counts):
                    counts = np.array(map(lambda x:
                                          map(lambda y: len(y) > 0, x),
                                          base_counts))
                    ind = np.argmax(np.sum(counts, axis=0))
                    readme_license_name = license_names[ind]
                c.execute("""update github set
                    readme_license=?, readme_license_name=?
                    where rowid=?""", (readme_license, readme_license_name,
                                       rowid))
        if readme_license_name is None:
            c.execute("""update github set
                readme_license=?, readme_license_name=?
                where rowid=?""", (0, None, rowid))

        if nm is None:
            nm = readme_license_name
        if nm is not None:
            c.execute("update github set license_common=? where rowid=?",
                      (nm, rowid))


import datetime
from sqlalchemy.sql.schema import Column
from sqlalchemy import Integer
from sqlalchemy.sql.sqltypes import String, Text, SmallInteger
from herovii.models.base import Base

__author__ = 'bliss'

class NewsOrg(Base):
    __tablename__ = 'hisihi_organization_notice'
    __bind_key__ = 'csu'

    id = Column(Integer, primary_key=True)
    organization_id = Column(Integer, default=0)

    # 新闻类型
    tag = Column(String(45))

    # 新闻标题
    title = Column(String(100), nullable=True)
    content = Column(Text)

    # 新闻是否推送到
    push_to_organization = Column(SmallInteger, default=1)
    create_time = Column(Integer, default=int(datetime.datetime.now().timestamp()))
    update_time = Column(Integer, default=int(datetime.datetime.now().timestamp()))
    status = Column(Integer, default=1)

    def keys(self):
        return (
            'id', 'tag', 'title', 'content',
            'create_time', 'update_time'
        )


import pandas as pd
import os
import re
import datetime

folderPath = './'
videoPath = folderPath + 'Videos/'

# main function
def main():
    timestamps, labels = readCSVFile()
    df = readFileAndCreateDataFrame(timestamps,labels)
    df.to_csv('./result.csv', index=False)

def readCSVFile():
    df = pd.read_csv(folderPath + 'label_data.csv', encoding="ISO-8859-1")
    header = df.columns.tolist()
    firstColumn = df[header[0]]
    secondColumn = df[header[1]]
    return firstColumn, secondColumn

def calculateMinuteToSec(start, end):
    sec = 0
    startDate = datetime.datetime.strptime(start,
                               '%Y%m%d%H%M%S')
    endDate = datetime.datetime.strptime(end,
                               '%Y-%m-%d %H:%M:%S')
    duration = str(endDate - startDate)
    for value in duration.split(':'):
        sec = 60 * sec + int(value)
    return sec

def sortAscendingOrder(array):
    # loop to access each array element
    for i in range(len(array)):

        # loop to compare array elements
        for j in range(0, len(array) - i - 1):

            # compare two adjacent elements
            if array[j]['video'][6:-4] > array[j + 1]['video'][6:-4]:

                # swapping elements if elements are not in the intended order
                temp = array[j]
                array[j] = array[j+1]
                array[j+1] = temp
    return array

def createDataFrame(loca, vid, first, second, seconds):
    data = {'timestamp': first, 'label': second, 'location': loca, 'video_file': vid, 'seconds_into': seconds}
    df = pd.DataFrame(data)
    return df

def readFileAndCreateDataFrame(timestamps, labels):
    locations = []
    video_files = []
    videos_dict= []
    seconds_into = []

    subfolders = os.listdir(folderPath + 'Videos')
    for subfolder in subfolders:
        for root,dirs,filenames in os.walk(videoPath + subfolder):
            # save video and folder name as dict
            for file in filenames:
                video_dict = {}
                video_dict['video'] = file
                video_dict['location'] = subfolder
                videos_dict.append(
                    video_dict
                )
    sortedDictArray = sortAscendingOrder(videos_dict)

    for index, value in enumerate(sortedDictArray):

        if index < len(sortedDictArray) - 1:
            cur_element = value['video'][6:-4]
            next_element = sortedDictArray[index + 1]['video'][6:-4]

            for file in timestamps:
                labeledTimestamp = re.sub('\ |\:|\-', '', str(file))
                # check labeled timestamp is within first video timestamp and second video timestamp
                if labeledTimestamp>= cur_element and labeledTimestamp< next_element:
                    locations.append(value['location'])
                    video_files.append(value['video'])
                    seconds_into.append(calculateMinuteToSec(cur_element, file))
                else:
                    pass
        else:
            for file in timestamps:
                labeledTimestamp = re.sub('\ |\:|\-', '', str(file))
                # check the record is greater than the last video or not
                # (for not losing the data which timestamp is greater than the last video)
                if labeledTimestamp>= next_element:
                    locations.append(value['location'])
                    video_files.append(value['video'])
                    seconds_into.append(calculateMinuteToSec(cur_element, file))
    df = createDataFrame(locations, video_files, timestamps, labels, seconds_into)
    return df

if __name__=="__main__":
    main()


# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# In[2]:

data = pd.read_csv('titanic.csv')
data = data.dropna()
data['age'] = data['age'].astype(int)
data.head()

# In[3]:

data.head()
data.sex = data.sex.map(dict(female=1,male=0))
data.embarked = data.embarked.map(dict(C=0,Q=1,S=2))
data['embarked'] = data['embarked'].astype(int)
data=data.drop('name',axis=1)

# In[4]:

data.head()

# In[5]:

newdata=data

# In[6]:

newdata.head()

# In[7]:

from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report

# In[8]:

def splitdata1(data):
    X=data.values[:,1:8]
    Y=data.values[:,0]
    X_train,X_test,y_train,y_test=train_test_split(X,Y,test_size=0.2,random_state=100)
    return X_train,X_test,y_train,y_test

# In[9]:

X_train1,X_test1,y_train1,y_test1=splitdata1(newdata)

# In[10]:

X_train1

# In[11]:

# Using all features:
# Here, we use all the features of the given dataset to create the decision tree and check the accuracy

# In[12]:

def generateGiniClassifierObject(X_train,X_test,y_train):
    clf_gini = DecisionTreeClassifier(criterion="gini",random_state=100,max_depth=3,min_samples_leaf=5)
    clf_gini.fit(X_train, y_train)
    return clf_gini

# In[13]:

clf1 = generateGiniClassifierObject(X_train1, X_test1, y_train1)

# In[14]:

def findPredictionForY(X_test, clf_object):
    y_pred = clf_object.predict(X_test)
    print("Predicted values:")
    print(y_pred)
    return y_pred

# In[15]:

# Function to calculate accuracy
def findAccuracyOfModel(y_test, y_pred):
    print("Confusion Matrix: ",confusion_matrix(y_test, y_pred))
    print ("Accuracy : ",accuracy_score(y_test,y_pred)*100)
    print("Report : ",classification_report(y_test, y_pred))

# In[16]:

# Using a subset of features:
# Here, we use a subset of the features of the given dataset to create the decision tree and check the accuracy
# Selected subset attributes: survived, pclass, age, sex, sibsp, parch
# Ignored attributes: fare, embarked

# In[17]:

newdata1=data.drop(['fare','embarked'],axis=1)

# In[18]:

newdata1.head()

# In[19]:

def splitdata2(data):
    X=data.values[:,1:6]
    Y=data.values[:,0]
    X_train,X_test,y_train,y_test=train_test_split(X,Y,test_size=0.2,random_state=100)
    return X_train,X_test,y_train,y_test

# In[20]:

X_train2,X_test2,y_train2,y_test2=splitdata2(newdata1)

# In[21]:

X_train2

# In[22]:

clf2=generateGiniClassifierObject(X_train2,X_test2,y_train2)

# In[23]:

# Using a modified subset of features (Changing the values of features and adding new attributes):
# Here, we use a modified subset of the features of the given dataset to create the decision tree and check the accuracy
# Modification criteria is:
# new feature family = sibsp + parch
# new feature weighted_class = pclass*2 if pclass =1 ; pclass*3 if pclass =2 ; pclass*4 if pclass =3, etc

# In[24]:

newdata2=data

# In[25]:

newdata2.head()

# In[26]:

newdata2['family']=newdata2['sibsp']+newdata2['parch']

# In[27]:

newdata2['pclass'].unique()

# In[28]:

newdata2.weighted_class=int(0)

# In[29]:

newdata2.loc[newdata2.pclass==1,'weighted_class']=2
newdata2.loc[newdata2.pclass==2,'weighted_class']=6
newdata2.loc[newdata2.pclass==3,'weighted_class']=12

# In[30]:

newdata2['weighted_class'].unique()

# In[31]:

newdata2.head()

# In[32]:

def splitdata3(data):
    X=data.values[:,1:10]
    Y=data.values[:,0]
    X_train,X_test,y_train,y_test=train_test_split(X,Y,test_size=0.2,random_state=100)
    return X_train,X_test,y_train,y_test

# In[33]:

X_train3,X_test3,y_train3,y_test3=splitdata3(newdata2)

# In[34]:

X_train3

# In[35]:

clf3=generateGiniClassifierObject(X_train3,X_test3,y_train3)

# In[36]:

def main():
    print('Decision Tree Classifier:\n')
    print('Case 1: Using all attributes:\n')
    y_pred1=findPredictionForY(X_test1,clf1)
    findAccuracyOfModel(y_test1,y_pred1)
    print('--------------------------------\n')
    print('Case 2: Using a subset of attributes:\n')
    y_pred2=findPredictionForY(X_test2,clf2)
    findAccuracyOfModel(y_test2,y_pred2)
    print('--------------------------------\n')
    print('Case 3: Using modified atttributes:\n')
    y_pred3=findPredictionForY(X_test3,clf3)
    findAccuracyOfModel(y_test3,y_pred3)
    print('--------------------------------\n')

# In[37]:

if __name__=='__main__':
    main()


import click
import os

from crawler.crawler import Crawler
from configs import configs
from utils.constants import HELP_CFGS, HELP_URL

# Some random defaults for configuration options.
default_config = configs.DEFAULT_CONFIGS
default_agent = default_config['user_agent']
default_search_method = default_config['traversal']
default_max_depth = default_config['max_depth']
default_max_pages = default_config['max_total']

# The entry point for our CLI.
@click.group()
def main():
    pass

# Sub-command for actually crawling the website.
@main.command(name='crawl')
# @click.option('--a', default=default_agent, help=HELP_AGENT)
# @click.option('--m', default=default_search_method, help=HELP_SEARCH_METHOD)
# @click.option('--depth', default=default_max_depth, help=HELP_MAX_DEPTH)
# @click.option('--pages', default=default_max_pages, help=HELP_MAX_PAGES)
@click.argument('url', type=click.STRING, metavar='<url>', required=True)
@click.argument('cfgs', type=click.STRING, metavar='<cfgs>', required=True)
def crawl(url, cfgs):
    click.secho("Crawler will begin on '{url}' with below settings:\n".format(url=url), fg='green')
    config = configs.load_config_section(config_section=cfgs)
    if config is None:
        print(f"Invalid config {cfgs}. Switching to DEFAULT.")
        config = configs.load_config_section(config_section='DEFAULT')
    else:
        print(f"Config set {cfgs} loaded.")
    click.echo()
    crawler = Crawler()
    print(f"Target URL = {url}")
    crawler.crawl(url, config['traversal'], config['user_agent'], int(config['max_depth']), int(config['max_total']))

# Sub-command that displays information about the project.
@main.command(name='info')
def info():
    # Gets the width of the terminal.
    terminal_width = os.get_terminal_size().columns

    # Project information and metadata.
    title = 'kumo [クモ]'.center(terminal_width)[:-2]
    description = 'A configurable, efficient web crawler and form brute-forcer written in Python.\n'.center(terminal_width)
    version = 'version: 0.1'.center(terminal_width)
    authors = 'authors: Johnny So, Andy Liang, Stanley Lim, Mankirat Gulati'.center(terminal_width)
    github = 'github: http://github.com/devhid/kumo'.center(terminal_width)

    # Output to STDOUT with ANSI coloring.
    click.secho(title, fg='green')
    click.secho(description, fg='blue')
    click.secho(version, fg='yellow')
    click.secho(authors, fg='yellow')
    click.secho(github, fg='yellow')

if __name__ == "__main__":
    main()


from utils import hex_string_to_bytearray, xor
from utils import alpha_and_printable_heuristic
from single_byte_xor_cipher import xor_cipher, brute_force_xor
from operator import itemgetter

def brute_force_messages(messages, heuristic_func):
    for message in messages:
        yield from brute_force_xor(message, heuristic_func)

def detect_xor_cipher(messages, heuristic_func):
    candidates = brute_force_messages(messages, heuristic_func)
    return max(candidates, key=itemgetter(0))

def main():
    with open("4.txt", 'r') as f: lines = f.readlines()
    hex_lines = [l.replace("\n", "") for l in lines]
    ciphertexts = [hex_string_to_bytearray(i) for i in hex_lines]

    score, message, key = detect_xor_cipher(ciphertexts, alpha_and_printable_heuristic)
    print("challenge 1.4 message: %s" % message)

if __name__ == '__main__':
    main()