import os
import sys
import argparse
import logging as log
import numpy as np
import networkx as nx
import MDAnalysis as mda
import itertools
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
from mpl_toolkits.axes_grid1 import make_axes_locatable
import matplotlib

def build_graph(fname, pdb = None):
    """Build a graph from the provided matrix"""

    try:
        adj_matrix = np.loadtxt(fname)
    except:
        errstr = f"Could not load file {fname} or wrong file format."
        raise ValueError(errstr)
    # if the user provided a reference structure
    if pdb is not None:
        try:
            # generate a Universe object from the PDB file
            u = mda.Universe(pdb)
        except FileNotFoundError:
            raise FileNotFoundError(f"PDB not found: {pdb}")
        except:
            raise Exception(f"Could not parse pdb file: {pdb}")
        # generate identifiers for the nodes of the graph
        identifiers = [f"{r.segment.segid}{r.resnum}" for r in u.residues]
        residue_names = [r.resname for r in u.residues]
    # if the user did not provide a reference structure
    else:
        # generate automatic identifiers going from 1 to the
        # total number of residues considered
        identifiers = [f"_{i}" for i in range(1, adj_matrix.shape[0]+1)]
        residue_names = None

    # generate a graph from the data loaded
    G = nx.Graph(adj_matrix)
    # set the names of the graph nodes (in place)
    node_names = dict(zip(range(adj_matrix.shape[0]), identifiers))
    nx.relabel_nodes(G, mapping = node_names, copy = False)
    # return the idenfiers and the graph
    return identifiers, residue_names, G

def convert_input_to_list(user_input, identifiers):
    """Take in a string (e.g. A12:A22,A13... if a PDB file is supplied)
    and a list of names of all the residues (graph nodes). Replaces the
    range indicated by the colon with all residues in that range and
    keeps all residues separated by commas. Removes duplicates.
    """

    res_list = []
    # Split by comma and then colon to make list of list
    split_list = [elem.split(':') for elem in user_input.split(',')]
    for sub_list in split_list:
        # If list size is one, word is separated by comma
        if len(sub_list) == 1:
            try:
                res = sub_list[0]
                # Find index
                identifiers.index(res)
                # Append to list
                res_list.append(res)
            except:
                raise ValueError(f"Residue not in PDB: {res}")
        # If list size is 2, word is separated by a single colon
        elif len(sub_list) == 2:
            try:
                # Get indexes for residue in identifiers
                u_idx, v_idx = [identifiers.index(res) for res in sub_list]
            except:
                raise ValueError(f"Residue range not in PDB: {':'.join(sub_list)}")
            # Check if order of residues is reversed
            if u_idx >= v_idx:
                raise ValueError(f"Range not specified correctly: {':'.join(sub_list)}")
            try:
                # This block should not cause an error
                # Create list with all residues in that range
                res_range = identifiers[u_idx:v_idx + 1]
                # Concatenate with output list
                res_list += res_range
            except Exception as e:
                raise e
        # Other list sizes means multiple colons
        else:
            err_str = f"Incorrect format, only one ':' allowed: {':'.join(sub_list)}"
            raise ValueError(err_str)
    # Remove duplicates
    res_list = list(set(res_list))
    return res_list

def get_persistence_graph(graph, paths, identifiers):
    """Takes in a PSN graph, a list of paths and a list of nodes in the
    PSN and returns a new graph of paths where each edge corresponds to
    persistence value of the original PSN. All nodes from the PSN are
    also included in the output graph.
    """

    pers_graph = nx.Graph()
    # Add all nodes
    pers_graph.add_nodes_from(identifiers)
    # Add all paths
    for p in paths:
        nx.add_path(pers_graph, p)
    # Add all edge weights
    edge_weights = [(u, v, graph[u][v]['weight']) for u, v in pers_graph.edges()]
    pers_graph.add_weighted_edges_from(edge_weights)
    return pers_graph

def get_shortest_paths(graph, source, target):
    """Find all shortest paths between all combinations of source and
    target.
    """

    # Get all combinations
    combinations = itertools.product(source, target)
    # Get all shortest paths
    paths = []
    for node1, node2 in combinations:
        if node1 != node2:
            try:
                # Get a list of shortest paths and append to output
                path = list(nx.algorithms.shortest_paths.generic.all_shortest_paths(
                                G = graph,
                                source = node1,
                                target = node2))
                for p in path:
                    paths.append(p)
            except nx.NetworkXNoPath:
                # If no path is found log info
                log.warning(f"No path found between {node1} and {node2}")
    return paths

def get_all_simple_paths(graph, source, target, maxl):
    """Find all simple paths between all combinations of source and
    target.
    """

    # Get all combinations
    combinations = itertools.product(source, target)
    # Get all simple paths
    paths = []
    for node1, node2 in combinations:
        # Get all simple paths for each combination of source and target
        path = list(nx.algorithms.simple_paths.all_simple_paths(G = graph,
                                                                source = node1,
                                                                target = node2,
                                                                cutoff = maxl))
        # Only add paths to output if they exist
        for p in path:
            if len(p) > 0:
                paths.append(p)
    return paths

def sort_paths(graph, paths, sort_by):
    """Takes in a list of paths and returns a table of sorted paths"""

    # Get source and target
    source = [p[0] for p in paths]
    target = [p[-1] for p in paths]
    # Calculate length of path
    lengths = [len(p) for p in paths]
    # Calculate weights of path
    weights = [[graph[p[i]][p[i+1]]["weight"] for i in range(len(p)-1)] \
                for p in paths]
    sum_weights = [np.sum(w) for w in weights]
    avg_weights = [np.mean(w) for w in weights]
    # Sort paths
    paths = zip(paths, source, target, lengths, sum_weights, avg_weights)
    if sort_by == "path":
        key = lambda x: x[0]
        reverse = False
    elif sort_by == "length":
        key = lambda x: x[3]
        reverse = False
    elif sort_by == "cumulative_weight":
        key = lambda x: x[4]
        reverse = True
    elif sort_by == "average_weight":
        key = lambda x: x[5]
        reverse = True
    sorted_paths = sorted(paths, key = key, reverse = reverse)
    return sorted_paths

def write_table(fname, table):
    """Save sorted table as txt file. """

    # Remove any file extensions
    fname = os.path.splitext(fname)[0]
    # For each line in table, add line in file
    with open(f"{fname}.txt", "w") as f:
        # Write the table header
        f.write("path\tsource\ttarget\tlength\tsum_weights\tavg_weight\n")
        # For each path found
        for path, source, target, length, sum_weights, avg_weight in table:
            path = ','.join(path)
            line = f"{path}\t{source}\t{target}\t{length}" \
                   f"\t{sum_weights}\t{avg_weight}\n"
            f.write(line)

def get_combinations(res_id, res_space):
    """ Takes in a list of residue identifiers and returns all pairs of
    residues that are at least res_space apart if they are on the same
    chain.
    """

    # Get all index combinations
    combinations = itertools.combinations(range(len(res_id)), 2)
    # Get all residue combinations if they are res_space distance apart
    # Or they are on different chains
    combinations = [(res_id[idx1], res_id[idx2]) for idx1, idx2 in combinations \
                    if abs(idx1 - idx2) >= res_space \
                    or res_id[idx1][0] != res_id[idx2][0]]
    return combinations

def get_all_shortest_paths(graph, res_id, res_space):
    """Find all shortest paths between all combinations of nodes in the
    graph that are at least res_space distance apart.
    """

    # Get all combinations
    combinations = get_combinations(res_id, res_space)
    # Get all shortest paths
    paths = []
    for node1, node2 in combinations:
        try:
            # Get a list of shortest paths and append to output
            path = list(nx.algorithms.shortest_paths.generic.all_shortest_paths(\
                            G = graph,
                            source = node1,
                            target = node2))
            for p in path:
                # Add path to output if it contains more than one edge
                if len(p) > 2:
                    paths.append(p)
        except Exception:
            # If no path is found log info
            log.debug(f"No path found between {node1} and {node2}")
    return paths

def get_graph_from_paths(paths):
    """Takes in a list of paths and returns a corresponding graph where
    the weight of each edge is its frequency and the weight of each node
    is its frequency.
    """

    # Increment amount
    inc = 1/len(paths)
    # Build graph
    graph = nx.Graph()
    for path in paths:
        for i in range(len(path) - 1):
            # Get edge
            node1 = path[i]
            node2 = path[i+1]
            # If the node already exists
            if graph.has_edge(node1, node2):
                # Always add first node
                if i == 0:
                    graph.nodes()[node1]["n_weight"] += inc
                # Increment second node weight
                graph.nodes()[node2]["n_weight"] += inc
                # Increment edge weight
                graph[node1][node2]["e_weight"] += inc
            # If edge does not exist
            else:
                # One of the nodes may exist, always check first node
                if i == 0:
                    # Increment node if present, else add
                    if node1 in graph:
                        graph.nodes()[node1]["n_weight"] += inc
                    else:
                        graph.add_node(node1, n_weight = inc)
                # Increment second node if present, else add
                if node2 in graph:
                    graph.nodes()[node2]["n_weight"] += inc
                else:
                    graph.add_node(node2, n_weight = inc)
                graph.add_edge(node1, node2, e_weight = inc)
    return graph

def filter_graph(graph, node_threshold, edge_threshold):
    """Takes in a graph, and returns a graph which only contains nodes
    and edges from the provided graph that have a weight higher than
    the provided threshold.
    """

    filterd_graph = nx.Graph()
    for u, v, d in graph.edges(data = True):
        # Check that both thresholds are met
        if graph.nodes()[u]["n_weight"] > node_threshold and \
           graph.nodes()[v]["n_weight"] > node_threshold and \
           d["e_weight"] > edge_threshold:
            # Add node weights first
            filterd_graph.add_node(u, n_weight = graph.nodes()[u]["n_weight"])
            filterd_graph.add_node(v, n_weight = graph.nodes()[v]["n_weight"])
            # Add edge weight
            filterd_graph.add_edge(u, v, e_weight = d["e_weight"])
    # Warn if no edges in metapath graph
    if nx.is_empty(filterd_graph):
        log.warning("No metapaths found.")
    return filterd_graph

def normalize_graph(graph):
    """Takes in a graph where all weights ranged between 0 to 1 and
    returns a graph where all weights are normalized (divided by max
    weight) and ranged between 0 and 100.
    """

    normalized_graph = nx.Graph()
    # Get max weights for nodes and edges
    max_edge = max([d["e_weight"] for u, v, d in graph.edges(data = True)])
    max_node = max([d["n_weight"] for n, d in graph.nodes(data=True)])

    for u, v, d in graph.edges(data = True):
        # Add nodes first
        normalized_graph.add_node(u, n_weight = \
                                    graph.nodes()[u]["n_weight"]/max_node*100)
        normalized_graph.add_node(v, n_weight = \
                                    graph.nodes()[v]["n_weight"]/max_node*100)
        # Add edge
        normalized_graph.add_edge(u, v, e_weight = d["e_weight"]/max_edge*100)

    return normalized_graph

def get_metapath(graph, res_id, res_space, node_threshold, edge_threshold, normalize):
    """Takes in a PSN graph where weights are persistence values. Returns
    a graph of the metapath
    """

    # Calculate all shortest paths
    paths = get_all_shortest_paths(graph, res_id, res_space)
    # Create graph from path list
    paths_graph = get_graph_from_paths(paths)
    # Normalize graph
    if normalize:
        paths_graph = normalize_graph(paths_graph)
    # Filter graph
    metapath_graph = filter_graph(paths_graph, node_threshold, edge_threshold)
    return metapath_graph

def plot_graph(fname, graph, hub_num, col_map_e, col_map_n, dpi, node_space, node_size, patch_col):
    """Takes in a graph and saves a pdf of the plot. Also takes in a hub
    cutoff value. Nodes with a larger number of edges than the cutoff
    are highlighted.
    """

    # Get attributes
    weights = [d["e_weight"] for u, v, d in graph.edges(data=True)]
    nodes = np.array([n for n, d in graph.degree(graph.nodes())])
    degrees = np.array([d for n, d in graph.degree(graph.nodes())])
    selection = degrees >= hub_num
    hubs = nodes[selection]
    hubs_deg = degrees[selection]
    non_hubs = nodes[np.logical_not(selection)]
    non_hubs_deg = degrees[np.logical_not(selection)]
    unique_hubs = len(np.unique(hubs_deg))
    # Get positions. Larger k values make the nodes spread out more
    pos = nx.spring_layout(graph, k=node_space, iterations=30)
    # Get cmap for edges
    edge_colors = sns.color_palette(palette = col_map_e, n_colors =  256)
    cmap_e = ListedColormap(edge_colors, name = 'edge_colors', N = 256)
    # Remove border
    _, ax = plt.subplots()
    ax.axis('off')
    # Draw non hubs
    nx.draw_networkx_nodes(graph,
                           pos,
                           nodelist = list(non_hubs),
                           node_size = node_size,
                           node_color = 'white',
                           edgecolors = 'black',
                           label = list(non_hubs_deg))
    # Draw hubs
    if unique_hubs == 1:
        # Color the unique hub gray
        nx.draw_networkx_nodes(graph,
                               pos,
                               nodelist = list(hubs),
                               node_size = node_size,
                               node_color = 'gray',
                               edgecolors = 'black')
        # Add label for single unique hub
        label = matplotlib.patches.Patch(color = patch_col,
                                         label = np.unique(hubs_deg)[0])
        plt.legend(handles=[label], title="Node degree")

    # If more than one unique hub, add colorbar
    if unique_hubs > 1:
        # Use colormap if more than 1 unique hubs
        # Gray scale for nodes (select how many greys to pick)
        node_colors = sns.color_palette(palette = col_map_n,
                                        n_colors = unique_hubs)

        cmap_n = LinearSegmentedColormap.from_list(name = 'node_colors',
                                                   colors = node_colors,
                                                   N = len(node_colors))
        nx.draw_networkx_nodes(graph,
                               pos,
                               nodelist = list(hubs),
                               node_size = node_size,
                               node_color = list(hubs_deg),
                               cmap = cmap_n,
                               edgecolors = 'black',
                               label = list(hubs_deg))
    # Draw edges
    nx.draw_networkx_edges(graph,
                           pos,
                           edge_color = weights,
                           edge_cmap = cmap_e,
                           edge_vmin = 0,
                           edge_vmax= 1,
                           width = 3)
    # Draw labels
    nx.draw_networkx_labels(graph, pos, font_size=9, font_color ='black')
    # Add color bar
    # Resize colorbar
    divider = make_axes_locatable(ax)
    cax_e = divider.append_axes("right", size="5%", pad=0.2)
    # Edge color bar
    sm_e = plt.cm.ScalarMappable(cmap=cmap_e, norm=plt.Normalize(vmin=0,
                                                                 vmax=1))
    cbar_e = plt.colorbar(sm_e, cax_e)
    cbar_e.set_label('Relative recurrence')
    # Node color bar (gray scale)
    if unique_hubs > 1:
        # colorbar start pos and end pos
        # extend past actual range by 0.5 to center ticks
        start = min(hubs_deg) - 0.5
        end = max(hubs_deg) + 0.5
        # shift ticks by 0.5 so they align
        tick_pos = np.arange(start, end) + 0.5
        # Colormap position
        cax_n = divider.append_axes("bottom", size="5%", pad=0.2)
        # Scalar mapper oject
        sm_n = plt.cm.ScalarMappable(cmap=cmap_n,
                                     norm=plt.Normalize(vmin=start,
                                                        vmax=end))
        cbar_n = plt.colorbar(sm_n,
                              cax_n,
                              orientation = 'horizontal',
                              ticks = tick_pos)
        # Add tick labels
        #cbar_n.ax.set_xticklabels(range(lo, hi + 1))
        cbar_n.set_label('Node Degree')
    # Save figure
    plt.savefig(f"{fname}.pdf", dpi = dpi, bbox_inches = 'tight', format = 'pdf')

def reorder_graph(graph, identifiers):
    ordered_graph = nx.Graph()
    ordered_graph.add_nodes_from(identifiers)
    weighted_edges = [(u, v, d["e_weight"]) for u, v, d in graph.edges(data=True)]
    ordered_graph.add_weighted_edges_from(weighted_edges)
    return(ordered_graph)

def generate_metapath_table(graph, identifiers, residue_names, normalize):
    '''The function returns a string formatted as a csv file, containing
    the information about the edges of the metapath graph and their weight.'''

    # Create dictionary of {ids : resnames}
    nodes_dict = dict(zip(identifiers, residue_names))

    # Write header depending on whether the recurrency
    # is normalized or not
    if normalize == True:
        w_label = 'relative_recurrency'
    else:
        w_label = 'recurrency'

    out = f'chain1,resid1,restype1,group1,' \
            f'chain2,resid2,restype2,group2,{w_label}\n'

    # Define chain, resid, and resname for each node
    # of each edge, and the edge weight
    for node1, node2, w in graph.edges(data=True):
        chain1, chain2 = node1[0], node2[0]
        resid1, resid2 = int(''.join([x for x in list(node1) if x.isdigit()])), \
                            int(''.join([x for x in list(node2) if x.isdigit()]))
        restype1, restype2 = nodes_dict[node1], nodes_dict[node2]
        e_weight = w['weight']

        # Write output
        out += f'{chain1},{resid1},{restype1},sidechain,' \
                f'{chain2},{resid2},{restype2},sidechain,{e_weight}\n'

    return out

def main():

    ######################### ARGUMENT PARSER #########################

    description = "Path analysis module for PyInteraph. It allows calculation of " \
                  "shortest or simple paths between a given source and target nodes" \
                  "(see option -l) and the metapath (option -m)."
    parser = argparse.ArgumentParser(description = description)

    i_helpstr = ".dat file matrix."
    parser.add_argument("-i", "--input-dat",
                        dest = "input_matrix",
                        help = i_helpstr,
                        type = str)

    r_helpstr = "Reference PDB file."
    parser.add_argument("-r", "--pdb",
                        dest = "pdb",
                        help = r_helpstr,
                        default = None,
                        type = str)

    # Path analysis
    p_helpstr = "Calculate shortest or all paths between source and target "\
                "(see option -l)."
    parser.add_argument("-p", "--path",
                        dest = "do_path",
                        action = "store_true",
                        default = False,
                        help = p_helpstr)

    l_default = "shortest"
    l_helpstr = f"Type of path to be calculated between source and target. " \
                f"If 'shortest' is used, calculates the shortest paths between " \
                f"the source and target. If an integer is used, calculates " \
                f"all paths with a length smaller than or equal to the integer " \
                f"between the source and target (default: {l_default})."
    parser.add_argument("-l", "--path_length",
                        dest = "path_l",
                        default = l_default,
                        type = str,
                        help = l_helpstr)

    s_helpstr = "Source residues for paths calculation (see option -l)."
    parser.add_argument("-s", "--source",
                        dest = "source",
                        default = None,
                        type = str,
                        help = s_helpstr)

    t_helpstr = "Target residues for paths calculation (see option -l)."
    parser.add_argument("-t", "--target",
                        dest = "target",
                        default = None,
                        type = str,
                        help = t_helpstr)

    b_choices = ["path", "length", "cumulative_weight", "average_weight"]
    b_default = "path"
    b_helpstr = "How to sort pathways in output. Possible choices are: " \
                f"{b_choices} (default: {b_default})."
    parser.add_argument("-b", "--sort-paths",
                        dest = "sort_by",
                        choices = b_choices,
                        default = b_default,
                        help =  b_helpstr)

    a_default = "shortest_paths"
    a_helpstr = f"Output file name for paths calculation (see option -l) " \
                f"(default: {a_default}.txt)."
    parser.add_argument("-a", "--path-output",
                        dest = "p_out",
                        default = a_default,
                        help = a_helpstr)

    # Metapath
    m_helpstr = "Calculate metapath."
    parser.add_argument("-m", "--metapath",
                        dest = "do_metapath",
                        action = "store_true",
                        default = False,
                        help = m_helpstr)

    w_helpstr = f"During metapath calculation, normalize the edge and node " \
                f"weights."
    parser.add_argument("-w", "--normalize-weights",
                        dest = "do_normalize",
                        action = "store_true",
                        default = False,
                        help = w_helpstr)

    g_default  = 2
    g_helpstr = f"During metapath calculation, only calculate paths between " \
                f"residues that are separated by this number of residues or " \
                f"more in the protein backbone. The last residue of a chain " \
                f"and the first residue of the subsequent chain are counted " \
                f"as adjacent residues (default: {g_default})."
    parser.add_argument("-g", "--residue-gap",
                        dest = "res_gap",
                        default = g_default,
                        type = int,
                        help = g_helpstr)

    e_default  = 0.1
    e_helpstr = f"During metapath filtering, only keep edges that occur more " \
                f"frequently than this value (default: {e_default}). Used " \
                f"when the normalization option is turned off."
    parser.add_argument("-e", "--edge-threshold",
                        dest = "edge_thresh",
                        default = e_default,
                        type = float,
                        help = e_helpstr)

    E_default  = 10
    E_helpstr = f"During metapath filtering, only keep edges that occur more " \
                f"frequently than this value (default: {E_default}). Used " \
                f"when the normalization option is turned on."
    parser.add_argument("-E", "--normalized-edge-threshold",
                        dest = "norm_edge_thresh",
                        default = E_default,
                        type = float,
                        help = E_helpstr)

    n_default  = 0.1
    n_helpstr = f"During metapath filtering, only keep nodes that occur more " \
                f"frequently than this value (default: {n_default}). Used  " \
                f"when the normalization option is turned off."
    parser.add_argument("-n", "--node-threshold",
                        dest = "node_thresh",
                        default = n_default,
                        type = float,
                        help = n_helpstr)

    N_default  = 10
    N_helpstr = f"During metapath filtering, only keep nodes that occur more " \
                f"frequently than this value (default: {N_default}). Used " \
                f"when the normalization option is turned on."
    parser.add_argument("-N", "--normalized-node-threshold",
                        dest = "norm_node_thresh",
                        default = N_default,
                        type = float,
                        help = N_helpstr)

    c_default = 3
    c_helpstr = f"During metapath plotting, nodes with a degree higher than " \
                f"this value are designated as hubs (default: {c_default})."
    parser.add_argument("-c", "--hub-cutoff",
                        dest = "hub",
                        default = c_default,
                        type = int,
                        help = c_helpstr)

    k_default = 0.2
    k_helpstr = f"Node spacing during metapath plotting. Higher values cause " \
                f"nodes to be more spread apart (default: {k_default})."
    parser.add_argument("-k", "--node-spacing",
                        dest = "node_space",
                        default = k_default,
                        type = float,
                        help = k_helpstr)

    z_default = 900
    z_helpstr = f"Node size during metapath plotting. Higher values create " \
                f"larger nodes (default: {z_default})."
    parser.add_argument("-z", "--node-size",
                        dest = "node_size",
                        default = z_default,
                        type = int,
                        help = z_helpstr)

    d_default = "metapath"
    d_helpstr = f"Metapath plot name (default: {d_default}.pdf). Output csv and" \
                f"dat matrix file will be written with the same base name."
    parser.add_argument("-d", "--metapath-output",
                        dest = "m_out",
                        default = d_default,
                        help = d_helpstr)

    args = parser.parse_args()

    # Check user input
    if not args.input_matrix:
        # exit if the adjacency matrix was not speficied
        log.error("Graph adjacency matrix must be specified. Exiting ...")
        exit(1)

    # Load file, build graphs and get identifiers for graph nodes
    identifiers, residue_names, graph = build_graph(fname = args.input_matrix,
                                                    pdb = args.pdb)

    # get graph nodes and edges
    nodes = graph.nodes()
    edges = graph.edges()
    # print nodes
    sys.stdout.write("Node list:\n")
    for node in identifiers:
        sys.stdout.write(f"{node}\n")
    sys.stdout.write(f"Graph loaded! {len(nodes)} nodes, {len(edges)} edges\n")
    if args.pdb is None:
        log.warning("No reference PDB file provided.")

    ############################## PATHS ##############################

    if args.do_path:
        # Check if source and target are provided
        if args.source and args.target:
            # Convert user input to a list of nodes
            source_list = convert_input_to_list(user_input = args.source,
                                                identifiers = identifiers)
            target_list = convert_input_to_list(user_input = args.target,
                                                identifiers = identifiers)
        else:
            log.error("Source and target must be provided when calculating paths.")
            exit(1)

        # Choost path type
        if args.path_l == "shortest":
            all_paths = get_shortest_paths(graph = graph,
                                           source = source_list,
                                           target = target_list)
        else:
            # Check integer is provided
            try:
                maxl = int(args.path_l)
            except Exception:
                errstr = "Integer must be specified for calculation of all " \
                         "simple paths. Otherwise use 'shortest'."
                raise ValueError(errstr)
            # Calculate all simple paths:
            all_paths = get_all_simple_paths(graph = graph,
                                             source = source_list,
                                             target = target_list,
                                             maxl = maxl)
            # Change output file name if user forgets to change
            if args.p_out == "shortest_paths":
                args.p_out = f"all_paths_{maxl}"

        # Get persistence graph
        pers_graph = get_persistence_graph(graph = graph,
                                           paths = all_paths,
                                           identifiers = identifiers)
        # Create sorted table from paths
        all_paths_table = sort_paths(graph = graph,
                                     paths = all_paths,
                                     sort_by = args.sort_by)

        # Save all/shortest path table
        sys.stdout.write(f"Saving output: {args.p_out}\n")
        write_table(f"{args.p_out}", all_paths_table)

        # Save all/shortest path matrix
        path_matrix = nx.to_numpy_matrix(pers_graph)
        np.savetxt(f"{args.p_out}.dat", path_matrix)

    ############################# METAPATH ############################

    if args.do_metapath:
        if args.res_gap <= 0:
            err_str = "Residue spacing (option -k, --node-spacing) " \
                      "must be a positive integer. Exiting..."
            log.error(err_str)
            exit(1)

        # Set node and edge cutoff
        if args.do_normalize == True:
            n_threshold, e_threshold = args.norm_node_thresh, args.norm_edge_thresh
        else:
            n_threshold, e_threshold  = args.node_thresh, args.edge_thresh

        # Get metapath graph
        metapath_graph = get_metapath(graph = graph,
                                      res_id = identifiers,
                                      res_space = args.res_gap,
                                      node_threshold = n_threshold,
                                      edge_threshold = e_threshold,
                                      normalize = args.do_normalize)

        # If metapath found
        if not nx.is_empty(metapath_graph):
            # Remove file extensions
            args.m_out = os.path.splitext(args.m_out)[0]
            # Plot graph
            plot_graph(fname = f"{args.m_out}",
                       graph = metapath_graph,
                       hub_num = args.hub,
                       col_map_e = "rocket_r",
                       col_map_n = "gray_r",
                       dpi = 100,
                       node_space = args.node_space,
                       node_size = args.node_size,
                       patch_col = 'gray')

            # Fill metapath graph with nodes for all residues
            metapath_graph = reorder_graph(metapath_graph, identifiers)
            # Create matrix
            sys.stdout.write(f"Saving output: {args.m_out}\n")
            metapath_matrix = nx.to_numpy_matrix(metapath_graph)
            np.savetxt(f"{args.m_out}.dat", metapath_matrix)

            # Save metapath csv
            table = generate_metapath_table(graph = metapath_graph,
                                           identifiers = identifiers,
                                           residue_names = residue_names,
                                           normalize = args.do_normalize)

            with open(f"{args.m_out}.csv", 'w') as fh:
                fh.write(table)

if __name__ == "__main__":
    main()


import torch
import torch.nn as nn
import torch.nn.functional as F
from tqdm import tqdm
import torchvision.utils as vutils
import torch.optim as optim
import numpy as np
import matplotlib.pyplot as plt
from torch import autograd
from torch.autograd import Variable
import torchsnooper
from torchvision.utils import save_image

Tensor = torch.cuda.FloatTensor

# @torchsnooper.snoop()
# This is new
def compute_gradient_penalty(D, real_samples, fake_samples):
    # Random weight term for interpolation between real and fake samples
    alpha = Tensor(np.random.random((real_samples.size(0), 1, 1, 1)))
    # Get random interpolation between real and fake samples
    interpolates = (alpha * real_samples + ((1 - alpha) * fake_samples)).requires_grad_(
        True
    )
    d_interpolates = D(interpolates)
    fake = Variable(Tensor(real_samples.shape[0], 1).fill_(1.0), requires_grad=False)
    # Get gradient w.r.t. interpolates
    gradients = autograd.grad(
        outputs=d_interpolates,
        inputs=interpolates,
        grad_outputs=fake,
        create_graph=True,
        retain_graph=True,
        only_inputs=True,
    )[0]
    gradients = gradients.view(gradients.size(0), -1)
    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
    return gradient_penalty

# @torchsnooper.snoop()
def train(
    args,
    device,
    train_loader,
    epoch,
    netD,
    netG,
    nz,
    ndf,
    nc,
    optimizerD,
    optimizerG,
    batches_done,
    lambda_gp,
):
    device = torch.device("cuda")  # Sending to GPU

    for i, (imgs, _) in tqdm(enumerate(train_loader), 0):
        real_imgs = Variable(imgs.type(Tensor), requires_grad=True)

        # Discriminator

        optimizerD.zero_grad()
        z = Variable(
            Tensor(np.random.normal(0, 1, (imgs.shape[0], 100)))
        )  # 100 -> Latent
        fake_imgs = netG(z)  # Batch

        real_validity = netD(real_imgs)  # real ims
        fake_validity = netD(fake_imgs)  # fake im
        gradient_penalty = compute_gradient_penalty(
            netD, real_imgs.data, fake_imgs.data
        )

        # Adversarial loss
        d_loss = (
            -torch.mean(real_validity)
            + torch.mean(fake_validity)
            + lambda_gp * gradient_penalty
        )
        d_loss.backward()

        optimizerD.step()
        optimizerG.zero_grad()

        # Gen train every n_critic steps

        if i % args.CRITIC_ITERS == 0:
            fake_imgs = netG(z)
            fake_validity = netD(fake_imgs)
            g_loss = -torch.mean(fake_validity)

            g_loss.backward()
            optimizerG.step()

            print(f"D loss: {d_loss.item()} ; G loss: {g_loss.item()}")

            if batches_done % args.log_interval == 0:
                save_image(fake_imgs.data[:25], f"outputs/{batches_done}.png")
        batches_done += args.CRITIC_ITERS


import torch

import sys
sys.path.insert(1, "..")

from consts import TOKEN_MAX_LEN, class_names
from preprocess import Preprocessing

def predict_single_review(review_text: str, preprocessing: Preprocessing, tokenizer, model, device):
    preprocessed_text = preprocessing.preprocess_text(review_text)
    encoded_review = tokenizer.encode_plus(
        preprocessed_text,
        max_length=TOKEN_MAX_LEN,
        add_special_tokens=True,
        return_token_type_ids=False,
        pad_to_max_length=True,
        return_attention_mask=True,
        return_tensors='pt',
        verbose=False
    )
    input_ids = encoded_review['input_ids'].to(device)
    attention_mask = encoded_review['attention_mask'].to(device)

    output = model(input_ids, attention_mask)
    _, prediction = torch.max(output, dim=1)

    print(f'Review text: {review_text}')
    print(f'Processed review text: {preprocessed_text}')
    print(f'Sentiment  : {class_names[prediction]}')


"""
    PyProfiler/tests/test_wraps.py

"""
# import sys
import pytest

from io import SEEK_END

from .functions import Math
from .functions import STREAM
from .functions import addition

add = (Math, "add", "verbose")
product = (Math, "product", "debug")

@pytest.mark.parametrize("cls, name, kwd, k_val, value, expect", [
    (*add, True, [1, 2, 3], 6),
    (*add, False, [1, 2, 3], 6),
    (*add, False, [0, 1, 2, 3], 6),
    (*product, True, [1, 2, 3], 6),
    (*product, False, [1, 2, 3], 6),
    (*product, False, [0, 1, 2, 3], 0),
    (None, addition, "verbose", True, [1, 2, 3], 6),
    (None, addition, "verbose", False, [1, 2, 3], 6),
    (None, addition, "verbose", True, range(11), 55),
    (None, addition, "verbose", False, range(11), 55)
])
def test_wrapped_methods(cls, name, kwd, k_val, value, expect):
    if cls is None:
        # testing functions (not methods within a class)
        assert name(value, **{kwd: k_val}) == expect
    else:
        m = cls(value)
        function = getattr(m, name)
        assert function(**{kwd: k_val}) == expect

@pytest.mark.parametrize("cls, name, kwds, value, attrib", [
    (Math, "new", {"debug": True}, [1, 3, 5], "_n"),
    (Math, "new", {"debug": False}, [1, 3, 5], "_n")
])
def test_wrapped_classmethod(cls, name, kwds, value, attrib):
    """Test that wrapped Classmethods can still access the class it is derived from."""
    previous = getattr(cls, attrib)
    m = cls(value)
    function = getattr(m, name)
    ret = function(value, **kwds)
    assert isinstance(ret, cls)
    assert previous != getattr(cls, attrib)

@pytest.mark.parametrize("cls, name, kwd, k_val, value, expect", [
    (*add, True, [1, 2, 3], 6),
    (*add, False, [1, 2, 3], 6),
    (*add, True, range(11), 55),
    (*add, False, range(11), 55),
    (*product, True, [1, 2, 3], 6),
    (*product, False, [1, 2, 3], 6),
    (*product, True, [4, 5, 6], 120),
    (*product, False, [4, 5, 6], 120),
])
def test_stream(cls, name, kwd, k_val, value, expect):
    # Catalog the length of the previous stream
    start = STREAM.seek(0, SEEK_END)

    # Call the (Profiled) Function
    m = cls(value)
    function = getattr(m, name)
    assert function(**{kwd: k_val}) == expect

    # Now read stream
    STREAM.seek(start)  # reset cursor to initial
    output = STREAM.read()

    if k_val:
        line_1 = f"Profiling {cls.__name__}.{name}()\n"
        assert line_1 in output
        assert "Ordered by: cumulative time" in output
        assert "ncalls  tottime  percall  cumtime  percall filename:lineno(function)" in output
    else:
        # Nothing is output to the stream if function is not profiled
        assert output == ""


# import libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import random

# Tensorflow
import tensorflow as tf
from keras.utils import load_img, img_to_array, to_categorical
from keras.layers import Conv2D, MaxPool2D, Flatten, Dense, Dropout
from keras.optimizers import Adam
from keras import Sequential
from tensorflow.keras.utils import load_img
from skimage import transform
from tensorflow.python.keras.models import load_model

# import Dataset
train_df = pd.read_csv('input/Train.csv')
test_df = pd.read_csv('input/Test.csv')

labels = {0:'Speed limit(20km/h)',
          1:'Speed limit(30km/h)',
          2:'Speed limit(50km/h)',
          3:'Speed limit(60km/h)',
          4:'Speed limit(70km/h)',
          5:'Speed limit(80km/h)',
          6:'End of speed limit(80km/h)',
          7:'Speed limit(100km/h)',
          8:'Speed limit(120km/h)',
          9:'No passing',
          10:'No passing for vehicles over 3.5 metric tons',
          11:'Right-of-way at the next intersection',
          12:'Priority road',
          13:'Yield',
          14:'Stop',
          15:'No vehicles',
          16:'Vehicles over 3.5 metric tons prohibted',
          17:'No entry',
          18:'General caution',
          19:'Dangerous curve to the left',
          20:'Dangerous curve to the right',
          21:'Double curve',
          22:'Bumpy road',
          23:'Slippery road',
          24:'Road narrows on the right',
          25:'Road work',
          26:'Traffic signals',
          27:'Pedestrians',
          28:'Children crossing',
          29:'Bicycles crossing',
          30:'Beware of ice/snow',
          31:'Wild animals crossing',
          32:'End of all speed and passing limits',
          33:'Turn right ahead',
          34:'Turn left ahead',
          35:'Ahead only',
          36:'Go straight or right',
          37:'Go straight or left',
          38:'Keep right',
          39:'Keep left',
          40:'Roundabout mandatory',
          41:'End of no passing',
          42:'End of no passing by vehicles over 3.5 metric tons'}
# getting only the Path and ClassId
train_df = train_df[['ClassId', 'Path']]
test_df = test_df[['ClassId', 'Path']]
# add the complete path
train_df['Path'] = train_df['Path'].apply(lambda x: 'input/' + x)
test_df['Path'] = test_df['Path'].apply(lambda x: 'input/' + x)
# Example of the new path in the data
print(f'The new path of the first image in "train_df" : {train_df.Path[0]}')
print(f'The new path of the first image in "test_df" : {test_df.Path[0]}')
# create a random number
random_nbr = random.randint(1, len(train_df)) # or randint(1, train_df.shape[0]))

# read the images randomly and plot'm
img = plt.imread(train_df.Path[random_nbr]) # read the image from the path
print(f'image shape : {img.shape}') # print the shape of the image
fig, ax = plt.subplots(figsize=(8,4))

ax.imshow(img)
ax.set_title(labels[train_df.ClassId[random_nbr]])
ax.axis('off')
plt.show()
# Split the data into training and validation sets
train_data = train_df.sample(frac=0.8, random_state=0)
valid_data = train_df.drop(train_df.index)

# Preprocess the data
IMG_SIZE = 32
NUM_CLASSES = len(labels)

def preprocess_data(df):
    X = []
    y = []
    for index, row in df.iterrows():
        img = load_img(row.Path, target_size=(IMG_SIZE, IMG_SIZE))
        img_array = img_to_array(img) / 255.0
        X.append(img_array)
        y.append(to_categorical(row.ClassId, num_classes=NUM_CLASSES))
    return np.array(X), np.array(y)

X_train, y_train = preprocess_data(train_data)
X_valid, y_valid = preprocess_data(valid_data)
X_test, y_test = preprocess_data(test_df)
model = Sequential([
    Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=(IMG_SIZE, IMG_SIZE, 3)),
    Conv2D(64, 3, activation='relu'),
    MaxPool2D(2),
    Conv2D(128, 3, activation='relu'),
    Conv2D(64, 3, activation='relu'),
    MaxPool2D(2),
    Flatten(),
    Dropout(0.5),
    Dense(128, activation='relu'),
    Dense(NUM_CLASSES, activation='softmax')
])

model.compile(loss='categorical_crossentropy',
              optimizer=Adam(),
              metrics=['accuracy'])
model.summary()
history = model.fit(X_train,
                    y_train,
                    epochs=10)

loss_train = history.history['loss']
epochs = range(1,10)
plt.plot(epochs, loss_train, 'g', label='Training loss')
plt.title('Training Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()

loss_train = history.history['acc']
epochs = range(1,11)
plt.plot(epochs, loss_train, 'g', label='Training accuracy')
plt.title('Training  accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()

model.save('traffic_model.h5')


"""
Incremental Feature Dependency Discovery
iFDD implementation based on ICML 2011 paper
"""
from copy import deepcopy
import numpy as np

from rlpy.tools import printClass, PriorityQueueWithNovelty
from rlpy.tools import powerset, combinations, add_new_features
from rlpy.tools import plt
from .representation import Representation
import warnings
from collections import defaultdict

__copyright__ = "Copyright 2013, RLPy http://acl.mit.edu/RLPy"
__credits__ = [
    "Alborz Geramifard",
    "Robert H. Klein",
    "Christoph Dann",
    "William Dabney",
    "Jonathan P. How",
]
__license__ = "BSD 3-Clause"
__author__ = "Alborz Geramifard"

class iFDD_feature:
    """
    This object represents a feature used for linear function approximation.
    The feature can be an initial feature or made using
    the conjunction of existing features.
    :params potential: iFDD_Potential to be used, or a single integer passed
        to generate initial feature.

    TODO: refactor these docs.
    members:
        index: is the location of the feature in the feature vector
        f_set: set of features that their conjunction creates this feature.
        p1, p2: parents of the current feature.
                Notice that multiple combinations of parents may lead to
                the same feature. For example given 3 basic features of X, Y, and Z.
                P1=XY and P2=XZ and P1=X and P2=XZ both lead to the new features XYZ.
                Both these parameters are used for visualizations only.
    """

    def __init__(self, potential):
        if isinstance(potential, iFDD_potential):
            self.index = potential.index
            self.f_set = deepcopy(potential.f_set)
            self.p1 = potential.p1
            self.p2 = potential.p2
        else:
            # This is the case where a single integer is passed to generate an
            # initial feature
            index = potential
            self.index = index
            self.f_set = frozenset([index])
            self.p1 = -1
            self.p2 = -1

    def __deepcopy__(self, memo):
        new_f = iFDD_feature(self.index)
        new_f.p1 = self.p1
        new_f.p2 = self.p2
        new_f.f_set = deepcopy(self.f_set)
        return new_f

    def show(self):
        printClass(self)

class iFDD_potential(object):
    """
    This object represents a potential feature that can be promoted
    to a permanent feature.
    The structure of this object is very similar to iFDD feature object,
    except it has a relevance parameter that measures the importance
    of creating a new feature out of this potential feature.
    :param f_set: Set of features it corresponds to [Immutable].
    :param parant1: Parent 1 feature index.
    :param parant2: Parent 2 feature index.
    """

    def __init__(self, f_set, parent1, parent2):
        self.f_set = deepcopy(f_set)
        self.p1 = parent1
        self.p2 = parent2
        # If this feature has been discovered set this to its index else -1
        self.index = -1
        self.cumtderr = 0
        self.cumabstderr = 0
        self.count = 1  # Number of times this feature was visited

    def __deepcopy__(self, memo):
        new_p = iFDD_potential(self.f_set, self.p1, self.p2)
        new_p.cumtderr = self.cumtderr
        new_p.cumabstderr = self.cumabstderr
        return new_p

    def show(self):
        printClass(self)

class iFDD(Representation):
    """ The incremental Feature Dependency Discovery Representation based on
    [Geramifard et al. 2011 ICML paper]. This representation starts with a set of given
    binary features and adds new features as the conjunction of existing features.
    Given n features iFDD can expand the set of features up to 2^n-1 features
    (i.e. conjunction of each subset of n features can be considered as a new feature.
    """

    IS_DYNAMIC = True

    def __init__(
        self,
        domain,
        discovery_threshold,
        initial_representation,
        sparsify=True,
        discretization=20,
        debug=0,
        useCache=False,
        max_batch_discovery=1,
        batch_threshold=0,
        iFDDPlus=True,
        seed=1,
        use_chirstoph_ordered_features=True,
    ):
        """
        :param domain: The problem :py:class:`~rlpy.domains.domain.Domain` to learn.
        :param discovery_threshold: Psi in the paper.
        :param initial_representation: A Representation that provides the initial
            set of features for iFDD.
        :param sparsify: A boolean specifying the use of the trick mentioned in the
            paper so that features are getting sparser with more feature discoveries
            (i.e. use greedy algorithm for feature activation).
        :param discretization: Number of bins used for each continuous dimension.
        :param debug: Print more stuff.
        :param useCache: Use cache for phi(s) or not.
        :param batch_threshold: Minimum value of feature relevance for
            the batch setting.
        :param max_batch_discovery: Number of features to be expanded in the batch
            setting.
        :param iFDDPlus: ICML 11 iFDD would add sum of abs(TD-errors) while the iFDD
            plus uses the abs(sum(TD-Error))/sqrt(potential feature presence count).
        :param use_chirstoph_ordered_features: As Christoph mentioned adding new
            features may affect the phi for all states.
            This idea was to make sure both conditions for generating active
            features generate the same result.
        """
        super().__init__(
            domain, initial_representation.features_num, discretization, seed
        )
        # dictionary mapping initial feature sets to iFDD_feature
        self.iFDD_features = {}
        # dictionary mapping initial feature sets to iFDD_potential
        self.iFDD_potentials = {}
        # dictionary mapping each feature index (ID) to its feature object
        self.featureIndex2feature = {}
        # dictionary mapping  initial active feature set phi_0(s) to its corresponding
        # active features at phi(s). Based on Tuna's Trick to speed up iFDD
        self.cache = {}
        self.discovery_threshold = discovery_threshold
        self.sparsify = sparsify
        self.set_bins_per_dim(domain, discretization)
        self.debug = debug
        self.useCache = useCache
        self.max_batch_discovery = max_batch_discovery
        self.batch_threshold = batch_threshold
        # This is a priority queue based on the size of the features (Largest ->
        # Smallest). For same size features, it is also sorted based on the newest
        # -> oldest. Each element is the pointer to feature object.
        self.sortediFDDFeatures = PriorityQueueWithNovelty()
        self.initial_representation = initial_representation
        self.iFDDPlus = iFDDPlus
        self.addInitialFeatures()
        self.use_chirstoph_ordered_features = use_chirstoph_ordered_features
        # Helper parameter to get a sense of appropriate threshold on the
        # relevance for discovery
        self.maxRelevance = -np.inf

    def phi_non_terminal(self, s):
        """ Based on Tuna's Master Thesis 2012 """
        F_s = np.zeros(self.features_num, "bool")
        F_s_0 = self.initial_representation.phi_non_terminal(s)
        activeIndices = np.where(F_s_0 != 0)[0]
        if self.useCache:
            finalActiveIndices = self.cache.get(frozenset(activeIndices))
            if finalActiveIndices is None:
                # run regular and update the cache
                finalActiveIndices = self.findFinalActiveFeatures(activeIndices)
        else:
            finalActiveIndices = self.findFinalActiveFeatures(activeIndices)
        F_s[finalActiveIndices] = 1
        return F_s

    def findFinalActiveFeatures(self, intialActiveFeatures):
        """
        Given the active indices of phi_0(s) find the final active indices of phi(s)
        based on discovered features
        """
        finalActiveFeatures = []
        k = len(intialActiveFeatures)
        initialSet = set(intialActiveFeatures)

        if 2 ** k <= self.features_num:
            # k can be big which can cause this part to be very slow
            # if k is large then find active features by enumerating on the
            # discovered features.
            if self.use_chirstoph_ordered_features:
                for i in range(k, 0, -1):
                    if len(initialSet) == 0:
                        break
                    # generate list of all combinations with i elements
                    cand_i = [
                        (c, self.iFDD_features[frozenset(c)].index)
                        for c in combinations(initialSet, i)
                        if frozenset(c) in self.iFDD_features
                    ]
                    # sort (recent features (big ids) first)
                    cand_i.sort(key=lambda x: x[1], reverse=True)
                    for candidate, ind in cand_i:
                        # No more initial features to be mapped to extended ones
                        if len(initialSet) == 0:
                            break

                        # This was missing from ICML 2011 paper algorithm.
                        # Example: [0,1,20], [0,20] is discovered, but if [0]
                        # is checked before [1] it will be added even though it
                        # is already covered by [0,20]
                        if initialSet.issuperset(set(candidate)):
                            feature = self.iFDD_features.get(frozenset(candidate))
                            if feature is not None:
                                finalActiveFeatures.append(feature.index)
                                if self.sparsify:
                                    # print "Sets:", initialSet, feature.f_set
                                    initialSet = initialSet - feature.f_set
                                    # print "Remaining Set:", initialSet
            else:
                for candidate in powerset(initialSet, ascending=0):
                    # No more initial features to be mapped to extended ones
                    if len(initialSet) == 0:
                        break

                    # This was missing from ICML 2011 paper algorithm. Example:
                    # [0,1,20], [0,20] is discovered, but if [0] is checked
                    # before [1] it will be added even though it is already
                    # covered by [0,20]
                    if initialSet.issuperset(set(candidate)):
                        feature = self.iFDD_features.get(frozenset(candidate))
                        if feature is not None:
                            finalActiveFeatures.append(feature.index)
                            if self.sparsify:
                                initialSet = initialSet - feature.f_set
        else:
            # Loop on all features sorted on their size and then novelty and
            # activate features
            for feature in self.sortediFDDFeatures.toList():
                # No more initial features to be mapped to extended ones
                if len(initialSet) == 0:
                    break

                if initialSet.issuperset(set(feature.f_set)):
                    finalActiveFeatures.append(feature.index)
                    if self.sparsify:
                        initialSet = initialSet - feature.f_set

        if self.useCache:
            self.cache[frozenset(intialActiveFeatures)] = finalActiveFeatures
        return finalActiveFeatures

    def post_discover(self, s, terminal, a, td_error, phi_s):
        """
        returns the number of added features
        """
        # Indices of non-zero elements of vector phi_s
        activeFeatures = phi_s.nonzero()[0]
        discovered = 0
        for g_index, h_index in combinations(activeFeatures, 2):
            discovered += self.inspectPair(g_index, h_index, td_error)
        return discovered

    def inspectPair(self, g_index, h_index, td_error):
        """
        Inspect feature f = g union h where g_index and h_index are the indices of
        features g and h.
        If the relevance is > Threshold, add it to the list of features.
        Returns True if a new feature is added.
        """
        g = self.featureIndex2feature[g_index].f_set
        h = self.featureIndex2feature[h_index].f_set
        f = g.union(h)
        feature = self.iFDD_features.get(f)
        if not self.iFDDPlus:
            td_error = abs(td_error)

        if feature is not None:
            # Already exists
            return False

        # Look it up in potentials
        potential = self.iFDD_potentials.get(f)
        if potential is None:
            # Generate a new potential and put it in the dictionary
            potential = iFDD_potential(f, g_index, h_index)
            self.iFDD_potentials[f] = potential

        potential.cumtderr += td_error
        potential.cumabstderr += abs(td_error)
        potential.count += 1
        # Check for discovery

        if self.random_state.rand() < self.iFDDPlus:
            relevance = abs(potential.cumtderr) / np.sqrt(potential.count)
        else:
            relevance = potential.cumabstderr

        if relevance >= self.discovery_threshold:
            self.maxRelevance = -np.inf
            self.addFeature(potential)
            return True
        else:
            self.updateMaxRelevance(relevance)
            return False

    def show(self):
        self.showFeatures()
        self.showPotentials()
        self.showCache()

    def updateWeight(self, p1_index, p2_index):
        """
        Add a new weight corresponding to the new added feature for all actions.
        The new weight is set to zero if sparsify = False, and equal to the
        sum of weights corresponding to the parents if sparsify = True
        """
        if self.sparsify:
            new_elem = self.weight[:, p1_index] + self.weight[:, p2_index]
        else:
            new_elem = None
        self.weight = add_new_features(self.weight, new_elem)
        # We dont want to reuse the hased phi because phi function is changed!
        self.hashed_s = None

    def addInitialFeatures(self):
        for i in range(self.initial_representation.features_num):
            feature = iFDD_feature(i)
            # shout(self,self.iFDD_features[frozenset([i])].index)
            self.iFDD_features[frozenset([i])] = feature
            self.featureIndex2feature[feature.index] = feature
            # priority is 1/number of initial features corresponding to the
            # feature
            priority = 1
            self.sortediFDDFeatures.push(priority, feature)

    def addFeature(self, potential):
        # Add it to the list of features
        # Features_num is always one more than the max index (0-based)
        potential.index = self.features_num
        self.features_num += 1
        feature = iFDD_feature(potential)
        self.iFDD_features[potential.f_set] = feature
        # Expand the size of the weight_vec
        self.updateWeight(feature.p1, feature.p2)
        # Update the index to feature dictionary
        self.featureIndex2feature[feature.index] = feature
        # Update the sorted list of features
        # priority is 1/number of initial features corresponding to the feature
        priority = 1 / len(potential.f_set)
        self.sortediFDDFeatures.push(priority, feature)

        # If you use cache, you should invalidate entries that their initial
        # set contains the set corresponding to the new feature
        if self.useCache:
            for initialActiveFeatures in list(self.cache.keys()):
                if initialActiveFeatures.issuperset(feature.f_set):
                    if self.sparsify:
                        self.cache.pop(initialActiveFeatures)
                    else:
                        # If sparsification is not used, simply add the new
                        # feature id to all cached values that have feature set
                        # which is a super set of the features corresponding to
                        # the new discovered feature
                        self.cache[initialActiveFeatures].append(feature.index)
        if self.debug:
            self.show()

    def batch_discover(self, td_errors, phi, states):
        """
        Discovers features using iFDD in batch setting.
        self.batch_threshold is the minimum relevance value for the feature
        to be expanded.
        :param td_errors: p-by-1 (How much error observed for each sample)
        :param phi: n-by-p features corresponding to all samples
            (each column corresponds to one sample).
        """
        maxDiscovery = self.max_batch_discovery
        n = self.features_num  # number of features
        p = len(td_errors)  # Number of samples
        counts = np.zeros((n, n))
        relevances = np.zeros((n, n))
        for i in range(p):
            phiphiT = np.outer(phi[i, :], phi[i, :])
            if self.iFDDPlus:
                relevances += phiphiT * td_errors[i]
            else:
                relevances += phiphiT * abs(td_errors[i])
            counts += phiphiT
        # Remove Diagonal and upper part of the relevances as they are useless
        relevances = np.triu(relevances, 1)
        non_zero_index = np.nonzero(relevances)
        if self.iFDDPlus:
            # Calculate relevances based on theoretical results of ICML 2013
            # potential submission
            relevances[non_zero_index] = np.divide(
                np.abs(relevances[non_zero_index]), np.sqrt(counts[non_zero_index])
            )
        else:
            # Based on Geramifard11_ICML Paper
            relevances[non_zero_index] = relevances[non_zero_index]

        # Find indexes to non-zero excited pairs
        # F1 and F2 are the parents of the potentials
        (F1, F2) = relevances.nonzero()
        relevances = relevances[F1, F2]
        if len(relevances) == 0:
            # No feature to add
            self.logger.debug("iFDD Batch: Max Relevance = 0")
            return False

        if self.debug:
            e_vec = relevances.flatten()
            e_vec = e_vec[e_vec != 0]
            e_vec = np.sort(e_vec)
            plt.ioff()
            plt.plot(e_vec, linewidth=3)
            plt.show()

        # Sort based on relevances
        # We want high to low hence the reverse: [::-1]
        sortedIndices = np.argsort(relevances)[::-1]
        max_relevance = relevances[sortedIndices[0]]
        # Add top <maxDiscovery> features
        self.logger.debug("iFDD Batch: Max Relevance = {0:g}".format(max_relevance))
        added_feature = False
        new_features = 0
        for j in range(len(relevances)):
            if new_features >= maxDiscovery:
                break
            max_index = sortedIndices[j]
            f1 = F1[max_index]
            f2 = F2[max_index]
            relevance = relevances[max_index]
            if relevance > self.batch_threshold:
                # print "Inspecting",
                # f1,f2,'=>',self.getStrFeatureSet(f1),self.getStrFeatureSet(f2)
                if self.inspectPair(f1, f2, np.inf):
                    self.logger.debug(
                        "New Feature %d: %s, Relevance = %0.3f"
                        % (
                            self.features_num - 1,
                            self.getStrFeatureSet(self.features_num - 1),
                            relevances[max_index],
                        )
                    )
                    new_features += 1
                    added_feature = True
            else:
                # Because the list is sorted, there is no use to look at the
                # others
                break
        # A signal to see if the representation has been expanded or not
        return added_feature

    def showFeatures(self):
        print("Features:")
        print("-" * 30)
        print(" index\t| f_set\t| p1\t| p2\t | Weights (per action)")
        print("-" * 30)
        for feature in reversed(self.sortediFDDFeatures.toList()):
            print(
                " %d\t| %s\t| %s\t| %s\t| Omitted"
                % (
                    feature.index,
                    self.getStrFeatureSet(feature.index),
                    feature.p1,
                    feature.p2,
                )
            )

    def showPotentials(self):
        print("Potentials:")
        print("-" * 30)
        print(" index\t| f_set\t| relevance\t| count\t| p1\t| p2")
        print("-" * 30)
        for _, potential in self.iFDD_potentials.items():
            print(
                " %d\t| %s\t| %0.2f\t| %d\t| %s\t| %s"
                % (
                    potential.index,
                    str(np.sort(list(potential.f_set))),
                    potential.relevance,
                    potential.count,
                    potential.p1,
                    potential.p2,
                )
            )

    def showCache(self):
        if self.useCache:
            print("Cache:")
            if len(self.cache) == 0:
                print("EMPTY!")
                return
            print("-" * 30)
            print(" initial\t| Final")
            print("-" * 30)
            for initial, active in self.cache.items():
                print(" %s\t| %s" % (str(list(initial)), active))

    def updateMaxRelevance(self, newRelevance):
        # Update a global max relevance and outputs it if it is updated
        if self.maxRelevance < newRelevance:
            self.maxRelevance = newRelevance
            if self.debug:
                self.logger.debug(
                    "iFDD Batch: Max Relevance = {0:g}".format(newRelevance)
                )

    def getFeature(self, f_id):
        # returns a feature given a feature id
        if f_id in list(self.featureIndex2feature.keys()):
            return self.featureIndex2feature[f_id]
        else:
            print("F_id %d is not valid" % f_id)
            return None

    def getStrFeatureSet(self, f_id):
        # returns a string that corresponds to the set of features specified by
        # the given feature_id
        if f_id in list(self.featureIndex2feature.keys()):
            return str(sorted(list(self.featureIndex2feature[f_id].f_set)))
        else:
            print("F_id %d is not valid" % f_id)
            return None

    def feature_type(self):
        return bool

    def __deepcopy__(self, memo):
        ifdd = iFDD(
            self.domain,
            self.discovery_threshold,
            self.initial_representation,
            self.sparsify,
            self.discretization,
            self.debug,
            self.useCache,
            self.max_batch_discovery,
            self.batch_threshold,
            self.iFDDPlus,
        )
        for s, f in list(self.iFDD_features.items()):
            new_f = deepcopy(f)
            new_s = deepcopy(s)
            ifdd.iFDD_features[new_s] = new_f
            ifdd.featureIndex2feature[new_f.index] = new_f
        for s, p in list(self.iFDD_potentials.items()):
            new_s = deepcopy(s)
            new_p = deepcopy(p)
            ifdd.iFDD_potentials[new_s] = deepcopy(new_p)
        ifdd.cache = deepcopy(self.cache)
        ifdd.sortediFDDFeatures = deepcopy(self.sortediFDDFeatures)
        ifdd.features_num = self.features_num
        ifdd.weight = deepcopy(self.weight)
        return ifdd

class iFDDK_potential(iFDD_potential):
    def __init__(self, f_set, parent1, parent2):
        """
        :param f_set: Set of features it corresponds to [Immutable].
        :param parant1: Parent 1 feature index.
        :param parant2: Parent 2 feature index.
        """
        self.f_set = deepcopy(f_set)
        # If this feature has been discovered set this to its index else -1
        self.index = -1
        self.p1 = parent1
        self.p2 = parent2
        try:
            self.hp_dtype = np.dtype("float128")
        except TypeError:
            self.hp_dtype = np.dtype("float64")
        self.a = np.array(0.0, dtype=self.hp_dtype)  # tE[phi |\delta|] estimate
        self.b = np.array(0.0, dtype=self.hp_dtype)  # tE[phi \delta] estimate
        self.e = np.array(0.0, dtype=self.hp_dtype)  # eligibility trace
        self.n_crho = 0  # rho episode index of last update
        self.last_stats = {}

    def relevance(self, kappa=None, plus=None):
        if plus is None:
            assert kappa is not None
            plus = self.random_state.rand() >= kappa

        if plus:
            return np.abs(self.b) / np.sqrt(self.c)
        else:
            return self.a / np.sqrt(self.c)

    def update_statistics(self, rho, td_error, lambda_, discount_factor, phi, n_rho):
        # phi = phi_s[self.p1] * phi_s[self.p2]
        if n_rho > self.n_crho:
            self.e = 0
        self.e = rho * (lambda_ * discount_factor * self.e + phi)

        self.a += np.abs(td_error) * self.e
        self.b += td_error * self.e
        self.c += phi ** 2

        self.n_crho = n_rho

    def update_lazy_statistics(
        self, rho, td_error, lambda_, discount_factor, phi, y_a, y_b, t_rho, w, t, n_rho
    ):
        # phi = phi_s[self.p1] * phi_s[self.p2]
        if lambda_ > 0:
            # catch up on old updates
            gl = np.power(
                np.array(discount_factor * lambda_, dtype=self.hp_dtype),
                t_rho[n_rho] - self.l,
            )
            sa, sb = self.a.copy(), self.b.copy()
            self.a += self.e * (y_a[self.n_crho] - self.x_a) * np.exp(-self.nu) * gl
            self.b += self.e * (y_b[self.n_crho] - self.x_b) * np.exp(-self.nu) * gl
            if not np.isfinite(self.a):
                self.a = sa
                warnings.warn("Overflow in potential relevance estimate")
            if not np.isfinite(self.b):
                self.b = sb
                warnings.warn("Overflow in potential relevance estimate")
            if n_rho > self.n_crho:
                self.e = 0
                # TODO clean up y_a and y_b
            else:
                self.e *= (
                    gl
                    * (lambda_ * discount_factor) ** (t - 1 - t_rho[n_rho])
                    * np.exp(w - self.nu)
                )

        # updates based on current transition
        # np.seterr(under="warn")
        self.e = rho * (lambda_ * discount_factor * self.e + phi)

        self.a += np.abs(td_error) * self.e
        self.b += td_error * self.e
        # np.seterr(under="raise")
        self.c += phi ** 2
        # save current values for next catch-up update
        self.last_stats["t"] = t
        self.last_stats["y_a"] = y_a[n_rho].copy()
        self.last_stats["y_b"] = y_b[n_rho].copy()
        self.last_stats["w"] = w
        self.n_crho = n_rho

    def __deepcopy__(self, memo):
        new_p = iFDD_potential(self.f_set, self.p1, self.p2)
        for i in ["a", "b", "c", "n_crho", "e", "last_stats"]:
            new_p.__dict__[i] = deepcopy(self.__dict__[i])
        return new_p

def _set_comp_lt(a, b):
    if len(a) < len(b):
        return -1
    elif len(a) > len(b):
        return 1
    l1 = list(a)
    l2 = list(b)
    l1.sort()
    l2.sort()
    for i in range(len(l1)):
        if l1[i] < l2[i]:
            return -1
        if l1[i] > l2[i]:
            return +1
    return 0

class iFDDK(iFDD):
    """iFDD(kappa) algorithm with support for elibility traces
    The iFDD(kappa) algorithm is a stochastic mixture of iFDD and iFDD+
    to retain the best properties of both algorithms.
    """

    def __init__(
        self,
        domain,
        discovery_threshold,
        initial_representation,
        sparsify=True,
        discretization=20,
        debug=0,
        useCache=0,
        kappa=1e-5,
        lambda_=0.0,
        lazy=False,
    ):
        try:
            self.hp_dtype = np.dtype("float128")
        except TypeError:
            self.hp_dtype = np.dtype("float64")
        self.y_a = defaultdict(lambda: np.array(0.0, dtype=self.hp_dtype))
        self.y_b = defaultdict(lambda: np.array(0.0, dtype=self.hp_dtype))
        self.lambda_ = lambda_
        self.kappa = kappa
        self.discount_factor = domain.discount_factor
        self.lazy = lazy  # lazy updating?
        self.w = 0  # log(rho) trace
        self.n_rho = 0  # index for rho episodes
        self.t = 0
        self.t_rho = defaultdict(int)
        super().__init__(
            domain,
            discovery_threshold,
            initial_representation,
            sparsify=sparsify,
            discretization=discretization,
            debug=debug,
            useCache=useCache,
        )

    def episode_terminated(self):
        self.n_rho += 1
        self.w = 0
        self.t_rho[self.n_rho] = self.t

    def showPotentials(self):
        print("Potentials:")
        print("-" * 30)
        print(" index\t| f_set\t| relevance\t| count\t| p1\t| p2")
        print("-" * 30)

        k = sorted(iter(self.iFDD_potentials.keys()), cmp=_set_comp_lt)
        for f_set in k:
            potential = self.iFDD_potentials[f_set]
            print(
                " %d\t| %s\t| %g\t| %d\t| %s\t| %s"
                % (
                    potential.index,
                    str(np.sort(list(potential.f_set))),
                    potential.relevance(plus=True),
                    potential.c,
                    potential.p1,
                    potential.p2,
                )
            )

    def post_discover(self, s, terminal, a, td_error, phi_s, rho=1):
        """
        returns the number of added features
        """
        self.t += 1
        discovered = 0
        plus = self.random_state.rand() >= self.kappa
        # if self.t == 22:
        #    import ipdb; ipdb.set_trace()
        activeFeatures = phi_s.nonzero()[
            0
        ]  # Indices of non-zero elements of vector phi_s
        # print "Active Features:", activeFeatures, phi_s
        if not self.lazy:
            dd = defaultdict(float)
            for g_index, h_index in combinations(activeFeatures, 2):
                # create potential if necessary
                dd[self.get_potential(g_index, h_index).f_set] = (
                    phi_s[g_index] * phi_s[h_index]
                )

            for f, potential in list(self.iFDD_potentials.items()):
                potential.update_statistics(
                    rho, td_error, self.lambda_, self.discount_factor, dd[f], self.n_rho
                )
                # print plus, potential.relevance(plus=plus)
                if potential.relevance(plus=plus) >= self.discovery_threshold:
                    self.addFeature(potential)
                    del self.iFDD_potentials[f]
                    discovered += 1

            return discovered

        for g_index, h_index in combinations(activeFeatures, 2):
            discovered += self.inspectPair(g_index, h_index, td_error, phi_s, rho, plus)

        if rho > 0:
            self.w += np.log(rho)
        else:
            # cut e-traces
            self.n_rho += 1
            self.w = 0
            self.t_rho[self.n_rho] = self.t

        if self.lambda_ > 0:
            self.y_a[self.n_rho] += (
                np.exp(self.w)
                * (self.discount_factor * self.lambda_)
                ** (self.t - self.t_rho[self.n_rho])
                * np.abs(td_error)
            )
            self.y_b[self.n_rho] += (
                np.exp(self.w)
                * (self.discount_factor * self.lambda_)
                ** (self.t - self.t_rho[self.n_rho])
                * td_error
            )
            assert np.isfinite(self.y_a[self.n_rho])
            assert np.isfinite(self.y_b[self.n_rho])

        return discovered

    def get_potential(self, g_index, h_index):
        g = self.featureIndex2feature[g_index].f_set
        h = self.featureIndex2feature[h_index].f_set
        f = g.union(h)
        feature = self.iFDD_features.get(f)

        if feature is not None:
            # Already exists
            return None

        # Look it up in potentials
        potential = self.iFDD_potentials.get(f)
        if potential is None:
            # Generate a new potential and put it in the dictionary
            potential = iFDDK_potential(f, g_index, h_index)
            self.iFDD_potentials[f] = potential
        return potential

    def inspectPair(self, g_index, h_index, td_error, phi_s, rho, plus):
        """
        Inspect feature f = g union h where g_index and h_index are the indices of
        features g and h.
        If the relevance is > Threshold add it to the list of features.
        Returns True if a new feature is added,
        """
        potential = self.get_potential(g_index, h_index)
        phi = phi_s[g_index] * phi_s[h_index]
        potential.update_lazy_statistics(
            rho,
            td_error,
            self.lambda_,
            self.discount_factor,
            phi,
            self.y_a,
            self.y_b,
            self.t_rho,
            self.w,
            self.t,
            self.n_rho,
        )
        # Check for discovery

        relevance = potential.relevance(plus=plus)
        if relevance >= self.discovery_threshold:
            self.maxRelevance = -np.inf
            self.addFeature(potential)
            del self.iFDD_potentials[potential.f_set]
            return 1
        else:
            self.updateMaxRelevance(relevance)
            return 0


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 24 20:05:52 2020

Plots Monte Carlo history for the Ising model
"""
import matplotlib.pyplot as plt
import numpy as np
from analysis import thermal
from tqdm import tqdm
import os

data_path = '../out/'

# Key assumption: betas are the same for every N
# If this is not the case, the code should be adapted
Ns = np.array([20,30,40,50,60])
betas = np.array([float(strbeta[6:]) for strbeta in os.listdir(f'{data_path}20')])
betas.sort()

for N in tqdm(Ns):
    for beta in tqdm(betas):
        e, m = np.loadtxt(f'{data_path}%d/ising_%s' % (N, beta), unpack=True)

        plt.subplot(2,1,1)
        plt.title(f'Monte Carlo history: N = {N}, beta = {beta}')
        plt.xlim(0,500)
        plt.ylabel('E')
        plt.plot(e)

        plt.subplot(2,1,2)
        plt.xlim(0,500)
        plt.ylabel('M')
        plt.plot(m)
        plt.show()


# coding=utf-8
import requests
import os
import fileutil
import re
import time
import json
import sys
from bs4 import BeautifulSoup
from PyQt5.QtWidgets import QInputDialog
from PyQt5.QtCore import QThread, pyqtSignal

class backup_blog(QThread):
    """ 下载日志的线程 """

    show_status_signal = pyqtSignal(str)

    def __init__(self, backup_dir, blog_name, user_id, blog_count, continue_on_failure):
        super().__init__()
        self.blog_source_file = "日志/source_blog.txt"
        self.blog_html_file = "日志/blog.html"

        self.backup_dir = backup_dir
        self.blog_name = blog_name
        self.user_id = user_id
        self.blog_count = blog_count
        self.continue_on_failure = continue_on_failure
        self.continue_index = 0
        if continue_on_failure :
            fileutil.check_and_create(backup_dir + "config.json")
            with open(backup_dir + "config.json","r") as f:
                index_content = f.read().strip()
                if f.read().strip():
                    self.continue_index = int(index_content)

        self.post_blog_data = {
            "callCount": "1",
            "scriptSessionId": "187",
            "c0-scriptName": "BlogBeanNew",
            "c0-methodName": "getBlogsNewTheme",
            "c0-id": "0",
            # ~ "c0-param0": "",
            "c0-param1": "0",
            "c0-param2": "1",
            "batchId": "159937",
        }
        self.headers = {
            "contents-Type": "text/plain",
            "Referer": "http://api.blog.163.com/crossdomain.html?publish_time=20100205",
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
        }

    def run(self):
        try:
            # ~ if self.continue_on_failure is not True:
            self.down_blog_source()
            self.convert_to_html_file()
            self.show_status_signal.emit("备份随笔结束")
        except:
            if self.continue_on_failure :
                fileutil.check_and_create(self.backup_dir + "config.json")
                with open(self.backup_dir + "config.json","r") as f:
                    f.write(str(self.continue_index))
            print(sys.exc_info())
            self.show_status_signal.emit("系统异常:" + str(sys.exc_info()[1]) + "\n下载失败")

    def down_blog_source(self):
        self.show_status_signal.emit("开始下载{}条日志".format(self.blog_count))

        get_blog_url = "http://api.blog.163.com/{}/dwr/call/plaincall/BlogBeanNew.getBlogsNewTheme.dwr".format(
            self.blog_name
        )
        self.post_blog_data["c0-param0"] = self.user_id
        self.post_blog_data["c0-param2"] = self.blog_count

        fileutil.check_and_create(self.backup_dir + self.blog_source_file)
        print(
            "get_blog_url:{},post_blog_data:{}".format(
                get_blog_url, self.post_blog_data
            )
        )
        r = requests.post(get_blog_url, data=self.post_blog_data, headers=self.headers)
        # ~ print(r.text)

        with open(self.backup_dir + self.blog_source_file, "w+") as f:
            f.write(r.text)
            self.show_status_signal.emit("下载日志的原始数据成功\n开始解析原始数据")

    def convert_to_html_file(self):
        fileutil.check_and_create(self.backup_dir + self.blog_html_file)
        with open(self.backup_dir + self.blog_source_file, "r") as f:
            str1 = f.read().strip()
            field_title = {}
            field_title["name"] = "title"
            field_title["pattern"] = r'(?<=title=").+?(?=";)'
            field_link = {}
            field_link["name"] = "link"
            field_link["pattern"] = r'(?<=permalink=").+?(?=";)'
            field_publishTime = {}
            field_publishTime["name"] = "publishTime"
            field_publishTime["pattern"] = r"(?<=publishTime=).+?(?=;)"
            field_accessCount = {}
            field_accessCount["name"] = "accessCount"
            field_accessCount["pattern"] = r"(?<=accessCount=).+?(?=;)"
            field_commentCount = {}
            field_commentCount["name"] = "commentCount"
            field_commentCount["pattern"] = r"(?<=commentCount=).+?(?=;)"
            field_id = {}
            field_id["name"] = "id"
            field_id["pattern"] = r'(?<=id=").+?(?=";)'
            fields = [
                field_title,
                field_link,
                field_publishTime,
                field_accessCount,
                field_commentCount,
                field_id,
            ]

            items = self.analyze_response(str1, fields)
            print("日志数量:{},日志内容:{}".format(len(items), items))

        html_content = "<meta charset='utf-8'>"
        items_count = len(items)
        for i, item in enumerate(items,self.continue_index -1):
            self.continue_index = i
            self.show_status_signal.emit("进度:{}/{},{}".format(i + 1, items_count, item["title"]))
            commentCount = item["commentCount"]
            html_content = (
                html_content
                + "

{}

{},评论({})

{}

".format(
                    item["title"], item["publishTime"], commentCount, item["link"]
                )
            )
            try:
                self.get_single_blog(item, i + 1)
            except:
                self.show_status_signal.emit("系统异常:" + str(sys.exc_info()[1]) + "\n下载失败")
            # ~ if commentCount != "0":
            # ~ self.show_status_signal.emit(
            # ~ "正在获取第{}条日志的{}条评论".format(i + 1, commentCount)
            # ~ )
            # ~ comment_items = get_commet(self.blog_name,item["id"],item["commentCount"],html_content)
            # ~ html_content = append_comment(html_content,comment_items)
        html_content += ""

        fileutil.check_and_create(self.backup_dir + self.blog_html_file)
        with open(self.backup_dir + self.blog_html_file, "w+") as f2:
            f2.write(html_content)
        print("保存html格式的日志成功")

    def convert_timestamp(self, t):
        publish_time_tmp = t[:-3]
        x = time.localtime(int(publish_time_tmp))
        return time.strftime("%Y-%m-%d %H:%M:%S", x)

    def str_decode(self, word):
        return eval("u'" + word + "'")

    def get_single_blog(self, item, item_index):
        link = item["link"]
        blog_title = item["title"]
        url = "http://{}.blog.163.com/{}".format(self.blog_name, link)
        print("url:", url)
        r = requests.get(url)
        html_doc = r.text
        fileutil.check_and_create_dir(self.backup_dir + "日志/source/")
        with open(self.backup_dir + "日志/source/" + blog_title + ".txt", "w+") as f:
            f.write(r.text)
        soup = BeautifulSoup(html_doc, "lxml")
        blog_sep = soup.find("span", class_="blogsep")
        blog_catelog = soup.find("a", class_="fc03 m2a")
        if blog_sep is None :
            print("无权限访问:" + item["title"])
            pass
            return
        print("发表时间:{},分类:{}".format(blog_sep.string, blog_catelog.string))
        blog_body = soup.find_all("div", class_="bct fc05 fc11 nbw-blog ztag")
        if blog_body:
            blog_imgs = BeautifulSoup(str(blog_body[0]), "lxml").find_all("img")
            print("正文:", blog_body[0])

        single_blog_dir = (
            self.backup_dir + "日志/" + blog_catelog.string + "/" + blog_title + "/"
        )
        fileutil.check_and_create_dir(single_blog_dir)
        with open(
            self.backup_dir + "日志/" + blog_catelog.string + "/" + blog_title + ".html",
            "w+",
        ) as f:
            content = "<meta charset='utf-8'>
{}

{} | 分类:  {}

".format(
                blog_title, blog_sep.string, blog_catelog.string
            )
            if blog_body:
                # ~ content += str(blog_body[0])
                str_body = str(blog_body[0])
                # r'(?<=title=").+?(?=";)'
                # ~ r'(?<=src=").+?(?/=\d{5})'
                pattern = re.compile(r'(?<=src=").+?(?=/\d{5})')
                print(pattern.findall(str_body))
                content += re.sub(pattern, blog_title, str_body)
            content += "
阅读({}) |  评论({})".format(
                item["accessCount"], item["commentCount"]
            )
            content += ""
            f.write(content)

            # ~ 下载图片
            if blog_imgs:
                self.show_status_signal.emit(
                    "正在获取日志:{}的{}张图片".format(item["title"], len(blog_imgs))
                )
                for img in blog_imgs:
                    print(img)
                    if img["src"]:
                        img_name = img["src"][img["src"].rindex("/") + 1 :]
                        print("下载图片:", img["src"], img_name)
                        self.show_status_signal.emit(
                            '下载图片:{}'.format(img["src"])
                        )
                        r = requests.get(img["src"])
                        img["src"] = blog_title + "/" + img_name
                        with open(single_blog_dir + img_name, "wb+") as f:
                            f.write(r.content)

    def get_commet(self, item_id, comment_count, html_content):
        get_blog_comment_url = "http://api.blog.163.com/{}/dwr/call/plaincall/FeelingsBeanNew.getRecentFeelingsComment.dwr".format(
            self.blog_name
        )
        self.post_blog_data["c0-methodName"] = "getRecentFeelingsComment"
        self.post_blog_data["c0-param0"] = "string:" + item_id
        self.post_blog_data["c0-param1"] = comment_count
        self.post_blog_data["c0-param2"] = "0"
        print(
            "get_blog_comment_url:{},data:{}".format(
                get_blog_comment_url, self.post_blog_data
            )
        )
        r = requests.post(
            get_blog_comment_url, data=self.post_blog_data, headers=self.headers
        )
        print("评论:", r.text)

        field_content = {}
        field_content["name"] = "content"
        field_content["pattern"] = r'(?<=content=").+?(?=";)'
        field_publishTime = {}
        field_publishTime["name"] = "publishTime"
        field_publishTime["pattern"] = r"(?<=publishTime=).+?(?=;)"
        field_commentCount = {}
        field_commentCount["name"] = "publisherNickname"
        field_commentCount["pattern"] = r'(?<=publisherNickname=").+?(?=";)'
        fields = [field_content, field_publishTime, field_commentCount]

        items = self.analyze_response(r.text, fields)
        print("评论:", items)
        return items

    def append_comment(self, html_content, items):
        for item in items:
            html_content = (
                html_content
                + '

{}

{}

{}

'.format(
                    item["publisherNickname"], item["content"], item["publishTime"]
                )
            )
        print("拼接评论后的html:", html_content)
        return html_content

    def analyze_response(self, text, fields):
        results = []
        itemCounts = 0
        a = []
        for field in fields:
            matchs = re.findall(field["pattern"], text)
            itemCounts = len(matchs)

            matchItem = {}
            matchItem["field_name"] = field["name"]
            matchItem["result"] = matchs
            results.append(matchItem)
        print("itemCounts", itemCounts)
        for i in range(itemCounts):
            b = {}
            for matchItem in results:
                field_name = matchItem["field_name"]
                if field_name == "publishTime":
                    print("i:", i)
                    b[matchItem["field_name"]] = self.convert_timestamp(
                        matchItem["result"][i]
                    )
                else:
                    b[matchItem["field_name"]] = self.str_decode(matchItem["result"][i])
            a.append(b)
        print("a :", len(a))
        return a


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

def bar_graph():
    # data
    time = [10, 20, 30, 60, 180]  # in terms of days
    performance = [(8.2188, 0.6047, 9.0604, 2.6340), (10.8215, 0.6105, 8.0006, 2.6579),
                   (15.3175, 0.6166, 10.3766, 3.5249),
                   (17.1332, 0.5846, 12.1121, 4.2286), (27.4012, 0.5928, 18.0708, 7.6385)]  # in terms of RMSE

    ten = [x for x in performance[0]]
    twenty = [x for x in performance[1]]
    thirty = [x for x in performance[2]]
    sixty = [x for x in performance[3]]
    one_eighty = [x for x in performance[4]]

    lake = [ten[0], twenty[0], thirty[0], sixty[0], one_eighty[0]]
    river = [ten[1], twenty[1], thirty[1], sixty[1], one_eighty[1]]
    waterspring = [ten[2], twenty[2], thirty[2], sixty[2], one_eighty[2]]
    aquifer = [ten[3], twenty[3], thirty[3], sixty[3], one_eighty[3]]
    names = ['lake', 'river', 'waterspring', 'aquifer']  # names of models

    # plotting
    N = len(time)
    barwidth = 0.5
    xloc = np.arange(N)

    p1 = [plt.bar(xloc, lake, width=barwidth, label='Lake'),
          plt.bar(xloc, waterspring, width=barwidth, label='Waterspring'),
          plt.bar(xloc, aquifer, width=barwidth, label='Aquifer'),
          plt.bar(xloc, river, width=barwidth, label='River')]

    # adding labels
    plt.ylabel('Validation RMSE - (lower is better)')
    plt.xlabel('Look Ahead Time - (In Days)')
    plt.title('Time Interval Analysis')
    plt.xticks(xloc, time)
    plt.legend()

    plt.show()

bar_graph()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import apps.rating.validators

class Migration(migrations.Migration):

    dependencies = [
        ('people', '0003_remove_teacher_average_record'),
        ('rating', '0003_auto_20141016_0943'),
    ]

    operations = [
        migrations.CreateModel(
            name='GlobalRating',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('total_evil_value', models.SmallIntegerField(verbose_name='evil value', validators=[apps.rating.validators.validate_rating_value])),
                ('total_easier_value', models.SmallIntegerField(verbose_name='easier value', validators=[apps.rating.validators.validate_rating_value])),
                ('total_vague_value', models.SmallIntegerField(verbose_name='vague value', validators=[apps.rating.validators.validate_rating_value])),
                ('total_brainy_value', models.SmallIntegerField(verbose_name='brainy value', validators=[apps.rating.validators.validate_rating_value])),
                ('average_record', models.DecimalField(default=0, verbose_name='average record', max_digits=5, decimal_places=3)),
                ('teacher', models.OneToOneField(related_name=b'global_rating', to='people.Teacher')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
    ]