#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import networkx as nx
import pandas as pd
import pickle
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
from collections import Counter
import warnings
import os
import time
#import nxviz as nv
from scipy import stats
from datetime import datetime
from sklearn import metrics

warnings.filterwarnings('ignore')  # suppress warning messages
get_ipython().run_line_magic('matplotlib', 'inline')
sns.set()

# In[ ]:

# Import artist_dictionary.data, tags_dict.data, and yahoo ratings.csv
# from working directory.

artist_dictionary = pickle.load(open('artist_dictionary.data','rb'))
tags_dictionary = pickle.load(open('tags_dict.data','rb'))
filtered_tags_dictionary = pickle.load(open('filtered_tags_dict.data','rb'))

ratings = pd.read_csv('artist-yahoo-ratings.csv')
ratings['artist'] = ratings['artist_name'].str.lower()
ratings = ratings.set_index('artist')
# fill missing ratings with 0 and convert column to integer
ratings['yahoo_rating_count'] = ratings['yahoo_rating_count'].fillna(0).astype(int)

spotify_rankings = pd.read_csv('10k_spotify_rankings.csv')

spotify_rankings['Artist'] = spotify_rankings['Artist'].str.lower()

# lower-case all artists - also running at end of cell to make manually adding artists easier

artist_dictionary = {str(key).lower():artist_dictionary[key] for key in artist_dictionary.keys()}

for key in artist_dictionary.keys():
    artist_dictionary[key]=[i.lower() for i in artist_dictionary[key]]

tags_dictionary = {str(key).lower():tags_dictionary[key] for key in tags_dictionary.keys()}

for key in tags_dictionary.keys():
    tags_dictionary[key]=[str(i).lower() for i in tags_dictionary[key]]

filtered_tags_dictionary = {str(key).lower():filtered_tags_dictionary[key] for key in filtered_tags_dictionary.keys()}

for key in filtered_tags_dictionary.keys():
    filtered_tags_dictionary[key]=[str(i).lower() for i in filtered_tags_dictionary[key]]

#remove duplicate values from each key
artist_dictionary = {key:list(set(artist_dictionary[key])) for key in artist_dictionary.keys()}
tags_dictionary = {key:list(set(tags_dictionary[key])) for key in tags_dictionary.keys()}

# In[ ]:

def get_edges(dictionary,label, pop_weight = True, weight = 0):
    """Create a list of edges from a dictionary.

    Create a list of labeled edges for a network graph by comparing the values (artists)
    of each key (band) in a dictionary.  If two values share a key, return a list of
    (key,key,{link:shared value}) tuples.  This function is called by the build_net function to
    generate the edge list for graph construction.

    Parameters
    ----------
    dictionary : dict

    label : str
            Label to be used as key in edge metadata, used to describe node relationship.

    Returns
    -------
    edgelist : list of str and dict

    """
    edgelist_raw = []
    for  key, value in dictionary.items():
        for items in value:
            for key1, value1 in dictionary.items():
                for items1 in value1:
                    if len(items) > 1 and len(items1) > 1: # Prevent single letter artists
                                                         # ('G' or 'K') from linking with
                                                         # every artist with a g or k in their name.

                        if items in items1:
                            if pop_weight == True:
                                # adding to the edge attributes the average of the Yahoo! rating count between two artists

                                #solve for df having multiple entires for artist ratings
                                if key in ratings.index:
                                    rating1 = ratings.loc[key, 'yahoo_rating_count'].astype(int)

                                    # TEMP - there may be multiple artists in the ratings file.  if this is the case,
                                    # select the first value in the series.  Remove when ratings file is consolidated.
                                    if type(rating1) == pd.core.series.Series:
                                        rating1 = list(ratings.loc[key, 'yahoo_rating_count'])[0]
                                else:
                                    rating1 = 1

                                if key1 in ratings.index:
                                    rating2 = ratings.loc[key1, 'yahoo_rating_count'].astype(int)

                                    if type(rating2) == pd.core.series.Series:
                                        rating2 = list(ratings.loc[key1, 'yahoo_rating_count'])[0]
                                else:
                                    rating2 = 1

                                avg_rating = (rating1 + rating2) / 2 # average rating between two connected artists
                                                                     # ex. The Beatles (1000) <--> The Who (500)
                                                                     # = 750 edge weight.

                                norm_rating = avg_rating / 783871    # naive normalize average rating onto a 0 - 1 scale.
                                                                     # 783871 is the highest rating in the Yahoo! data

                               # # check to that the edge does not already exist in edge_list
                               # if (key,key1,{label:items, 'weight':norm_rating}) not in edgelist_raw:

                                edgelist_raw.append((key,key1,{'kind':label, 'link':items, 'weight': norm_rating}))

                            # return edgelist without edge weight passed to function
                            else:
                                edgelist_raw.append((key,key1,{'kind':label, 'link':items, 'weight': weight}))

    # remove duplicate edges from edgelist_raw
    edgelist = []
    edgelist = [i for i in edgelist_raw if i not in edgelist]

    return edgelist

def build_net(seed, goal=None, size=600, graph=True):
    """ Build an artist-based network graph.

    Create an undirected, artist-based network graph with labeled edges. Basic
    graph structure is equivelant to (band1) <-- shared artist --> (band2). By
    default, function will build networks of size 600 (node count / bands.)
    Optionally, the function can also produce a network that starts with a
    band (seed) and builds a network until it contains the a second band (goal.)
    Warning! There is no check for whether two bands reside in the same network.  It is
    possible to specify a seed and goal for which a relationship does not exist.

    Parameters
    ----------
    seed : str
           Seed band around wihch a network graph is built.  Also root of tree search.

    goal : str
            The function will stop growing the network graph once it contains the goal (band).
            Optional, default = None.

    size : int
           Size of the network graph, defined by node count.
           Optional, default = 600 nodes (bands.)

    graph : bool
            True = return graph object, False = return node/edge lists.

    Returns
    -------
    graph : networkX graph object
            if graph=True, return undirected graph

    nodes,edges : list of str
                  if graph=False, return node and edge lists

    """
    artist_list = []
    band_list = []

    artistindex = 0  # Use index markers to keep track of progress and
    bandindex = 0    # avoid iterating over the entire artist and band lists.

    band_list.append(seed)  # Level 1 seed band (root).

    if goal is not None:

        if nx.has_path(seed,goal) == True:  # Check that a path between seed and goal exists.

            while goal not in band_list:
                for band in band_list[bandindex:]:               # Only retrieve artists for 'new' bands
                    for artist in artist_dictionary[band]:       # in band_list.
                        if artist not in artist_list:
                            artist_list.append(artist)           # Add new artists to artist_list.

                bandindex = len(band_list)                       # Mark current bands in band_list as 'worked'
                                                                 # by setting index marker to the first index
                                                                 # following last element in band_list.

                for artist in artist_list[artistindex:]:         # Only retrieve bands for 'new' artists
                    for key, value in artist_dictionary.items(): # in artist_list.
                        if artist in value:
                            if key not in band_list:
                                if goal not in band_list:
                                    band_list.append(key)        # Add new bands to band_list.

                artistindex = len(artist_list)                  # Mark current artists in artist_list as 'worked'
                                                                 # by setting index marker to the first index
        else:                                                    # following the last element in artist_list.

            print('A path between ',seed,' and ',goal,' does not exist.')

    else:
        prev_band_len = 0
        prev_art_len = 0
        go = True                                   # exit var will flip and end loop if no new artists e
                                                    # are added to the band list (see note below)
        while len(band_list) < size and go == True:

            if len(band_list) > prev_band_len or len(artist_list) > prev_art_len:
                                                        # check that the len of bandlist has increased, if not
                prev_art_len = len(artist_list)         # set go = no.  added to prevent infinite loops caused
                prev_band_len = len(band_list)          # by artists who have only worked solo ex Madonna.

                for band in band_list[bandindex:]:
                    for artist in artist_dictionary[band]:
                        if artist not in artist_list:
                            artist_list.append(artist)

                bandindex = len(band_list) -1

                for artist in artist_list[artistindex:]:
                    for key, value in artist_dictionary.items():
                        if artist in value:
                            if key not in band_list:
                                if len(band_list) < size:
                                    band_list.append(key)

                artistindex = len(artist_list) -1

            else:
                go = False

    dictionary = {band : artist_dictionary[band] for band in band_list}  # local artist dictionary

    for key, value in dictionary.items():
        dictionary[key] = set(value)   # filter out duplicate group members.

    nodes = list(dictionary.keys())

    edges = get_edges(dictionary, 'Artist')

    for i in edges:              # remove single edge loops from edge list
            if i[0] == i[1]:     # ex. ('The Who','The Who',{'artist':'Roger Daltrey'})
                edges.remove(i)

    if graph == False:

        return(nodes, edges)

    else:
        net_graph = nx.MultiGraph()
        net_graph.add_nodes_from(nodes)
        net_graph.add_edges_from(edges)
        return(net_graph)

# In[ ]:

def add_tag_edges(graph, just_edges=False):
    """Add user tag-based edges to an existing artist network graph.

    Append user tag-based edges to an existing artist graph and return a
    multigraph. Creates edges in the form of (band1,band2,{link:tag})
    tuple.  This function will not add new nodes to the graph.

    Parameters
    ----------
    graph : graph object
            Network graph from which tags will be retrieved, and tag-edges will be added

    just_edges : bool
                 If just_edges = True, return new, undirected multigraph object.
                 If just_edges = False, return list of tag-based edges.

    Returns
    -------
    new_graph : networkX multigraph object

    tag_edges : list of str

    """

    new_graph = nx.MultiGraph()

    #new_graph.add_nodes_from(graph.nodes()) # add nodes from original graph

    new_graph.add_edges_from(graph.edges.data()) # add edges w/attribute dictionary from original graph

    bands = [node for node in graph.nodes()]

    dictionary = {band : tags_dictionary[band] for band in bands
                  if band in tags_dictionary.keys()}  # Check if band exists as a key in
                                                      # in tags_dictionary.

    #if len(dictionary.keys()) >= 2:                   # comparing keys requires atleast two keys

    tag_edges = get_edges(dictionary,'User-Tag', pop_weight = False, weight = 0.05) #<<<--- hard-coded tag-edge weight.

    for i in tag_edges:          # remove single edge loops from edge list
        if i[0] == i[1]:         # ex. ('The Who','The Who',{'LINK':'classic rock'})
            tag_edges.remove(i)

    if len(tag_edges) > 0:
        if just_edges == True:

            return tag_edges

        else:
            new_graph.add_edges_from(tag_edges)

            return new_graph

def new_centrality(graph):
    """ Compute centrality scores for a network graph.

    Compute a number of different centrality and misc. scores for all nodes in a network graph.

    Parameters
    ----------
    graph : networkX graph object

    Returns
    -------
    core_df : Pandas DataFrame object

    """

    core_df = pd.DataFrame()
    core_df['artist'] = graph.nodes() # Add to the artist column all nodes (artists) in a graph.
    scores_list = []

    try:
        deg_cent = pd.DataFrame.from_dict(nx.degree_centrality(graph), orient = 'index',  columns = ['deg_cent'])
        scores_list.append(deg_cent)
    except:
        pass

    try:
        load_cent = pd.DataFrame.from_dict(nx.load_centrality(graph), orient = 'index',  columns = ['load_cent'])
        scores_list.append(load_cent)
    #between_cent = nx.betweenness_centrality(graph)
    except:
        pass

    try:
        page_rank = pd.DataFrame.from_dict(nx.pagerank_numpy(graph), orient = 'index',  columns = ['page_rank'])
        scores_list.append(page_rank)
    except:
        pass

    try:
        ev_cent = pd.DataFrame.from_dict(nx.eigenvector_centrality_numpy(graph), orient = 'index',  columns = ['ev_cent'])
        scores_list.append(ev_cent)
    except:
        pass

    try:
        cl_cent = pd.DataFrame.from_dict(nx.closeness_centrality(graph), orient = 'index',  columns = ['close_cent'])
        scores_list.append(cl_cent)
    except:
        pass

    try:
        cfcc = pd.DataFrame.from_dict(nx.current_flow_closeness_centrality(graph), orient = 'index',  columns = ['cf_close_cent'])
        scores_list.append(cfcc)
    except:
        pass
    """
    try:
        ic = pd.DataFrame.from_dict(nx.information_centrality(graph), orient = 'index',  columns = ['info_cent'])
        scores_list.append(ic)
    except:
        pass

        #ebc = pd.DataFrame.from_dict(nx.edge_betweenness_centrality(graph), orient = 'index',  columns = ['edge_bet_cent'])

    try:
        cfbc = pd.DataFrame.from_dict(nx.current_flow_betweenness_centrality(graph), orient = 'index',  columns = ['edge_cflow_cent'])
        scores_list.append(cfbc)
    except:
        pass
    #ecfbc = pd.DataFrame.from_dict(nx.edge_current_flow_betweenness_centrality(graph), orient = 'index',  columns = ['cf_between_cent'])

    try:
        acfbc = pd.DataFrame.from_dict(nx.approximate_current_flow_betweenness_centrality(graph), orient = 'index',  columns = ['appx.cfbt_cent'])
        scores_list.append(acfbc)
    except:
        pass
    #elc = pd.DataFrame.from_dict(nx.edge_load_centrality(graph), orient = 'index',  columns = ['edge_load_cent'])
    """
    try:
        hc = pd.DataFrame.from_dict(nx.harmonic_centrality(graph), orient = 'index',  columns = ['harm_cent'])
        scores_list.append(hc)
    except:
        pass
    #d = pd.DataFrame.from_dict(nx.dispersion(graph), orient = 'index',  columns = ['dispersion'])
    """
    try:
        soc = pd.DataFrame.from_dict(nx.second_order_centrality(graph), orient = 'index',  columns = ['sec_ord_cent'])
        scores_list.append(soc)
    except:
        pass
    """
    df = pd.concat(scores_list, axis = 1)

    core_df = core_df.merge(df,
                            left_on = 'artist',
                            right_index = True)

    core_df['mean_cent'] = core_df.apply(lambda row: np.mean(row[1:]),   #  Calculate the mean of the row
                                    axis = 1)
    return core_df

# In[ ]:

def layer_graphs(artists, add_tags = True,  size = 600):
    """Layer n artist networks together.

    Layer together multiple artist-based graphs, add tag-based edges,
    and return a single multigraph.

    Parameters
    ----------
    args : str
           Band(s) around which network graphs will be built and a layered
           together.
    size : int
           Size in # of nodes for the individual artist networks.

    Returns
    -------
    graph : networkX multigraph object

    """
    layered_graph = nx.MultiGraph()

    for band in artists:
        if band in artist_dictionary.keys():

            artist_graph = build_net(band, size = size)  # Build the artist-based network

            layered_graph.add_edges_from(artist_graph.edges.data())  # Add edges to layered_graph, nodes
                                                                # will be added if they do not already exist.
        else:
            print(band, ' not in artist_dictionary')

    if add_tags == True:
        layered_graph = add_tag_edges(layered_graph)  # Add tag-based edges to layered_graph and
                                                  # connect the artist graph layers.

    return layered_graph

# In[ ]:

"""  This is a catch-all section for manually adding missing artists and bands
to the artist dictionary.  They will eventually be moved intoartist_dictionary.data.
One day..."""

artist_dictionary['bob seger'] = ['bob seger']
artist_dictionary['tom petty'] = ['tom petty']
artist_dictionary['phil collins'] = ['phil collins']
artist_dictionary['madonna'] = ['madonna']
artist_dictionary['the english beat'] = artist_dictionary['the beat']
artist_dictionary['gloria estefan'] = ['gloria estefan']
artist_dictionary['don henley'] = ['don henley']
artist_dictionary['daryl hall & john oates'] = artist_dictionary['hall & oates']
artist_dictionary['boy george'] = ['boy george']
artist_dictionary['a-ha'] = ['magne furuholmen', 'morten harket','paul waaktaar-savoy']
artist_dictionary['huey lewis & the news'] = artist_dictionary['huey lewis and the news']
artist_dictionary['ilse delange'] =  ['ilse delange']
artist_dictionary['chris isaak'] = ['chris isaak']
artist_dictionary['vanessa carlton'] = ['vanessa carlton']
artist_dictionary['alicia keys'] = ['alicia keys']
artist_dictionary['adele'] = ['adele']
artist_dictionary['amy winehouse'] = ['amy winehouse']
artist_dictionary['rihanna'] = ['rihanna']
artist_dictionary['marco borsato']= ['marco borsato']
artist_dictionary['mary j. blige'] = ['mary j. blige']
artist_dictionary['amy macdonald'] = ['amy macdonald']
artist_dictionary['tom jones'] = ['tom jones']
artist_dictionary['george michael'] = ['george michael']
artist_dictionary['james morrison'] = ['james morrison']
artist_dictionary['iggy pop'] = ['iggy pop']
artist_dictionary['joan osborne'] = ['joan osborne']
artist_dictionary['anti-flag'] = ['andy wright', 'justin sane', 'pat thetic','chris head']
artist_dictionary['kayzo'] = ['kayzo']
artist_dictionary['fame on fire'] = ['bryan kuznitz', 'blake saul', 'paul spirou', 'alex roman']
artist_dictionary['rev theory'] = ['rich luzzi', 'julien jorgensen','rikki lixx','matt mccloskey','dave agoglia']
artist_dictionary['the anix'] = ['brandon smith', 'logan smith','chris dinger']
artist_dictionary['ded'] = ['joe cotela', 'kyle koelsch', 'david ludlow', 'matt reinhard']
artist_dictionary['blind channel'] = ['joel hokka', 'niko moilanen', 'joonas porko', 'olli matela', 'tommi lalli']
artist_dictionary['mammoth mammoth'] = ['mikey tucker', 'ben couzens', 'frank trobbiani','pete bell','gareth sweet','simon jaunay','marco gennaro','kris sinister']
artist_dictionary['dayseeker'] = ['rory rodriguez', 'mike karle', 'alex polk', 'gino sgambelluri','andrew sharp']
artist_dictionary['dream state'] = ['charlotte gilpin', 'aled evans', 'rhys wilcox', 'danny rayer', 'sam harrison-little', 'jamie lee']
artist_dictionary['wage war'] = ['briton bond', 'cody quistad',  'seth blake', 'jordan pierce', 'david rau', 'chris gaylord', 'stephen kluesener']
artist_dictionary['soulja boy'] = ['']
artist_dictionary['fergie'] = ['fergie']
artist_dictionary['pitbull'] = ['pitbull']
artist_dictionary['ne-yo'] = ['ne-yo']
artist_dictionary['jessie j'] = ['jessie j']
artist_dictionary['nicki minaj'] = ['nicki minaj']
artist_dictionary['jennifer lopez'] = ['jennifer lopez']
artist_dictionary['chris brown'] = ['chris brown']
artist_dictionary['justin bieber'] = ['justin bieber']
artist_dictionary['taio cruz'] = ['taio cruz']
artist_dictionary['jason derulo'] = ['jason derulo']
artist_dictionary['flo rida'] = ['flo rida']
artist_dictionary['iyaz'] = ['iyaz']
artist_dictionary['black eyed peas'] = artist_dictionary['the black eyed peas']
artist_dictionary['alexandra stan'] = ['alexandra stan']
artist_dictionary['david guetta'] = ['david guetta']
artist_dictionary['fidlar'] = ['zac carper', 'elvis kuehn', 'max kuehn','brandon schwartzel']

artist_dictionary["swingin' utters"] = ['kevin wickersham','greg mcentee', 'darius koski', 'max huber', 'spike slawson','jack dalrymple', 'miles peck']
artist_dictionary['the 4-skins'] = ['steve harmer','gary hodges','tom mccourt']
artist_dictionary['4 skins'] = artist_dictionary['the 4-skins']
artist_dictionary['jfa']=['Mike Tracy','Brian Brannon','Don Pendleton','Corey Stretz','Todd Barnes',                          'Alan Bishop','Scott Chazan','Michael Cornelius','Bob Cox','Brian Damage',                          'Joel DuBois','Trace Element','Matt Etheridge','Jim Moore','Don Pendelton',                          'Al Penzone','Jaime Reidling','Mike Sversvold','Bruce Taylor','Mike Tracy']
artist_dictionary['nihilistics'] = ['Dave Phinster','Mike King','Ron Rancid','Ajax Lepinski']
artist_dictionary['the neighborhoods'] = ['Dave Minehan','John Hartcorn', 'Michael Quaglia']
artist_dictionary['nobodys'] = ['Geoff Palmer', 'J. J. Nobody', 'Justin Disease', 'Randy The Kid']
artist_dictionary['china white'] = ['Marc Martin','James Rodriguez','Joey Ruffino','Frank Ruffino',                                    'Scott Sisunik','Richard Katchadoorian','Vince Mesa','Corey Stretz',                                    'Steven Barrios','Jeff Porter','James Lugo','Sandy Hancock']
artist_dictionary['the fairlanes'] = ['Andy Baldwin', 'Jason Zumbrunnen', 'Robbie Kalinowski', 'Scott Weigel']
artist_dictionary['pink lincolns'] = ['Chris Barrows','Dorsey Martin','Kevin Coss','Jeff Fox']
artist_dictionary['zeke'] = ['Blind Marky Felchtone','Kyle Whitefoot','Kurt Colfelt','Dayne Porras',                             'Jeff Hiatt','Chris Johnsen','Buzzy','Kurt Colfelt','Jeff Matz',                             'Mark Pierce','Abe Zanuel Riggs III','Dizzy Lee Roth','Donny Paycheck']
artist_dictionary['x-ray spex'] = ['B.P. Hurding','Poly Styrene','Steve Thompson','Lora Logic']
artist_dictionary['fidlar'] = ['Zac Carper','Elvis Kuehn','Max Kuehn','Brandon Schwartzel']
artist_dictionary['johnny thunders & the heartbreakers'] = artist_dictionary['the heartbreakers']
artist_dictionary['mike v & the rats'] = ['Jason Hampton', 'Mike Vallely', 'P.T. Pugh', 'Reid Black']
artist_dictionary['g-eazy'] = ['g-eazy']
artist_dictionary['post malone'] = ['post malone']
artist_dictionary['lil skies'] = ['lil skies']
artist_dictionary['lil mosey'] = ['lil mosey']
artist_dictionary['youngboy never broke again'] = ['youngboy never broke again']
artist_dictionary['a boogie wit da hoodie'] =['a boogie wit da hoodie']
artist_dictionary['lil baby'] = ['ll baby']
artist_dictionary['pnb rock'] = ['pnb rock']
artist_dictionary['tee grizzley'] = ['tee grizzley']
artist_dictionary['meek mill'] = ['meek mill']
artist_dictionary['lil zay osama'] = ['lil zay osama']
artist_dictionary['montana of 300'] = ['montana of 300']
artist_dictionary['playboi'] =['playboi']
artist_dictionary['strap da fool'] = ['strap da fool']
artist_dictionary['lil durk'] = ['lil durk']
artist_dictionary['juice wrld'] = ['juice wrld']
artist_dictionary['quando rondo'] = ['quando rondo']
artist_dictionary['yk osiris'] = ['yk osiris']
artist_dictionary['pnb meen'] = ['pnb meen']
artist_dictionary['lil uzi vert'] = ['lil uzi vert']
artist_dictionary['knuckle puck'] =['Joe Taylor','Kevin Maida','John Siorek','Nick Casasanto','Ryan Rumchaks']
artist_dictionary['the story so far'] = ['Parker Cannon','Kelen Capener','Kevin Geyer','Will Levy','Ryan Torf']
artist_dictionary['derek and the dominos'] = ['Eric Clapton','Bobby Whitlock','Carl Radle','Jim Gordon','Dave Mason','Duane Allman']
artist_dictionary['derek & the dominos'] = ['Eric Clapton','Bobby Whitlock','Carl Radle','Jim Gordon','Dave Mason','Duane Allman']

#adding comment to diagnose git issues

artist_dictionary = {str(key).lower():artist_dictionary[key] for key in artist_dictionary.keys()}
for key in artist_dictionary.keys():
    artist_dictionary[key]=[i.lower() for i in artist_dictionary[key]]


'''
Creating to run Model D Conv1d_mfcc40
different layer system
full mfcc features
Created on 2020.12.08 at 1843

'''
# libraries
import numpy as np
import pandas as pd
import pickle
import seaborn as sns
import tensorflow as tf

from tensorflow.keras.layers import Dense, Dropout, Flatten, Input
from tensorflow.keras.layers import Activation, BatchNormalization
from tensorflow.keras.layers import Conv1D, Conv2D, LSTM
from tensorflow.keras.layers import AveragePooling1D, GlobalAveragePooling2D, MaxPooling1D, MaxPool2D
from tensorflow.keras.models import Model, model_from_json, Sequential

# added this to plot
import matplotlib.pyplot as plt

from sklearn.metrics import confusion_matrix, accuracy_score, classification_report

# assigning the pickle files
with open('./Data_Array_Storage/male_pol_us_X_train.pkl', 'rb') as f:
    X_train = pickle.load(f)

with open('./Data_Array_Storage/male_pol_us_X_test.pkl', 'rb') as f:
    X_test = pickle.load(f)

with open('./Data_Array_Storage/male_pol_us_y_train.pkl', 'rb') as f:
    y_train = pickle.load(f)

with open('./Data_Array_Storage/male_pol_us_y_test.pkl', 'rb') as f:
    y_test = pickle.load(f)

input_shape = (X_train.shape[1], X_train.shape[2],1)

learning_rate=0.0001
nclass = 3

def get_2d_conv_model(input_shape):

    inp = Input(shape=input_shape)
    x = Conv2D(32, (4,10), padding="same")(inp)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Conv2D(32, (4,10), padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Conv2D(32, (4,10), padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Conv2D(32, (4,10), padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Flatten()(x)
    x = Dense(64)(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    out = Dense(nclass, activation='softmax')(x)

    model = Model(inputs=inp, outputs=out)
    opt = tf.keras.optimizers.Adam(learning_rate)

    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['acc'])
    return model

model = get_2d_conv_model(input_shape)
model.summary()

# callback list: ModelCheckpoint, reduceLROnPlat, EarlyStopping
checkpoint_path = "./models_saved/male_pol_conv2d.h5"

# Create a callback that saves the model's weights
callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=False,
                                                 save_best_only=True,
                                                 verbose=1), # 1 tells your which epoch is saving
               tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',
                                                patience=5,
                                                restore_best_weights=True),
               tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
                                                    patience=2,
                                                    factor=0.5,
                                                    min_lr=0.00001,
                                                    verbose=1)]

batch_size = 32

model_history=model.fit(X_train,
                        y_train,
                        batch_size=batch_size,
                        epochs=150,
                        validation_data=(X_test, y_test),
                        verbose=2,
                        callbacks=callbacks)

print('model training complete')

plt.plot(model_history.history['loss'])
plt.plot(model_history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

score,acc = model.evaluate(X_test, y_test, verbose = 2, batch_size = batch_size)
print("score: %.2f" % (score))
print("acc: %.2f" % (acc))

with open('./Data_Array_Storage/male_pol_us_labels.pkl', 'rb') as f:
    lb = pickle.load(f)

preds = model.predict(X_test,
                        batch_size=batch_size,
                        verbose=1)

preds=preds.argmax(axis=1)
preds

# predictions
preds = preds.astype(int).flatten()
preds = (lb.inverse_transform((preds)))
preds = pd.DataFrame({'predictedvalues': preds})

# Actual labels
actual = y_test.argmax(axis=1)
actual = actual.astype(int).flatten()
actual = (lb.inverse_transform((actual)))
actual = pd.DataFrame({'actualvalues': actual})

# Lets combined both of them into a single dataframe
finaldf = actual.join(preds)
finaldf[150:180]

def print_confusion_matrix(confusion_matrix, class_names, figsize = (10,7), fontsize=14):
    """Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap.

    Arguments
    ---------
    confusion_matrix: numpy.ndarray
        The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix.
        Similarly constructed ndarrays can also be used.
    class_names: list
        An ordered list of class names, in the order they index the given confusion matrix.
    figsize: tuple
        A 2-long tuple, the first value determining the horizontal size of the ouputted figure,
        the second determining the vertical size. Defaults to (10,7).
    fontsize: int
        Font size for axes labels. Defaults to 14.

    Returns
    -------
    matplotlib.figure.Figure
        The resulting confusion matrix figure
    """
    df_cm = pd.DataFrame(
        confusion_matrix, index=class_names, columns=class_names,
    )
    fig = plt.figure(figsize=figsize)
    try:
        heatmap = sns.heatmap(df_cm, annot=True, fmt="d")
    except ValueError:
        raise ValueError("Confusion matrix values must be integers.")

    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)
    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

    # Get the predictions file
# finaldf = pd.read_csv("Predictions.csv")
classes = finaldf.actualvalues.unique()
classes.sort()

# Confusion matrix
c = confusion_matrix(finaldf.actualvalues, finaldf.predictedvalues)
print(accuracy_score(finaldf.actualvalues, finaldf.predictedvalues))
print_confusion_matrix(c, class_names = classes)

# Classification report
classes = finaldf.actualvalues.unique()
classes.sort()
print(classification_report(finaldf.actualvalues, finaldf.predictedvalues, target_names=classes))


from keras.models import Model
from keras.layers import Dense, Input, multiply, concatenate, Dropout
import matplotlib.pyplot as plt
import numpy as np

# Load data
train_wic1 = np.array(np.load('../../processed_data/train_wic1.npy'))
train_wic2 = np.array(np.load('../../processed_data/train_wic2.npy'))
dev_wic1 = np.array(np.load('../../processed_data/dev_wic1.npy'))
dev_wic2 = np.array(np.load('../../processed_data/dev_wic2.npy'))
train_labels = np.load('../../processed_data/train_labels.npy')
dev_labels = np.load('../../processed_data/dev_labels.npy')

# Build model
layers = [1024, 512]
num_layer = len(layers)  # Number of layers in the MLP
dropouts = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
max_length = 1024
acc = []
loss = []

for dropout in dropouts:
    # Input variables
    wic1_input = Input(shape=(max_length,), dtype='float32')
    wic2_input = Input(shape=(max_length,), dtype='float32')

    wic1_latent = wic1_input
    wic2_latent = wic2_input

    # GMF part
    gmf_vector = multiply([wic1_latent, wic2_latent])
    gmf_vector = Dropout(dropout)(gmf_vector)

    # MLP part
    mlp_vector = concatenate([wic1_latent, wic2_latent])
    mlp_vector = Dropout(dropout)(mlp_vector)
    for idx in range(0, num_layer):
        layer = Dense(layers[idx], activation='relu')
        mlp_vector = layer(mlp_vector)
        mlp_vector = Dropout(dropout)(mlp_vector)

    # Concatenate MF and MLP parts
    neumf_vector = concatenate([gmf_vector, mlp_vector])

    # Output layer
    output = Dense(1, activation='sigmoid', init='lecun_uniform', name="prediction")(neumf_vector)

    model = Model(input=[wic1_input, wic2_input],
                  output=output)
    # Compile the model
    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    # print(model.summary())

    # Fit the model
    history = model.fit(
        [train_wic1, train_wic2],
        train_labels, epochs=10, verbose=2)

    los, ac = model.evaluate([dev_wic1, dev_wic2], dev_labels)

    acc.append(ac)
    loss.append(los)

np.save('acc.npy', acc)
np.save('loss.npy', loss)

xi = list(range(len(dropouts)))
plt.plot(acc)
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Dropout')
plt.xticks(xi, dropouts)
plt.show()

plt.plot(loss)
plt.title('Model loss')
plt.xticks(xi, dropouts)
plt.ylabel('Loss')
plt.xlabel('Dropout')
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Sat Nov  2 22:41:49 2019

@author: Veda
"""
import numpy as np
import pandas as pd
from bokeh.plotting import figure, output_file, show
from bokeh.models import ColumnDataSource, Plot, LinearAxis, Grid
from bokeh.models.tools import HoverTool
from bokeh.palettes import Spectral5
from bokeh.transform import factor_cmap
from bokeh.layouts import row,column
from bokeh.io import *
from bokeh.layouts import *
from bokeh.plotting import *
from bokeh.models.widgets import *
from bokeh.models.widgets import MultiSelect
from bokeh.layouts import widgetbox
from bokeh.layouts import gridplot
output_file("dropdown.html")

menu = [("TELANGANA", "item_1"), ("ANDHRA PRADESH", "item_2")]
dropdown = Dropdown(label="REGION/STATE", button_type="default", width=130, menu=menu)

menu1 = [("KARIMNAGAR", "item_1"), ("MAHABUBNAGAR", "item_2")]
dropdown1 = Dropdown(label="DISTRICTS", button_type="default", width=123
                    , menu=menu1)

bt = Button(label="MTD", button_type="default", width=48)
bt1 = Button(label="QTD", button_type="default", width=48)
bt2 = Button(label="YTD", button_type="default", width=48)
bt3 = Button(label="CUSTOM", button_type="default", width=85)

menu2 = [("Trade", "item_1"), ("Non-Trade", "item_2")]
dropdown2 = Dropdown(label="CUSTOMER TYPE", button_type="default", menu=menu2,width=144)

menu3 = [("OPC53", "item_1"), ("OPC43", "item_2"),("PPC", "item_3"),("PSC", "item_4")]
dropdown3 = Dropdown(label="PRODUCT GRADE", button_type="default", menu=menu3, width=140)

menu4 = [None]
dropdown4 = Dropdown(label="OFFICER LEVEL", button_type="default", menu=menu4, width=140)

menu5 = [("Bulk", "item_1"), ("Bag", "item_2")]
dropdown5 = Dropdown(label="PACKING TYPE", button_type="default", menu=menu5,width=140
                    )

bt8 = Button(label="DEFAULT/RESET", button_type="default",width=140)

layout=layout(column(row(dropdown,dropdown1,bt, bt1, bt2, bt3, dropdown2,dropdown3, dropdown4,dropdown5, bt8)))
#show(layout)

df = pd.read_excel(r"C:/Users/Veda/Downloads/Book1.xlsx")

sales_data=df.groupby(df['Inv Date'].dt.strftime('%B'))['Sales Qty'].sum().sort_values()
#sales_data1=df.sort_values(by='Inv Date', ascending= False)
grouped = sales_data/ 1000
print(grouped)

source = ColumnDataSource(pd.DataFrame(grouped))
states = source.data['Inv Date'].tolist()
p = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#FFFF99")

p.title.text ='Total Sales'
p.title.align = 'center'
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.yaxis.visible = False
p.xaxis.visible=True
p.outline_line_width = 7
p.outline_line_alpha = 0.3
p.outline_line_color = "#E5FFCC"

p1 = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p1.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#FFCC99")

p1.title.text ='Realization'
p1.title.align = 'center'
p1.xgrid.grid_line_color = None
p1.ygrid.grid_line_color = None
p1.yaxis.visible = False
p1.xaxis.visible=True
p1.outline_line_width = 7
p1.outline_line_alpha = 0.3
p1.outline_line_color = "#E5FFCC"

p2 = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p2.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#CCFFCC")

p2.title.text ='Collections'
p2.title.align = 'center'
p2.xgrid.grid_line_color = None
p2.ygrid.grid_line_color = None
p2.yaxis.visible = False
p2.xaxis.visible=True
p2.outline_line_width = 7
p2.outline_line_alpha = 0.3
p2.outline_line_color = "#E5FFCC"
p3 = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p3.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#CCE5FF")

p3.title.text ='Over Due'
p3.title.align = 'center'
p3.xgrid.grid_line_color = None
p3.ygrid.grid_line_color = None
p3.yaxis.visible = False
p3.xaxis.visible=True
p3.outline_line_width = 7
p3.outline_line_alpha = 0.3
p3.outline_line_color = "#E5FFCC"

grouped=df.groupby(['Region Description'])['Naked Realization(Rs'].sum()
grouped=(grouped)/10000000
source = ColumnDataSource(pd.DataFrame(grouped))
states = source.data['Region Description'].tolist()
p5 = figure(y_range=states)
color_map = factor_cmap(field_name='Region Description',
                    palette=Spectral5, factors=states)

p5.hbar(y='Region Description',source=source,right="Naked Realization(Rs",left=0,height=0.5, color="#FFFF99")

p5.title.text ='Naked Realization by state'
p5.xaxis.axis_label = 'Naked Realization in Rs'
p5.yaxis.axis_label = 'state'
hover = HoverTool()
hover.tooltips = [
    ("Totals","@Naked Realization(Rs")]

hover.mode = 'hline'

p.add_tools(hover)

hover = HoverTool(tooltips=[("sales","@Sales%20Qty,")],mode='vline')
#layout.add_tools(hover)

layout1= row (p,p1,p2,p3)
layout2= column(layout,layout1,p5)

#curdoc().add_root(layout)
show(layout2)


import copy
import random
import draw_config
from collections import namedtuple
from PIL import Image, ImageDraw, ImageFont

BoardLayout = namedtuple('BoardLayout', ['orient', 'layout'])

class Board:
    def __init__(self, enemy_white):
        self.enemy_white = enemy_white

        self._bot = BoardLayout(1, dict.fromkeys(
            [12, 14, 16, 18, 21, 23, 25, 27, 32, 34, 36, 38], False))
        self._enemy = BoardLayout(-1, dict.fromkeys(
            [61, 63, 65, 67, 72, 74, 76, 78, 81, 83, 85, 87], False))

    @staticmethod
    def make_move(pos, target, me, enemy):
        """Returns True if it was king's move"""
        y = me.orient if (target - pos) * me.orient > 0 else -me.orient
        x = 1 if target % 10 - pos % 10 > 0 else -1

        current_pos = pos + y * 10 + x
        while current_pos != target:
            if current_pos in enemy.layout:
                enemy.layout.pop(current_pos)

            current_pos = current_pos + y * 10 + x

        row = target // 10
        if (row, me.orient) in ((8, 1), (1, -1)):
            me.layout[target] = True
            me.layout.pop(pos)
        else:
            me.layout[target] = me.layout.pop(pos)

        if me.layout[target]:
            return True
        else:
            return False

    def move_options(self, target, bot):
        moves = {}
        compelled_board = False

        me, enemy = self.get_sides(bot)

        if target == -1:
            source = me.layout
        else:
            source = {target: me.layout[target]}

        for pos, is_king in source.items():
            if is_king:
                moves, compelled_board = self._add_options(
                    *self._king_options(pos, me, enemy), moves,
                    compelled_board)
            else:
                moves, compelled_board = self._add_options(
                    *self._men_options(pos, me, enemy), moves,
                    compelled_board)

        return moves, compelled_board

    def get_sides(self, bot):
        me, enemy = self._enemy, self._bot
        if bot:
            me, enemy = enemy, me

        return me, enemy

    def bot_score(self, bot):
        score = 0

        score += 4 * self._side_score(self._bot)
        score -= 3 * self._side_score(self._enemy)

        if bot:
            return score
        else:
            return -score

    def _side_score(self, me):
        score = 0

        for pos, is_king in me.layout.items():
            if is_king:
                score += 10
            else:
                score += 5
                if (pos // 10, me.orient) in ((1, 1), (8, -1)):
                    score += 2

            for cell in (pos - 11 * me.orient, pos - 9 * me.orient):
                if (cell in me.layout) or (not self._is_valid(cell)):
                    score += 1

        return score

    @staticmethod
    def _add_options(pos_options, pos, compelled_pos, moves,
                     compelled_board):
        if len(pos_options) > 0:
            if compelled_pos == compelled_board:
                moves[pos] = pos_options

            elif not compelled_board:
                compelled_board = True
                moves = {pos: pos_options}

        return moves, compelled_board

    def _king_options(self, pos, me, enemy):
        pos_options = set()
        compelled_pos = False

        for direction in ((-1, -1), (-1, 1), (1, -1), (1, 1)):
            met_enemy = False
            opt = pos + 10 * direction[0] + direction[1]

            while self._is_valid(opt):
                if opt in enemy.layout:
                    if opt + 10 * direction[0] + direction[1] in enemy.layout:
                        break
                    met_enemy = True
                elif opt in me.layout:
                    break
                elif met_enemy == compelled_pos:
                    pos_options.add(opt)
                elif met_enemy:
                    compelled_pos = True
                    pos_options = {opt}

                opt += 10 * direction[0] + direction[1]

        return pos_options, pos, compelled_pos

    def _men_options(self, pos, me, enemy):
        pos_options = set()
        compelled_pos = False

        for opt in ((pos - 11 * me.orient, pos - 22 * me.orient),
                    (pos - 9 * me.orient, pos - 18 * me.orient),
                    (pos + 11 * me.orient, pos + 22 * me.orient),
                    (pos + 9 * me.orient, pos + 18 * me.orient)):

            if (opt[0] in enemy.layout) and self._free_cell(opt[1], me, enemy):
                if compelled_pos:
                    pos_options.add(opt[1])
                else:
                    compelled_pos = True
                    pos_options = {opt[1]}

            elif not compelled_pos and self._free_cell(opt[0], me, enemy) and \
                    ((opt[0] - pos) * me.orient > 0):
                pos_options.add(opt[0])

        return pos_options, pos, compelled_pos

    @staticmethod
    def _is_valid(pos):
        return 1 <= pos // 10 <= 8 and 1 <= pos % 10 <= 8

    @staticmethod
    def _free_cell(pos, me, enemy):
        return Board._is_valid(pos) and \
               (pos not in enemy.layout) and (pos not in me.layout)

def to_str(checker, enemy_white):
    return to_str_format(checker // 10 - 1, checker % 10 - 1, enemy_white)

def to_str_format(row, col, enemy_white):
    if enemy_white:
        row = 7 - row
        col = 7 - col

    return chr(ord('A') + 7 - col) + str(row + 1)

class Painter:
    def __init__(self):
        self._white_blank = self._draw_blank_board(enemy_white=True)
        self._black_blank = self._draw_blank_board(enemy_white=False)

    def draw(self, board):
        if board.enemy_white:
            image = self._white_blank.copy()
        else:
            image = self._black_blank.copy()

        canvas = ImageDraw.Draw(image)

        enemy_palette = ('white', draw_config.black_draught_color,
                         draw_config.edge_color)
        bot_palette = ('black', draw_config.white_draught_color,
                       draw_config.edge_color)

        if board.enemy_white:
            enemy_palette, bot_palette = bot_palette, enemy_palette

        bot, enemy = board.get_sides(bot=True)

        self._draw_side(canvas, enemy.layout, enemy_palette, board.enemy_white)
        self._draw_side(canvas, bot.layout, bot_palette, board.enemy_white)

        image.save('tmp.png')

    def _draw_blank_board(self, enemy_white):
        image = Image.new('RGB', (550, 550), draw_config.white_cell_color)
        canvas = ImageDraw.Draw(image)

        canvas.rectangle(self._square_place(3, 515),
                         fill=draw_config.black_cell_color)
        canvas.rectangle(self._square_place(0, 512),
                         fill=draw_config.white_cell_color)

        font = ImageFont.truetype(*draw_config.font)

        for row in range(8):
            for col in range(8):
                if (row + col) % 2 == 1:
                    canvas.rectangle(self._cell_place(row, col),
                                     fill=draw_config.black_cell_color,
                                     outline=draw_config.black_cell_color)

                    text = to_str_format(row, col, enemy_white)
                    canvas.text(self._text_place(row, col, text, font, 'b'),
                                text, font=font, fill='white')

        return image

    def _draw_side(self, canvas, layout, palette, enemy_white):
        for checker, is_king in layout.items():
            if is_king:
                self._draw_checker(canvas, checker, *palette[:-1],
                                   draw_config.red_color, enemy_white)
            else:
                self._draw_checker(canvas, checker, *palette, enemy_white)

    def _draw_checker(self, canvas, checker, font_color, main_color,
                      other_color, enemy_white):

        row = checker // 10 - 1
        col = checker % 10 - 1

        canvas.ellipse(self._ellipse_place(row, col, 10, 5), fill=other_color)

        canvas.ellipse(self._ellipse_place(row, col, 5, 10),
                       fill=main_color, outline=other_color)

        font = ImageFont.truetype(*draw_config.font)
        text = to_str_format(row, col, enemy_white)
        canvas.text(self._text_place(row, col, text, font, 'c'), text,
                    font=font, fill=font_color)

    @staticmethod
    def _square_place(up_left, down_right):
        return (draw_config.margin - up_left, draw_config.margin - up_left,
                draw_config.margin + down_right,
                draw_config.margin + down_right)

    @staticmethod
    def _cell_place(row, col):
        return (col * 64 + draw_config.margin, row * 64 + draw_config.margin,
                (col + 1) * 64 + draw_config.margin,
                (row + 1) * 64 + draw_config.margin)

    @staticmethod
    def _ellipse_place(row, col, up_shift, down_shift):
        return (col * 64 + draw_config.margin + 5,
                row * 64 + draw_config.margin + up_shift,
                (col + 1) * 64 + draw_config.margin - 5,
                (row + 1) * 64 + draw_config.margin - down_shift)

    @staticmethod
    def _text_place(row, col, text, font, context):

        width, height = font.getsize(text)

        return (draw_config.margin + col * 64 + (64 - width) // 2,
                (draw_config.margin + row * 64 + 3 + (49 - height) // 2
                 if context == 'c' else
                 draw_config.margin + row * 64 + (64 - height) // 2 - 1))

class ArtificialIntelligence:
    def bot_move(self, board):
        moves, compelled_board = board.move_options(-1, bot=True)
        king_move = False
        if len(moves) == 0:
            return [], king_move

        moves_done = []

        first_move = True
        murder = -1
        while first_move or murder != -1:
            first_move = False

            score, best_moves = self._variant_score(board, depth=5, bot=True,
                                                    murder=murder)

            best_moves = self._remove_far(best_moves)

            pos, target = random.choice(best_moves)
            king_move = board.make_move(pos, target,
                                        *board.get_sides(bot=True))
            moves_done.append((to_str(pos, board.enemy_white),
                               to_str(target, board.enemy_white)))

            if compelled_board and board.move_options(target, bot=True)[1]:
                murder = target
            else:
                murder = -1

        return moves_done, king_move

    @staticmethod
    def _remove_far(best_moves):
        new_list = []
        best_dist = 0

        for move in best_moves:
            row = move[1] // 10

            if row > best_dist:
                best_dist = row
                new_list = [move]
            elif row == best_dist:
                new_list.append(move)

        return new_list

    def _variant_score(self, board, depth, bot, murder):
        if depth == 0:
            return self._change_sign(board.bot_score(bot), murder), []

        best_score = -1000
        best_moves = []

        moves, compelled_board = board.move_options(murder, bot=bot)
        if len(moves) == 0:
            return self._change_sign(-1000, murder), []

        for pos, pos_options in moves.items():
            for target in pos_options:
                move_board = copy.deepcopy(board)
                move_board.make_move(pos, target, *move_board.get_sides(bot))

                if compelled_board and move_board.move_options(target,
                                                               bot=bot)[1]:
                    score, b_m = self._variant_score(move_board, depth - 1,
                                                     bot, target)
                else:
                    score, b_m = self._variant_score(move_board, depth - 1,
                                                     not bot, -1)

                if score > best_score:
                    best_score = score
                    best_moves = [(pos, target)]
                elif score == best_score:
                    best_moves.append((pos, target))

        return self._change_sign(best_score, murder), best_moves

    @staticmethod
    def _change_sign(score, murder):
        if murder == -1:
            return -score
        else:
            return score

class Game:
    painter = Painter()
    ai = ArtificialIntelligence()

    def __init__(self, enemy_white=True):
        self._board = Board(enemy_white=enemy_white)
        Game.painter.draw(self._board)

        self.murder = -1
        self.chosen_checker = -1
        self.until_draw = 15

    def black_first_move(self):
        moves_done, king_move = Game.ai.bot_move(self._board)
        self._update_draw(king_move)

        Game.painter.draw(self._board)
        return moves_done

    def button_variants(self):
        moves, c_b = self._board.move_options(-1, bot=False)
        if self.chosen_checker == -1:
            res = moves.keys()
        else:
            res = moves[self.chosen_checker]

        return [to_str(elem, self._board.enemy_white) for elem in list(res)]

    def external_session(self, ans):
        if ans not in self.button_variants():
            return (2 if self.chosen_checker == -1 else 1), []

        if self.chosen_checker == -1:
            self.chosen_checker = self._to_pos(ans, self._board.enemy_white)
            return 4, []
        else:
            return self.move_session(self.chosen_checker,
                                     self._to_pos(ans,
                                                  self._board.enemy_white))

    @staticmethod
    def _to_pos(ans, enemy_white):
        if enemy_white:
            return (9 - int(ans[1])) * 10 + (ord(ans[0]) - ord('A') + 1)
        else:
            return int(ans[1]) * 10 + (8 - ord(ans[0]) + ord('A'))

    def _update_draw(self, king_move):
        if king_move:
            self.until_draw -= 1
        else:
            self.until_draw = 15

    def move_session(self, pos, target):  #
        """Returns 0 if you lose, 3 if choose checker, 4 if choose cell,
        5 if you win, 6 if choose target again, 7 if draw"""

        if self.murder == -1:
            moves, compelled_board = self._board.move_options(-1, bot=False)
        else:
            moves, compelled_board = self._board.move_options(self.murder,
                                                              bot=False)

        king_move = self._board.make_move(pos, target,
                                          *self._board.get_sides(bot=False))
        self._update_draw(king_move)

        if compelled_board and self._board.move_options(target, bot=False)[1]:
            self.murder = target
            self.chosen_checker = target
            Game.painter.draw(self._board)
            return 6, []
        else:
            self.murder = -1
            self.chosen_checker = -1

        moves_done, king_move = Game.ai.bot_move(self._board)
        self._update_draw(king_move)
        Game.painter.draw(self._board)

        if len(moves_done) == 0:
            return 5, moves_done

        moves, compelled_board = self._board.move_options(-1, bot=False)

        if len(moves) > 0:
            if self.until_draw > 0:
                return 3, moves_done
            else:
                return 7, moves_done
        else:
            return 0, moves_done


import sys
import os
import re
import xlwt
# import pandas as pd
import xlwt, xlrd
from xlutils.copy import copy

def default_process():
    if (len(sys.argv) == 3):
        path = sys.argv[1]
        filename = sys.argv[2]
    elif (len(sys.argv) == 2):
        if (sign in sys.argv[1] for sign in ['/', '\\']):
            path = sys.argv[1]
            filename = 'All'
        else:
            path = './'
            filename = sys.argv[1]
    else:
        path = './'
        filename = 'All'

    return path, filename

def write_file(filepath, filename, mode='r'):
    if filename == 'All':
        listdir = os.listdir(filepath)
        print(listdir)
        for name in listdir:
            # if name == '...': continue
            process_file(filepath, name)
    else:
        process_file(filepath, filename)
    return

def write_txt(filename, data, data_header):
    with open(filename, 'a+') as w:
        for key in data.keys():
            data_line = ' '.join(data[key])
            data_line = data_header + ' ' + key + ' ' + data_line

        w.writelines([data_line, '\n'])
        w.flush()

def write_excel_xls(path, data, tablename, sec, opcs, avg_opc):
    # index = len(value)  # 获取需要写入数据的行数
    # print('1')
    workbook = xlwt.Workbook()  # 新建一个工作簿
    # sheet_name = []
    for key in data.keys():
        name = key
        # sheet_name.append(name)
        sheet = workbook.add_sheet(name)  # 在工作簿中新建一个表格
        for i, n in enumerate(tablename):
            sheet.write(0, i , n)

        sheet.write(1, 0, sec)
        sheet.write(1, 1, opcs)
        sheet.write(1, 2, avg_opc)

        for i in range(len(data[key])):
            sheet.write(1, i + 3, data[key][i])
    workbook.save(path)  # 保存工作簿
    # print("xls格式表格写入数据成功！")

def write_excel_xls_append(path, data, tablename, sec, opcs, avg_opc):
    # index = len(value)  # 获取需要写入数据的行数

    # print(sheets)
    for key in data.keys():
        workbook = xlrd.open_workbook(path)  # 打开工作簿
        sheets = workbook.sheet_names()  # 获取工作簿中的所有表格
        if key not in sheets:
            # print(key)
            new_workbook = copy(workbook)
            # new_worksheet = new_workbook.add_sheet(key)

            sheet = new_workbook.add_sheet(key)
            for i, n in enumerate(tablename):
                sheet.write(0, i, n)

            sheet.write(1, 0, sec)
            sheet.write(1, 1, opcs)
            sheet.write(1, 2, avg_opc)

            for i in range(len(data[key])):
                sheet.write(1, i + 3, data[key][i])

            new_workbook.save(path)
        else:
            # print("1")
            # print(key)
            worksheet = workbook.sheet_by_name(key)
            rows_old = worksheet.nrows
            new_workbook = copy(workbook)
            new_worksheet = new_workbook.get_sheet(key)

            new_worksheet.write(rows_old, 0, sec)
            new_worksheet.write(rows_old, 1, opcs)
            new_worksheet.write(rows_old, 2, avg_opc)

            for i in range(len(data[key])):
                new_worksheet.write(rows_old, i + 3, data[key][i])
            new_workbook.save(path)
            # print("xls格式表格【追加】写入数据成功！")

def process_file(filepath, filename):
    # print("filepath : %s\nfilename : %s" % (filepath, filename))
    with open(filepath + filename, 'r') as f:
        # print('1')
        flag = 0  # excel第一次写入标志位

        for line in f:
            # print('2',line)
            if (line.find('Command line') != -1):
                # print('3', line)
                DB_pat = re.compile('com\.yahoo\.ycsb\.db\.(.*)Client')
                Thread_pat = re.compile('workloads/workload(.) -s -threads (.*) -t')
                mbj1 = DB_pat.search(line)
                mbj2 = Thread_pat.search(line)
                if mbj1 and mbj2:
                    wname = mbj1.group(1)
                    postx_txt = ".%s.%s.txt" % (mbj2.group(1), mbj2.group(2))
                    postx_xls = ".%s.%s.xls" % (mbj2.group(1), mbj2.group(2))
                # print(wname+postx_xls)
            elif 'wname' and 'postx_txt' in locals().keys():
                opc_pat = re.compile('\[([^\[\]]*)?\]')
                list_opc = opc_pat.findall(line)
                # print(line, len(list_opc), list_opc)
                if len(list_opc) > 1:
                    sec_pat = re.compile(' ([^ ]*) sec:')
                    pat_opc = re.compile('sec: (.*) operations; (.*) current ops/sec;')

                    sec = sec_pat.search(line).group(1)
                    opcs = pat_opc.search(line).group(1)
                    avg_opc = pat_opc.search(line).group(2)

                    data = {}

                    for i in range(len(list_opc)):
                        opc = list_opc[i].split()
                        opc_name = opc[0][:-1]
                        opc_count = opc[1][6:-1]
                        opc_maxL = opc[2][4:-1]
                        opc_minL = opc[3][4:-1]
                        opc_avg = opc[4][4:-1]
                        opc_90 = opc[5][3:-1]
                        opc_99 = opc[6][3:-1]
                        opc_999 = opc[7][5:-1]
                        opc_9999 = opc[8][6:]

                        data[opc_name] = [opc_count, opc_maxL,
                                          opc_minL, opc_avg,
                                          opc_90, opc_99, opc_999, opc_9999]

                    data_header = ' '.join([sec, opcs, avg_opc])

                    tablename = ['SEC(m)', 'TOTAL_OPC', 'QPS(ops/sec)', 'OPC_COUNT', 'OPC_MAXL(us)',
                                 'OPC_MINL(us)','OPC_AVG', '90%(us)', '99%(us)', '99.9%(us)', '99.99%(us)']

                    #  去掉注释 输出txt文件
                    #  write_txt(wname+postx_txt, data, data_header)

                    if flag == 0:
                        # print("1")
                        # print(avg_opc)
                        write_excel_xls(wname + postx_xls, data, tablename, sec, opcs, avg_opc)
                        # print("2")
                        flag = 1
                    else:
                        write_excel_xls_append(wname + postx_xls, data, tablename, sec, opcs, avg_opc)

            else:
                continue

if __name__ == '__main__':
    filepath, filename = default_process()
    print("filepath : %s\nfilename : %s" % (filepath, filename))
    # print(filepath,filename)
    write_file(filepath, filename)


import datetime

from utilities.encoding import ensure_unicode

def get_time_show(real_time, just_now_range=60):
    """
    评论的时间显示逻辑
    """
    now = datetime.datetime.now().replace(tzinfo=None)
    seconds = int((now - real_time.replace(tzinfo=None)).total_seconds())
    if seconds < just_now_range:
        result = str(seconds) + u"秒前"
    elif seconds < 30 * 60:
        result = str(int(seconds / 60)) + u"分钟前"
    elif seconds < 60 * 60:
        result = u"半小时前"
    elif seconds < 24 * 60 * 60:
        result = str(int(seconds / 3600)) + u"小时前"
    else:
        result = real_time.strftime("%-m月%-d日")
    return ensure_unicode(result)

def get_age_by_birthday(birthday):
    """
    根据生日获取年龄，向上取整
    :param birthday: datetime.date
    :return: int
    """
    if not birthday:
        return 0
    return ((datetime.date.today() - birthday).days + 364) / 365


import time
from collections import OrderedDict
from datetime import datetime

import pymysql
import pytz
import rethinkdb as rdb

class ZabbixAlert(object):
    """ Zabbix alert object used to push alerts to the db"""

    def __init__(self, event_id, event_time=None, event_date=None, event_name='', priority=10, db_connection=True):
        self.timezone = pytz.timezone('UTC')
        self.database = None
        self.db_table = None
        self.event_id = event_id
        if event_date and event_time:
            self.event_datetime = self.timezone.localize(
                datetime.strptime('{0} {1}'.format(event_date, event_time), '%Y.%m.%d %H:%M:%S'))
            self.event_ts = self.event_datetime.strftime('%H:%M:%S %d-%m-%Y')
        else:
            self.event_ts = None
            self.event_datetime = None

        self.priority = priority
        self.event_name = event_name
        self.alert_entry = {'event_id': self.event_id, 'ts': self.event_datetime,
                            'priority': priority, 'name': self.event_name}
        if db_connection:
            self.connect_db()

    def connect_db(self, host='rdb', port='28015', database='monitoring', db_table='alerts'):
        """ Initialise connection to Rethink"""
        self.database = database
        self.db_table = db_table
        rdb.connect(host, port, ).repl()

    def exist(self):
        cursor = rdb.db(self.database).table(self.db_table).filter(
            rdb.row['event_id'] == self.alert_entry['event_id']).run()
        results = list(cursor)

        if len(results) == 1:
            return True

        return False

    def push(self):
        if not self.exist():
            rdb.db(self.database).table(self.db_table).insert(self.alert_entry).run()
        else:
            self.update()

    def pop(self):
        if self.exist():
            rdb.db(self.database).table(self.db_table).filter(
                rdb.row['event_id'] == self.alert_entry['event_id']).delete().run()

    def update(self):
        if self.exist():
            rdb.db(self.database).table(self.db_table).filter(
                rdb.row['event_id'] == self.alert_entry['event_id']).update(self.alert_entry).run()

def get_alerts(count=None, index='ts', order='desc', host='rdb', port='28015', database='monitoring',
               db_table='alerts'):
    """ Function to get specific number of current alerts ordered by 'index'"""
    alerts = []
    rdb.connect(host, port, ).repl()
    if count:
        if order == 'desc':
            cursor = rdb.db(database).table(db_table).order_by(rdb.desc(index)).limit(count).run()
        elif order == 'asc':
            cursor = rdb.db(database).table(db_table).order_by(rdb.acs(index)).limit(count).run()
        else:
            cursor = rdb.db(database).table(db_table).limit(count).run()
    else:
        if order == 'desc':
            cursor = rdb.db(database).table(db_table).order_by(rdb.desc(index)).run()
        elif order == 'asc':
            cursor = rdb.db(database).table(db_table).order_by(rdb.acs(index)).run()
        else:
            cursor = rdb.db(database).table(db_table).run

    for alert in cursor:
        alerts.append(alert)

    return alerts

def find_parent_id_db(child_id, db_connection):
    cursor = db_connection.cursor()
    cursor.execute('SELECT templateid FROM triggers WHERE triggerid={}'.format(child_id))
    record = cursor.fetchall()
    if len(record) != 1:
        return None

    parent_id = record[0][0]
    if not parent_id:
        return child_id

    return find_parent_id_db(parent_id, db_connection)

def fetch_parents_tree(db_connection):
    cursor = db_connection.cursor()
    cursor.execute('SELECT triggerid, templateid FROM triggers')
    data = cursor.fetchall()

    return data

def find_parent_id_cache(child_id, parents_data):
    parent_id = None
    for pair in parents_data:
        if pair[0] == child_id:
            parent_id = pair[1]

            break

    if not parent_id:
        return child_id

    return find_parent_id_cache(parent_id, parents_data)

def stats(limit=100):
    db = pymysql.connect('mysql', 'root', 'secret', 'zabbix')
    child_parent = dict()
    event_stats = dict()
    statistics = list()
    start_time = int(time.time()) - 2592000
    cursor = db.cursor()
    cursor.execute('SELECT objectid FROM events WHERE clock>{}'.format(start_time))
    data = cursor.fetchall()
    # parents_tree = fetch_parents_tree(db)
    for zabbix_event in data:
        child_id = zabbix_event[0]
        if child_id in child_parent.keys():
            parent_id = child_parent[child_id]
        else:
            parent_id = find_parent_id_db(child_id, db)
            # parent_id = find_parent_id_cache(int(child_id), parents_tree)
            child_parent[child_id] = parent_id

        if parent_id in event_stats.keys():
            event_stats[parent_id] += 1
        else:
            event_stats[parent_id] = 1

    event_stats = OrderedDict(reversed(sorted(event_stats.items(), key=lambda t: t[1])))

    for event in event_stats:
        if event == None:
            continue

        cursor = db.cursor()
        cursor.execute('SELECT description FROM triggers WHERE triggerid={}'.format(event))
        data = cursor.fetchall()
        description = str(data[0][0])
        record = {'description': description, 'count': event_stats[event]}
        statistics.append(record)

    db.close()
    return statistics[:limit]


import os
from concurrent.futures import ThreadPoolExecutor as PoolExecutor

import requests

listIdsFilename = '/home/kfr2359/msufldr/specsem 11 term/pub_id_msc.txt'
descrURLBase = 'http://e-heritage.ru/ras/view/publication/general.xml?id='
contentsURLBase = 'http://books.e-heritage.ru/Book/Book/getCt/15'

book_contents_path_str = 'book_contents/{}.xml'
book_description_path_str = 'book_description/{}.rdf'

def get_id_msc_pairs():
    idMSCpairs = {}
    with open(listIdsFilename, "r") as fListIds:
        lines = fListIds.readlines()

        # remove first line because it's the name of column
        lines = lines[1:-1]

        for line in lines:
            pair = line.split('\t')
            pair[1] = pair[1].replace('\n', '')
            idMSCpairs[pair[0]] = pair[1]
    return idMSCpairs

def fetch_description(id):
    url = descrURLBase + id
    filename = book_description_path_str.format(id)
    if not os.path.exists(filename) or os.path.getsize(filename) == 0:
        with open(filename, 'w+') as file:
            file.write(requests.get(url).text)

def fetch_contents(msc):
    url = contentsURLBase
    filename = book_contents_path_str.format(msc)
    if not os.path.exists(filename) or os.path.getsize(filename) == 0:
        variables = {
            'bok': msc
        }
        with open(filename, 'w+') as file:
            file.write(requests.post(url, data=variables).content.decode('utf-8'))

def download_book_info(id_msc_pairs):
    print(f'total: {len(id_msc_pairs)}')
    with PoolExecutor(max_workers=8) as executor:
        for _ in executor.map(fetch_description, id_msc_pairs.keys()):
            pass

    with PoolExecutor(max_workers=8) as executor:
        for _ in executor.map(fetch_contents, id_msc_pairs.values()):
            pass

if __name__ == '__main__':
    id_msc_pairs = get_id_msc_pairs()

    download_book_info(id_msc_pairs)


# Boston Housing Study (Python)
# using data from the Boston Housing Study case
# as described in "Marketing Data Science: Modeling Techniques
# for Predictive Analytics with R and Python" (Miller 2015)

# Here we use data from the Boston Housing Study to evaluate
# regression modeling methods within a cross-validation design.

# program revised by Thomas W. Milller (2017/09/29)
# program revised by John Kiley (2018/07/14)
# program revised by John Kiley (2018/07/21)

# Scikit Learn documentation for this assignment:
# http://scikit-learn.org/stable/modules/model_evaluation.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.model_selection.KFold.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.LinearRegression.html
# http://scikit-learn.org/stable/auto_examples/linear_model/plot_ols.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.Ridge.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.Lasso.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.ElasticNet.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.metrics.r2_score.html

# Textbook reference materials:
# Geron, A. 2017. Hands-On Machine Learning with Scikit-Learn
# and TensorFlow. Sebastopal, Calif.: O'Reilly. Chapter 3 Training Models
# has sections covering linear regression, polynomial regression,
# and regularized linear models. Sample code from the book is
# available on GitHub at https://github.com/ageron/handson-ml

# prepare for Python version 3x features and functions
# comment out for Python 3.x execution
# from __future__ import division, print_function
# from future_builtins import ascii, filter, hex, map, oct, zip

# seed value for random number generators to obtain reproducible results
RANDOM_SEED = 1

# although we standardize X and y variables on input,
# we will fit the intercept term in the models
# Expect fitted values to be close to zero
SET_FIT_INTERCEPT = True

# import base packages into the namespace for this program
import numpy as np
import pandas as pd

# read data for the Boston Housing Study
# creating data frame restdata
boston_input = pd.read_csv('boston.csv')

# check the pandas DataFrame object boston_input
print('\nboston DataFrame (first and last five rows):')
print(boston_input.head())
print(boston_input.tail())

print('\nGeneral description of the boston_input DataFrame:')
print(boston_input.info())

# drop neighborhood from the data being considered
boston = boston_input.drop('neighborhood', 1)
print('\nGeneral description of the boston DataFrame:')
print(boston.info())

print('\nDescriptive statistics of the boston DataFrame:')
print(boston.describe())

# set up preliminary data for data for fitting the models
# the first column is the median housing value response
# the remaining columns are the explanatory variables
prelim_model_data = np.array([boston.mv,\
    boston.crim,\
    boston.zn,\
    boston.indus,\
    boston.chas,\
    boston.nox,\
    boston.rooms,\
    boston.age,\
    boston.dis,\
    boston.rad,\
    boston.tax,\
    boston.ptratio,\
    boston.lstat]).T

print(prelim_model_data)
# dimensions of the polynomial model X input and y response
# preliminary data before standardization
print('\nData dimensions:', prelim_model_data.shape)

# standard scores for the columns... along axis 0
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
print(scaler.fit(prelim_model_data))
# show standardization constants being employed
print(scaler.mean_)
print(scaler.scale_)

# the model data will be standardized form of preliminary model data
model_data = scaler.fit_transform(prelim_model_data)

# dimensions of the polynomial model X input and y response
# all in standardized units of measure
print('\nDimensions for model_data:', model_data.shape)

###############################################################################
# exploratory data analysis
###############################################################################

# import visualization packages
import matplotlib.pyplot as plt

# create a list of plots to make
cols = list(boston)
print(cols)

# loop through columns to produce histograms of the various metrics
for i in cols:
     fig, axes = plt.subplots()
     x = axes.hist(x=boston[i], alpha=0.75)
     axes.set_xlabel('Frequency')
     axes.set_ylabel(i, labelpad=20)
     axes.set_title('Histogram of ' + i,
             fontsize=13)
     fig.savefig('boston_details_'+ i + '.pdf',
                     bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b',
                     orientation='portrait', papertype=None, format=None,
                     transparent=True, pad_inches=0.25, frameon=None)
     plt.show()

     fig, axes = plt.subplots()
     x = axes.scatter(x= boston['mv'], y=boston[i], alpha=0.25)
     axes.set_xlabel('Median Home Value in 1970$s')
     axes.set_ylabel(i, labelpad=20)
     axes.set_title('Scatter MV vs ' + i,
             fontsize=13)
     fig.savefig('boston_scatter'+ i + '.pdf',
                     bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b',
                     orientation='portrait', papertype=None, format=None,
                     transparent=True, pad_inches=0.25, frameon=None)
     plt.show()

###############################################################################
# select models |||| linear models
###############################################################################

# modeling routines from Scikit Learn packages
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.metrics import mean_squared_error, r2_score
from math import sqrt  # for root mean-squared error calculation

# Set list of names and hyper parameters
names = ["Linear Regression", "Ridge Regression",
         "Lasso Regression", "Elastic Net"]
models = [LinearRegression(), Ridge(alpha=1, solver='cholesky'),
          Lasso(alpha=0.1), ElasticNet(alpha=0.1, l1_ratio=0.5)]

###############################################################################
# kfolds validation |||| ten folds
###############################################################################

# --------------------------------------------------------
# specify the k-fold cross-validation design
from sklearn.model_selection import KFold

# ten-fold cross-validation employed here
N_FOLDS = 5

# set up numpy array for storing results
cv_results = np.zeros((N_FOLDS, len(names)))
cv_results2 = np.zeros((N_FOLDS, len(names)))

kf = KFold(n_splits = N_FOLDS, shuffle=False, random_state = RANDOM_SEED)
# check the splitting process by looking at fold observation counts
index_for_fold = 0  # fold count initialized
for train_index, test_index in kf.split(model_data):
    print('\nFold index:', index_for_fold,
          '------------------------------------------')
#   note that 1:model_data.shape[1] slices for explanatory variables
#   and model_data.shape[0] is the index for the response variable
    X_train = model_data[train_index, 0:model_data.shape[1]-1]
    X_test = model_data[test_index,  0:model_data.shape[1]-1]
    y_train = model_data[train_index, model_data.shape[1]-1]
    y_test = model_data[test_index, model_data.shape[1]-1]
    print('\nShape of input data for this fold:',
          '\nData Set: (Observations, Variables)')
    print('X_train:', X_train.shape)
    print('X_test:',X_test.shape)
    print('y_train:', y_train.shape)
    print('y_test:',y_test.shape)

    index_for_method = 0  # initialize
    for name, clf in zip(names, models):
        print('\nClassifier evaluation for:', name)
        print('  Scikit Learn method:', clf)
        clf.fit(X_train, y_train)  # fit on the train set for this fold
        # evaluate on the test set for this fold
        y_test_predict = clf.predict(X_test)
        fold_method_result = sqrt(mean_squared_error(y_test,y_test_predict))
        fold_method_result2 = r2_score(y_test,y_test_predict)
        print('MSE:', fold_method_result)
        print('R2 Score:', fold_method_result2)
        cv_results[index_for_fold, index_for_method] = fold_method_result
        cv_results2[index_for_fold, index_for_method] = fold_method_result2
        index_for_method += 1

    index_for_fold += 1

# set up the R2 and MSE score outputs
cv_results_df = pd.DataFrame(cv_results)
cv_results_df2 = pd.DataFrame(cv_results2)
cv_results_df.columns = names
cv_results_df2.columns = names

cv_results_df2

print('\n----------------------------------------------')
print('Average results from ', N_FOLDS, '-fold cross-validation\n',
      '\nMethod                 MSE', sep = '')
print(cv_results_df.mean())
print('\n----------------------------------------------')
print('Average results from ', N_FOLDS, '-fold cross-validation\n',
      '\nMethod                 R2', sep = '')
print(cv_results_df2.mean())
print('\n----------------------------------------------')
###############################################################################
# Select method, train a model, and create predicted regression line
###############################################################################

# fit regression to full data set
clf = models[3]
X_train = model_data[:, 0:model_data.shape[1]-1]
y_train = model_data[:, model_data.shape[1]-1]
clf.fit(X_train, y_train)

# view the shape of the data
X_train.shape
y_train.shape

# predict
y_predict = clf.predict(X_train)

###############################################################################
# View relevant statistics for the specific model
###############################################################################
# print coefficents

print('Coefficients: \n', list(zip(cols[:][:12],clf.coef_)))
print('\n----------------------------------------------')
# print intercept
print('Intercept: \n', clf.intercept_)

###############################################################################
# View Graph
###############################################################################
from matplotlib.lines import Line2D

fig, axes = plt.subplots()
axes.scatter(y_train,y_predict, color='blue', alpha=0.25)
axes.plot(axes.get_xlim(), axes.get_ylim(), ls="--", c="red")
axes.set_xlabel('actual values')
axes.set_xlim(-4,4)
axes.set_ylabel('predicted values', labelpad=20)
axes.set_ylim(-4,4)
axes.set_title('Regression ',
             fontsize=13)

legend_elements = [Line2D([0], [0], color='R', lw=1, label='Perfect Prediction'),
                   Line2D([0], [0], marker='o', color='w', label='Actual v Predicted',
                          markerfacecolor='b', markersize=15),]

axes.legend(handles=legend_elements)
fig.savefig('regression_output'+ '.pdf',
                     bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b',
                     orientation='portrait', papertype=None, format=None,
                     transparent=True, pad_inches=0.25, frameon=None)


#!/usr/bin/env python3

import argparse
import sys

class Image(object):
  def __init__(self, layers, width, height):
    for l in layers:
      assert l.width == width
      assert l.height == height
    self.layers = layers
    self.width = width
    self.height = height

  def layer_number(self, l):
    for i in range(len(self.layers)):
      if l == self.layers[i]:
        return i
    return None

  @staticmethod
  def parse_from(data, width, height):
    data = data.strip()
    segment_length = width * height
    print("parsing...")
    print("layers:", len(data) // segment_length, "frame size:", segment_length)
    print("width:", width, "height:", height)
    print(len(data), len(data) / segment_length)
    # Turns out this is a bad assert?
    # assert len(data) == segment_length * (len(data) // segment_length)
    segments = []
    for idx in range(len(data) // segment_length):
      segments.append(data[idx * segment_length:(idx+1) * segment_length])

    return Image([Layer(l, width, height) for l in segments], width, height)

class Layer(object):
  def __init__(self, data, width, height):
    assert len(data) == width * height
    self.data = data
    self.width = width
    self.height = height

  def __repr__(self):
    output = ''
    for r in range(self.height):
      output += self.data[r * self.width : (r+1) * self.width] + "\n"

    mapped = {'0': ' ', '1': '#', '2': ' ', '\n': '\n'}
    return "".join([mapped[x] for x in output])

def count(func, iterable):
  c = 0
  for i in filter(func, iterable):
    c += 1
  return c

def p1(img):
  # fewest zeros
  zeros = img.width * img.height + 1
  layer = None
  for l in img.layers:
    c = count(lambda x: x == '0', l.data)
    #print(c, zeros)
    if c < zeros:
      zeros = c
      layer = l

  print("zeros:", zeros)
  ones = count(lambda x: x=='1', layer.data)
  twos = count(lambda x: x=='2', layer.data)
  print("ones:", ones, "twos:", twos)
  print("product:", ones * twos)
  print("from layer:", img.layer_number(layer))

def p2(img):
  # flatten the image
  # 0 - black
  # 1 - weight
  # 2 - transparent

  for l in img.layers:
    #print(l)
    pass

  out = []
  for pos in range(img.height * img.width):
    pixel = '2'
    for l in img.layers:
      pixel = l.data[pos]
      if pixel != '2':
        break
    out.append(pixel)
  assert len(out) == img.width * img.height
  print("---")
  out = Layer("".join(out), img.width, img.height)
  print(out)

if __name__ == "__main__":

  parser = argparse.ArgumentParser()
  parser.add_argument('input', type=argparse.FileType('r'), nargs='?', default=sys.stdin)
  parser.add_argument('--width', default=25, type=int)
  parser.add_argument('--height', default=6, type=int)

  args = parser.parse_args(sys.argv[1:])

  data = args.input.read()
  img = Image.parse_from(data, args.width, args.height)

  p1(img)
  p2(img)


# Copyright 2023 The KerasNLP Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for loading pretrained model presets."""

import pytest
from absl.testing import parameterized

from keras_nlp.backend import ops
from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
from keras_nlp.models.xlm_roberta.xlm_roberta_classifier import (
    XLMRobertaClassifier,
)
from keras_nlp.models.xlm_roberta.xlm_roberta_preprocessor import (
    XLMRobertaPreprocessor,
)
from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
    XLMRobertaTokenizer,
)
from keras_nlp.tests.test_case import TestCase

@pytest.mark.large
@pytest.mark.tf_only  # TODO: jax OOM.
class XLMRobertaPresetSmokeTest(TestCase):
    """
    A smoke test for XLM-RoBERTa presets we run continuously.

    This only tests the smallest weights we have available. Run with:
    `pytest keras_nlp/models/xlm_roberta/xlm_roberta_presets_test.py --run_large`
    """

    def test_tokenizer_output(self):
        tokenizer = XLMRobertaTokenizer.from_preset(
            "xlm_roberta_base_multi",
        )
        outputs = tokenizer("The quick brown fox.")
        expected_outputs = [581, 63773, 119455, 6, 147797, 5]
        self.assertAllEqual(outputs, expected_outputs)

    def test_preprocessor_output(self):
        preprocessor = XLMRobertaPreprocessor.from_preset(
            "xlm_roberta_base_multi",
            sequence_length=4,
        )
        outputs = preprocessor("The quick brown fox.")["token_ids"]
        expected_outputs = [0, 581, 63773, 2]
        self.assertAllEqual(outputs, expected_outputs)

    @parameterized.named_parameters(
        ("preset_weights", True), ("random_weights", False)
    )
    def test_backbone_output(self, load_weights):
        input_data = {
            "token_ids": ops.array([[0, 581, 63773, 2]]),
            "padding_mask": ops.array([[1, 1, 1, 1]]),
        }
        model = XLMRobertaBackbone.from_preset(
            "xlm_roberta_base_multi", load_weights=load_weights
        )
        outputs = model(input_data)
        if load_weights:
            outputs = outputs[0, 0, :5]
            expected = [0.084763, 0.097018, 0.051329, -0.000805, 0.028415]
            self.assertAllClose(outputs, expected, atol=0.01, rtol=0.01)

    @parameterized.named_parameters(
        ("preset_weights", True), ("random_weights", False)
    )
    def test_classifier_output(self, load_weights):
        input_data = ["The quick brown fox."]
        model = XLMRobertaClassifier.from_preset(
            "xlm_roberta_base_multi", num_classes=2, load_weights=load_weights
        )
        # Never assert output values, as the head weights are random.
        model.predict(input_data)

    @parameterized.named_parameters(
        ("preset_weights", True), ("random_weights", False)
    )
    def test_classifier_output_without_preprocessing(self, load_weights):
        input_data = {
            "token_ids": ops.array([[0, 581, 63773, 2]]),
            "padding_mask": ops.array([[1, 1, 1, 1]]),
        }
        model = XLMRobertaClassifier.from_preset(
            "xlm_roberta_base_multi",
            num_classes=2,
            load_weights=load_weights,
            preprocessor=None,
        )
        # Never assert output values, as the head weights are random.
        model.predict(input_data)

    @parameterized.named_parameters(
        ("xlm_roberta_tokenizer", XLMRobertaTokenizer),
        ("xlm_roberta_preprocessor", XLMRobertaPreprocessor),
        ("xlm_roberta", XLMRobertaBackbone),
        ("xlm_roberta_classifier", XLMRobertaClassifier),
    )
    def test_preset_docstring(self, cls):
        """Check we did our docstring formatting correctly."""
        for name in cls.presets:
            self.assertRegex(cls.from_preset.__doc__, name)

    @parameterized.named_parameters(
        ("xlm_roberta_tokenizer", XLMRobertaTokenizer, {}),
        ("xlm_roberta_preprocessor", XLMRobertaPreprocessor, {}),
        ("xlm_roberta", XLMRobertaBackbone, {}),
        ("xlm_roberta_classifier", XLMRobertaClassifier, {"num_classes": 2}),
    )
    def test_unknown_preset_error(self, cls, kwargs):
        # Not a preset name
        with self.assertRaises(ValueError):
            cls.from_preset("xlm_roberta_base_clowntown", **kwargs)

@pytest.mark.extra_large
class XLMRobertaPresetFullTest(TestCase):
    """
    Test the full enumeration of our preset.

    This tests every XLM-RoBERTa preset and is only run manually.
    Run with:
    `pytest keras_nlp/models/xlm_roberta/xlm_roberta_presets_test.py --run_extra_large`
    """

    @parameterized.named_parameters(
        ("preset_weights", True), ("random_weights", False)
    )
    def test_load_xlm_roberta(self, load_weights):
        for preset in XLMRobertaBackbone.presets:
            model = XLMRobertaBackbone.from_preset(
                preset, load_weights=load_weights
            )
            input_data = {
                "token_ids": ops.random.uniform(
                    shape=(1, 512), dtype="int64", maxval=model.vocabulary_size
                ),
                "padding_mask": ops.array([1] * 512, shape=(1, 512)),
            }
            model(input_data)

    @parameterized.named_parameters(
        ("preset_weights", True), ("random_weights", False)
    )
    def test_load_xlm_roberta_classifier(self, load_weights):
        for preset in XLMRobertaClassifier.presets:
            classifier = XLMRobertaClassifier.from_preset(
                preset,
                num_classes=4,
                load_weights=load_weights,
            )
            input_data = ["The quick brown fox."]
            classifier.predict(input_data)

    @parameterized.named_parameters(
        ("preset_weights", True), ("random_weights", False)
    )
    def test_load_xlm_roberta_classifier_without_preprocessing(
        self, load_weights
    ):
        for preset in XLMRobertaClassifier.presets:
            classifier = XLMRobertaClassifier.from_preset(
                preset,
                num_classes=4,
                load_weights=load_weights,
                preprocessor=None,
            )
            input_data = {
                "token_ids": ops.random.uniform(
                    shape=(1, 512),
                    dtype="int64",
                    maxval=classifier.backbone.vocabulary_size,
                ),
                "padding_mask": ops.array([1] * 512, shape=(1, 512)),
            }
            classifier.predict(input_data)

    def test_load_tokenizers(self):
        for preset in XLMRobertaTokenizer.presets:
            tokenizer = XLMRobertaTokenizer.from_preset(preset)
            tokenizer("The quick brown fox.")

    def test_load_preprocessors(self):
        for preset in XLMRobertaPreprocessor.presets:
            preprocessor = XLMRobertaPreprocessor.from_preset(preset)
            preprocessor("The quick brown fox.")


from citylearn import  CityLearn, building_loader, auto_size
from energy_models import HeatPump, EnergyStorage, Building
import matplotlib.pyplot as plt
import torch
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import collections
import gym
from gym.utils import seeding

from gym import core, spaces

import os
import ptan
import time
import argparse

import model, common
from matplotlib.pyplot import figure
import numpy as np

class AgentD4PG(ptan.agent.BaseAgent):
    """
    Agent implementing noisy agent
    """
    def __init__(self, net, device="cpu", epsilon=1.0):
        self.net = net
        self.device = device
        self.epsilon = epsilon

    def __call__(self, states, agent_states):
        states_v = ptan.agent.float32_preprocessor(states).to(self.device)
        mu_v = self.net(states_v)
        actions = mu_v.data.cpu().numpy()
        actions += self.epsilon * np.random.normal(size=actions.shape)
        actions = np.clip(actions, -1, 1)
        return actions, agent_states

class DDPGActor(nn.Module):
    def __init__(self, obs_size, act_size):
        super(DDPGActor, self).__init__()

        self.net = nn.Sequential(
            nn.Linear(obs_size, 4),
            nn.ReLU(),
            nn.Linear(4, 4),
            nn.ReLU(),
            nn.Linear(4, act_size),
            nn.Tanh()
        )

    def forward(self, x):
        return self.net(x)

class DDPGCritic(nn.Module):
    def __init__(self, obs_size, act_size):
        super(DDPGCritic, self).__init__()

        self.obs_net = nn.Sequential(
            nn.Linear(obs_size, 8),
            nn.BatchNorm1d(8),
            nn.ReLU(),
        )

        self.out_net = nn.Sequential(
            nn.Linear(8 + act_size, 6),
            nn.BatchNorm1d(6),
            nn.ReLU(),
            nn.Linear(6, 1)
        )

    def forward(self, x, a):
        obs = self.obs_net(x)
        return self.out_net(torch.cat([obs, a], dim=1))

from pathlib import Path

data_folder = Path("data/")

demand_file = data_folder / "AustinResidential_TH.csv"
weather_file = data_folder / 'Austin_Airp_TX-hour.csv'

#building_ids = [4, 5, 9, 16, 21, 26, 33, 36, 49, 59]
building_ids = [4]

heat_pump, heat_tank, cooling_tank = {}, {}, {}

#Ref: Assessment of energy efficiency in electric storage water heaters (2008 Energy and Buildings)
loss_factor = 0.19/24
buildings = {}
for uid in building_ids:
    heat_pump[uid] = HeatPump(nominal_power = 9e12, eta_tech = 0.22, t_target_heating = 45, t_target_cooling = 10)
    heat_tank[uid] = EnergyStorage(capacity = 9e12, loss_coeff = loss_factor)
    cooling_tank[uid] = EnergyStorage(capacity = 9e12, loss_coeff = loss_factor)
    buildings[uid] = Building(uid, heating_storage = heat_tank[uid], cooling_storage = cooling_tank[uid], heating_device = heat_pump[uid], cooling_device = heat_pump[uid])
    buildings[uid].state_action_space(np.array([24.0, 40.0, 1.001]), np.array([1.0, 17.0, -0.001]), np.array([0.5]), np.array([-0.5]))

building_loader(demand_file, weather_file, buildings)

auto_size(buildings, t_target_heating = 45, t_target_cooling = 10)

env = {}
for uid in building_ids:
    env[uid] = CityLearn(demand_file, weather_file, buildings = {uid: buildings[uid]}, time_resolution = 1, simulation_period = (3500,6000))
    env[uid](uid)

if __name__ == "__main__":
    N_AGENTS = 2
    GAMMA = 0.99
    BATCH_SIZE = 5000
    LEARNING_RATE_ACTOR = 1e-4
    LEARNING_RATE_CRITIC = 1e-3
    REPLAY_SIZE = 5000
    REPLAY_INITIAL = 100
    TEST_ITERS = 120
    EPSILON_DECAY_LAST_FRAME = 1000
    EPSILON_START = 1.2
    EPSILON_FINAL = 0.02

    device = torch.device("cpu")

    act_net, crt_net, tgt_act_net, tgt_crt_net, agent, exp_source, buffer, act_opt, crt_opt, frame_idx = {}, {}, {}, {}, {}, {}, {}, {}, {}, {}
    rew_last_1000, rew, track_loss_critic, track_loss_actor = {}, {}, {}, {}
#     for uid in buildings:
#         env[uid].reset()
    for uid in building_ids:
        #Create as many actor and critic nets as number of agents
        #Actor: states_agent_i -> actions_agent_i
        act_net[uid] = DDPGActor(buildings[uid].observation_spaces.shape[0], buildings[uid].action_spaces.shape[0]).to(device)

        #Critic: states_all_agents + actions_all_agents -> Q-value_agent_i [1]
        crt_net[uid] = DDPGCritic(buildings[uid].observation_spaces.shape[0], buildings[uid].action_spaces.shape[0]).to(device)

        tgt_act_net[uid] = ptan.agent.TargetNet(act_net[uid])
        tgt_crt_net[uid] = ptan.agent.TargetNet(crt_net[uid])

        agent[uid] = model.AgentD4PG(act_net[uid], device=device)
        exp_source[uid] = ptan.experience.ExperienceSourceFirstLast(env[uid], agent[uid], gamma=GAMMA, steps_count=1)
        buffer[uid] = ptan.experience.ExperienceReplayBuffer(exp_source[uid], buffer_size=REPLAY_SIZE)
        act_opt[uid] = optim.Adam(act_net[uid].parameters(), lr=LEARNING_RATE_ACTOR)
        crt_opt[uid] = optim.Adam(crt_net[uid].parameters(), lr=LEARNING_RATE_CRITIC)

        frame_idx[uid] = 0

        rew_last_1000[uid], rew[uid], track_loss_critic[uid], track_loss_actor[uid] = [], [], [], []

    batch, states_v, actions_v, rewards_v, dones_mask, last_states_v, q_v, last_act_v, q_last_v, q_ref_v, critic_loss_v, cur_actions_v, actor_loss_v = {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}
    cost, price_list, buffer_reward = {},{},{}
    for uid in buildings:
        cost[uid] = []
        price_list[uid] = []
        buffer_reward[uid] = []
    while not env[building_ids[-1]]._terminal():
        if frame_idx[4]%100 == 0:
            print(frame_idx[uid])
        for uid in buildings:
#             print(env[uid].time_step)
            agent[uid].epsilon = max(EPSILON_FINAL, EPSILON_START - frame_idx[uid] / EPSILON_DECAY_LAST_FRAME)
            frame_idx[uid] += 1
            buffer[uid].populate(1)
#             print(buffer[uid].buffer[-1])
#             print(env[uid].buildings[uid].time_step)

        price = env[uid].total_electric_consumption[-1]*3e-5 + 0.045
        price_list[uid].append(price)
        for uid in buildings:
            buffer_reward[uid].append(buffer[uid].buffer[-1].reward)
            electricity_cost = buffer[uid].buffer[-1].reward*price
            cost[uid].append(-electricity_cost)
            buffer[uid].buffer[-1] = buffer[uid].buffer[-1]._replace(reward=electricity_cost)

        if len(buffer[uid]) < REPLAY_INITIAL:
            continue

        for uid in buildings:
            for k in range(6):
                batch[uid] = buffer[uid].sample(BATCH_SIZE)
                states_v[uid], actions_v[uid], rewards_v[uid], dones_mask[uid], last_states_v[uid] = common.unpack_batch_ddqn(batch[uid], device)

                # TRAIN CRITIC
                crt_opt[uid].zero_grad()
                #Obtaining Q' using critic net with parameters teta_Q'
                q_v[uid] = crt_net[uid](states_v[uid], actions_v[uid])

                #Obtaining estimated optimal actions a|teta_mu from target actor net and from s_i+1.
                last_act_v[uid] = tgt_act_net[uid].target_model(last_states_v[uid]) #<----- Actor to train Critic

                #Obtaining Q'(s_i+1, a|teta_mu) from critic net Q'
                q_last_v[uid] = tgt_crt_net[uid].target_model(last_states_v[uid], last_act_v[uid])
                q_last_v[uid][dones_mask[uid]] = 0.0

                #Q_target used to train critic net Q'
                q_ref_v[uid] = rewards_v[uid].unsqueeze(dim=-1) + q_last_v[uid] * GAMMA
                critic_loss_v[uid] = F.mse_loss(q_v[uid], q_ref_v[uid].detach())
                critic_loss_v[uid].backward()
                crt_opt[uid].step()

                # TRAIN ACTOR
                act_opt[uid].zero_grad()
                #Obtaining estimated optimal current actions a|teta_mu from actor net and from s_i
                cur_actions_v[uid] = act_net[uid](states_v[uid])

                #Actor loss = mean{ -Q_i'(s_i, a|teta_mu) }
                actor_loss_v[uid] = -crt_net[uid](states_v[uid], cur_actions_v[uid]) #<----- Critic to train Actor
                actor_loss_v[uid] = actor_loss_v[uid].mean()
                #Find gradient of the loss and backpropagate to perform the updates of teta_mu
                actor_loss_v[uid].backward()
                act_opt[uid].step()

                if frame_idx[uid] % 1 == 0:
                    tgt_act_net[uid].alpha_sync(alpha=1 - 0.1)
                    tgt_crt_net[uid].alpha_sync(alpha=1 - 0.1)

from matplotlib.pyplot import figure

#Plotting all the individual actions
print(env)

figure(figsize=(18, 6))
for uid in buildings:
    print(env[uid].buildings[uid].time_step)
    plt.plot(env[uid].action_track[uid][2400:2500])
plt.show()