# -*- coding: utf-8 -*-
"""
Created on Sat Nov  2 22:41:49 2019

@author: Veda
"""
import numpy as np
import pandas as pd
from bokeh.plotting import figure, output_file, show
from bokeh.models import ColumnDataSource, Plot, LinearAxis, Grid
from bokeh.models.tools import HoverTool
from bokeh.palettes import Spectral5
from bokeh.transform import factor_cmap
from bokeh.layouts import row,column
from bokeh.io import *
from bokeh.layouts import *
from bokeh.plotting import *
from bokeh.models.widgets import *
from bokeh.models.widgets import MultiSelect
from bokeh.layouts import widgetbox
from bokeh.layouts import gridplot
output_file("dropdown.html")

menu = [("TELANGANA", "item_1"), ("ANDHRA PRADESH", "item_2")]
dropdown = Dropdown(label="REGION/STATE", button_type="default", width=130, menu=menu)

menu1 = [("KARIMNAGAR", "item_1"), ("MAHABUBNAGAR", "item_2")]
dropdown1 = Dropdown(label="DISTRICTS", button_type="default", width=123
                    , menu=menu1)

bt = Button(label="MTD", button_type="default", width=48)
bt1 = Button(label="QTD", button_type="default", width=48)
bt2 = Button(label="YTD", button_type="default", width=48)
bt3 = Button(label="CUSTOM", button_type="default", width=85)

menu2 = [("Trade", "item_1"), ("Non-Trade", "item_2")]
dropdown2 = Dropdown(label="CUSTOMER TYPE", button_type="default", menu=menu2,width=144)

menu3 = [("OPC53", "item_1"), ("OPC43", "item_2"),("PPC", "item_3"),("PSC", "item_4")]
dropdown3 = Dropdown(label="PRODUCT GRADE", button_type="default", menu=menu3, width=140)

menu4 = [None]
dropdown4 = Dropdown(label="OFFICER LEVEL", button_type="default", menu=menu4, width=140)

menu5 = [("Bulk", "item_1"), ("Bag", "item_2")]
dropdown5 = Dropdown(label="PACKING TYPE", button_type="default", menu=menu5,width=140
                    )

bt8 = Button(label="DEFAULT/RESET", button_type="default",width=140)

layout=layout(column(row(dropdown,dropdown1,bt, bt1, bt2, bt3, dropdown2,dropdown3, dropdown4,dropdown5, bt8)))
#show(layout)

df = pd.read_excel(r"C:/Users/Veda/Downloads/Book1.xlsx")

sales_data=df.groupby(df['Inv Date'].dt.strftime('%B'))['Sales Qty'].sum().sort_values()
#sales_data1=df.sort_values(by='Inv Date', ascending= False)
grouped = sales_data/ 1000
print(grouped)

source = ColumnDataSource(pd.DataFrame(grouped))
states = source.data['Inv Date'].tolist()
p = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#FFFF99")

p.title.text ='Total Sales'
p.title.align = 'center'
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.yaxis.visible = False
p.xaxis.visible=True
p.outline_line_width = 7
p.outline_line_alpha = 0.3
p.outline_line_color = "#E5FFCC"

p1 = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p1.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#FFCC99")

p1.title.text ='Realization'
p1.title.align = 'center'
p1.xgrid.grid_line_color = None
p1.ygrid.grid_line_color = None
p1.yaxis.visible = False
p1.xaxis.visible=True
p1.outline_line_width = 7
p1.outline_line_alpha = 0.3
p1.outline_line_color = "#E5FFCC"

p2 = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p2.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#CCFFCC")

p2.title.text ='Collections'
p2.title.align = 'center'
p2.xgrid.grid_line_color = None
p2.ygrid.grid_line_color = None
p2.yaxis.visible = False
p2.xaxis.visible=True
p2.outline_line_width = 7
p2.outline_line_alpha = 0.3
p2.outline_line_color = "#E5FFCC"
p3 = figure(x_range=states, plot_width=250, plot_height=200)

color_map = factor_cmap(field_name='Inv Date',
                    palette=Spectral5, factors=states)
p3.vbar(x='Inv Date',top='Sales Qty', source=source, width=0.70,color="#CCE5FF")

p3.title.text ='Over Due'
p3.title.align = 'center'
p3.xgrid.grid_line_color = None
p3.ygrid.grid_line_color = None
p3.yaxis.visible = False
p3.xaxis.visible=True
p3.outline_line_width = 7
p3.outline_line_alpha = 0.3
p3.outline_line_color = "#E5FFCC"

grouped=df.groupby(['Region Description'])['Naked Realization(Rs'].sum()
grouped=(grouped)/10000000
source = ColumnDataSource(pd.DataFrame(grouped))
states = source.data['Region Description'].tolist()
p5 = figure(y_range=states)
color_map = factor_cmap(field_name='Region Description',
                    palette=Spectral5, factors=states)

p5.hbar(y='Region Description',source=source,right="Naked Realization(Rs",left=0,height=0.5, color="#FFFF99")

p5.title.text ='Naked Realization by state'
p5.xaxis.axis_label = 'Naked Realization in Rs'
p5.yaxis.axis_label = 'state'
hover = HoverTool()
hover.tooltips = [
    ("Totals","@Naked Realization(Rs")]

hover.mode = 'hline'

p.add_tools(hover)

hover = HoverTool(tooltips=[("sales","@Sales%20Qty,")],mode='vline')
#layout.add_tools(hover)

layout1= row (p,p1,p2,p3)
layout2= column(layout,layout1,p5)

#curdoc().add_root(layout)
show(layout2)


"""
    Logger Agent

    @author: jldupont
    @date: May 21, 2010
"""
import os
import logging
from eventor.system.base    import AgentThreadedBase

__all__=["LoggerAgent"]

class LoggerAgent(AgentThreadedBase):

    mlevel={"info":     logging.INFO
            ,"warning": logging.WARNING
            ,"error":   logging.ERROR
            }

    def __init__(self, app_name, logpath):
        """
        @param interval: interval in seconds
        """
        AgentThreadedBase.__init__(self)
        self._logger=None
        self.fhdlr=None
        self._shutdown=False
        self.appname=app_name
        self.logpath=os.path.expanduser(logpath)
        self._setup()

    def _setup(self):
        if self._logger is not None:
            return

        self._logger=logging.getLogger(self.appname)

        path=os.path.expandvars(os.path.expanduser(self.logpath))
        self.fhdlr=logging.FileHandler(path)

        formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
        self.fhdlr.setFormatter(formatter)
        self._logger.addHandler(self.fhdlr)
        self._logger.setLevel(logging.INFO)

    def h_shutdown(self):
        if self._logger:
            self._shutdown=True
            logging.shutdown([self.fhdlr])

    def h_log(self, *arg):
        if self._shutdown:
            return

        self._setup()

        if len(arg) == 1:
            self._logger.log(logging.INFO, arg[0])
        else:
            level=self.mlevel.get(arg[0], logging.INFO)
            self._logger.log(level, arg[1])

"""
_=LoggerAgent()
_.start()
"""


import pandas as pd
import gspread
from oauth2client.service_account import ServiceAccountCredentials
import numpy as np
from classes import specie, pm_doc
import math
import logging
from theone_con import from_theone
logging.basicConfig(level=logging.INFO) # DEBUG, INFO, WARN

#Storing the links on gdrive for every site
gsheets = {
    'ROSPA0004' : 'https://docs.google.com/spreadsheets/d/1FRh6F8wh9iEcAOI_7gt-h7YDm38YaraEsMl7BCs9c_c/edit#gid=0',
    'ROSPA0012' : 'https://docs.google.com/spreadsheets/d/1oeUT7zVmWmNeIxOuCNX2xNZ8KNxlGA-_Mt7QNYW_Z1o/edit#gid=0',
    'ROSPA0112' : 'https://docs.google.com/spreadsheets/d/1nM5UR7hykwL0mffkkd6YQIvpZzHUPfJBEfrfxf_llHI/edit#gid=0',
    'ROSPA0111' : 'https://docs.google.com/spreadsheets/d/1LrmO_aMmvEyCzuAtsiejMjtSzADsAkQsIGuQPX6hxug/edit#gid=0',
    'ROSPA0109' : 'https://docs.google.com/spreadsheets/d/1H4epQ_iR9jZGugIlo7i9pGrP5Hmf8IocBiuRBUscKLA/edit#gid=0',
    'ROSPA0101' : 'https://docs.google.com/spreadsheets/d/1XNYEqTLr4EuL9Ghb3x8OhWno9VYqRgmtsNpV4icCX5o/edit#gid=0',
    'ROSPA0064' : 'https://docs.google.com/spreadsheets/d/1jxUQQU15eAe6HnLVuNGHDYbi-xFYuc5UbG2BT7eKKdA/edit#gid=0',
    'ROSPA0061' : 'https://docs.google.com/spreadsheets/d/1awwKOi1lY8KTQeRPp3D7BS_mLB3oBXrb98VgyJyvWeY/edit#gid=0',
    'ROSPA0051' : 'https://docs.google.com/spreadsheets/d/1ck3Q4dv55TdI7Uo98qZ4-E0-2-xxaWzq9WBL5X0SSKk/edit#gid=0',
    'ROSPA0042' : 'https://docs.google.com/spreadsheets/d/1A_NPum_EEw5eHNO7AAqm4ZFkezFq07ZRQj32UMQp-YE/edit#gid=0',
    'descrieri' : 'https://docs.google.com/spreadsheets/d/13qN_1-CXxNzcUNV52s8tERspjqxij4Rn0uyHSbPSebg/edit#gid=839134727'}

#Setting the authentification
scope = ['https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive']
credentials = ServiceAccountCredentials.from_json_keyfile_name('Driveacc-017ff6c04252.json', scope)
gc = gspread.authorize(credentials)

#Getting the site
the_site = 'ROSPA0'+input('What are the last 3 numbers on your site code?  ')

#Getting the table for the site from db
bf = from_theone("select * from poim.sit_table('"+the_site+"')")
bf.fillna('na', inplace=True)

#Getting the master table from gdrive
mst = gc.open_by_url(gsheets[the_site]).worksheet('master')
master = mst.get_all_records()
headers_m = master.pop(0)
df_master = pd.DataFrame(master, columns = headers_m)
df_master = df_master[df_master.id != '']
df_master.fillna('na', inplace=True)

#Getting the impacts table from gdrive
imp = gc.open_by_url(gsheets[the_site]).worksheet('presiuni_sit')
impacts = imp.get_all_records()
df_impacts = pd.DataFrame(impacts)
df_impacts = df_impacts[df_impacts.impact != '']
df_impacts.fillna('na', inplace=True)

#Getting the measures table from gdrive
mas = gc.open_by_url(gsheets[the_site]).worksheet('masuri_sit')
masuri = mas.get_all_records()
df_masuri = pd.DataFrame(masuri)
df_masuri = df_masuri[df_masuri.masura != '']
df_masuri.fillna('na', inplace=True)

#Getting the descriptions table from gdrive
desc = gc.open_by_url(gsheets['descrieri']).worksheet('descrieri_tst')
descrieri = desc.get_all_records()
df_desc = pd.DataFrame(descrieri)
df_desc.dropna(inplace=True)
df_desc.fillna('na', inplace=True)

#Getting the list of the rows from the master table
df_rows = list(df_master.index)

#Creation of the document
doc = pm_doc()

#Open the text file for gis
f=open(the_site + '_gis_final.txt','w+', encoding="utf-8")

                #-------CREATION OF THE CONSERVATION TABLES------

#Itering through rows and creating species objects&tables
for row in df_rows:
    #Initialize sp object
    sp = specie(row, df_master, bf, df_desc, df_impacts, df_masuri)
    f.write(str(sp.cod_sp) +','+ str(sp.lat_sp) + ',' + str(sp.cod_n)+'-'+str(sp.feno) + ',' + str(sp.d3)+'\n')

    #Put the species header
    doc.sp_header(sp)

    #Create the population based conservation chapter
    doc.chapter_h2('Evaluarea stării de conservare a speciei din punctul de vedere al populației speciei')
    #Add the table title
    doc.chapter_h3('Tabelul A: Parametri pentru evaluarea stării de conservare a speciei din punct de vedere al populației')
    #Create the population based conservation tabels (tabels A)
    doc.ftblA_cons()
    doc.ftblA_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 1) Matricea de evaluare a stării de conservare a speciei din punct de vedere al populației speciei')
    doc.ftblM_1()
    doc.ftblM_1_fill(sp)

    #Create the habitat based conservation chapter
    doc.chapter_h2('Evaluarea stării de conservare a speciei din punctul de vedere al habitatului speciei')
    #Add the table title
    doc.chapter_h3('Tabelul B: Parametri pentru evaluarea stării de conservare a speciei din punct de vedere al habitatului speciei')
    #Create the habitat based conservation tabels (tabels B)
    doc.ftblB_cons()
    doc.ftblB_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 2) Matricea pentru evaluarea tendinței globale a habitatului speciei')
    doc.ftblM_2()
    doc.ftblM_2_fill(sp)
    doc.chapter_h3('Matricea 3) Matricea de evaluare a stării de conservare a speciei din punct de vedere al habitatului speciei')
    doc.ftblM_3()
    doc.ftblM_3_fill(sp)

    #Create the perspectives based conservation chapter
    doc.chapter_h2('Evaluarea stării de conservare a speciei din punctul de vedere al perspectivelor speciei')
    #Add the table title
    doc.chapter_h3('Tabelul C: Parametri pentru evaluarea stării de conservare a speciei din punct de vedere al perspectivelor speciei în viitor')
    #Create the perspectives based conservation tabels (tabels C)
    doc.ftblC_cons()
    doc.ftblC_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 4) Matricea pentru evaluarea perspectivelor speciei din punct de vedere al populației speciei')
    doc.ftblM_4()
    doc.ftblM_4_fill(sp)
    doc.chapter_h3('Matricea 5) Perspectivele speciei în viitor, după implementarea planului de management actual')
    doc.ftblM_5()
    doc.ftblM_5_fill(sp)
    doc.chapter_h3('Matricea 6) Matricea evaluării stării de conservare a speciei din punct de vedere al perspectivelor speciei în viitor, după implementarea planului de management actual')
    doc.ftblM_6()
    doc.ftblM_6_fill(sp)

    #Create the global conservation chapter
    doc.chapter_h2('Evaluarea globală a speciei')
    #Add the table title
    doc.chapter_h3('Tabelul D: Parametri pentru evaluarea stării globale de conservare a speciei în cadrul ariei naturale protejate')
    #Create the global conservation tabels (tabels D)
    doc.ftblD_cons()
    doc.ftblD_cons_fill(sp)
    #Adding matrices tables
    doc.chapter_h3('Matricea 7) Evaluarea stării globale de conservare a speciei')
    doc.ftblM_7()
    doc.ftblM_7_fill(sp)

f.close()
                #-------CREATION OF THE MEASURES TABLES------

doc.chapter_h1('Măsurile de conservare pentru menținerea sau îmbunătățirea stării de conservare a speciilor de păsări')

doc.measures_ch(df_masuri, df_impacts)

#Salvation ^^ of the document
doc.save(the_site+'_export_final'+'.docx')


import pickle
import numpy as np
import matplotlib.pyplot as plt

def main():
	episodes=200
	steps=200
	lambdas=[0,0.3,0.7,0.9,1]
	alphas=[1/4,1/8,1/16]
	colors=['r','b','g']
	with open('data_2.pkl', 'rb') as handle:
		data = pickle.load(handle)

	x=np.linspace(1,episodes,episodes)
	for i in range(len(data)): ## lambdas
		plt.figure()
		for j in range(len(data[i])): ## alphas
			data_plot=np.zeros((data[0][0][0].shape))
			count=0
			for k in range(len(data[i][j])): ## runs
				data_plot+=data[i][j][k]
				count+=1
			data_plot/=count
			data_inf= 'alpha = '+str(alphas[j])
			plt.plot(x,data_plot,colors[j], label=data_inf)

		plt.ylabel('Value of theta = 0, velocity=0')
		plt.xlabel('Episodes')
		plt.legend()
		plt.title('lambda = '+str(lambdas[i])+', 10 runs with 200 episodes')
		plt.savefig('lambda_'+str(lambdas[i])+'.png')

if __name__ == "__main__":
	main()


# DNN Modeling

import numpy as np
from numpy import array

                        # 함수는 keras23_multiple2에서 그대로 copy
def split_sequence3(sequence, n_steps):
    X,y = list(), list()
    for i in range(len(sequence)):
        end_ix = i + n_steps
        if end_ix > len(sequence)-1: # 수정
            break
        seq_x, seq_y = sequence[i:end_ix, :], sequence[end_ix, :] # 수정
        X.append(seq_x) # X 리스트에 추가
        y.append(seq_y) # y 리스트에 추가
    return array(X), array(y) # numpy 배열로 만들기

in_seq1 = array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
in_seq2 = array([15, 25, 35, 45, 55, 65, 75, 85, 95, 105])
out_seq = array([in_seq1[i] + in_seq2[i] for i in range(len(in_seq1))]) # 인자를 두 개씩 더한다!

# print(in_seq1.shape)# (10,) 벡터
# print(out_seq.shape)# (10,) 벡터

# 차원 변경(벡터(스칼라 수, ) => 행렬(행, 열))
in_seq1 = in_seq1.reshape(len(in_seq1), 1) # 그냥 10으로 써도 되지만 가능한 변수를 사용하자
in_seq2 = in_seq2.reshape(len(in_seq2), 1)
out_seq = out_seq.reshape(len(out_seq), 1)

# print(in_seq1.shape) # (10, 1)
# print(in_seq2.shape) # (10, 1)
# print(out_seq.shape) # (10, 1)

dataset = np.hstack((in_seq1, in_seq2, out_seq)) # 참고 : https://m.blog.naver.com/PostView.nhn?blogId=wideeyed&logNo=221205882962&proxyReferer=https%3A%2F%2Fwww.google.com%2F
n_steps = 3

# print(dataset)

x, y = split_sequence3(dataset, n_steps)

for i in range(len(x)):
    print(x[i], y[i])

print(x.shape)
print(y.shape)

x = x.reshape(x.shape[0], -1) # 가능한 변수를 이용해서 작업할 것! # -1 => 컴퓨터에서 숫자에 맞게 알아서 넣으라는 명령

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.7)

# 4. 모델 구성
from keras.models import Sequential, Model
from keras.layers import Dense, Input, LSTM, Dropout, BatchNormalization

input1 = Input(shape=(9,))
dense1 = Dense(256)(input1)
# dense1 = BatchNormalization()
dense1 = Dense(256)(dense1)
dense1 = Dense(256)(dense1)
dense1 = Dense(256)(dense1)
dense1 = Dense(256)(dense1)
# dense1 = Dropout(0.2))
output1 = Dense(3)(dense1)

model = Model(inputs=input1, outputs=output1)

# td_hist = TensorBoard(log_dir='./graph',
#                       histogram_freq=0,
#                       write_graph=True,
#                       write_images=True)

# 5. 모델 훈련
# from keras.callbacks import EarlyStopping, TensorBoard

# early_stopping = EarlyStopping(monitor='loss', patience=80, mode='auto')
model.compile(loss='mse', optimizer='adam',
              metrics=['mae']) # adam=평타는 침. # 이 때문에 아래서 acc가 나온다.
model.fit(x_train, y_train, epochs=150, batch_size=1, verbose=2)

# 6. 평가예측
loss, mse = model.evaluate(x_test, y_test, batch_size=1) # loss는 자동적으로 출력
# mae = mean_absolute_error, mse와 다른 손실함수
# 데이터 크기보다 더 큰 batch size를 줄 경우 데이터 크기로 계산된다.
print('loss :', loss)

# model.summary()

# 7. 새로운 데이터 넣어보기
x_prd = array([[90, 95, 105],[100, 105, 115],[110, 115, 125]])
x_prd = x_prd.reshape(1, -1) # -1 => 알아서 넣어주기
g = model.predict(x_prd, batch_size=1)
print(g)

y_predict = model.predict(x_test, batch_size=1)

# 8. RMSE 구하기
from sklearn.metrics import mean_squared_error

def RMSE(y_test, y_predict): # 실제 정답값, 모델을 통한 예측값
    return np.sqrt(mean_squared_error(y_test, y_predict))
print('RMSE :', RMSE(y_test, y_predict))

# 9. R2 구하기
from sklearn.metrics import r2_score

r2_y_predict = r2_score(y_test, y_predict)
print("R2 :", r2_y_predict)


import json

from decouple import config
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail

from django.core.exceptions import PermissionDenied
from django.http import HttpResponse
from django.shortcuts import redirect, render, get_object_or_404
from django.views.decorators.csrf import csrf_exempt

from .models import RelayAddress, Message

@csrf_exempt
def index(request):
    if not request.user:
        raise PermissionDenied
    user_profile = request.user.profile_set.first()
    if not request.POST['api_token']:
        raise PermissionDenied
    if not str(request.POST['api_token']) == str(user_profile.api_token):
        raise PermissionDenied
    RelayAddress.objects.create(user=request.user)
    return redirect('profile')

def messages(request):
    if not request.user:
        raise PermissionDenied
    relay_address_id = request.GET.get('relay_address_id', False)
    if not relay_address_id:
        raise Http404("Relay address not found")
    relay_address = RelayAddress.objects.get(id=relay_address_id)
    if relay_address.user != request.user:
        raise PermissionDenied
    messages = Message.objects.filter(relay_address=relay_address)
    return render(request, 'emails/messages.html',{
        'messages': messages,
        'relay_address': relay_address
    })

@csrf_exempt
def inbound(request):
    email_to = request.POST.get('to')
    local_portion = email_to.split('@')[0]
    from_address = request.POST.get('from')
    subject = request.POST.get('subject')
    text = request.POST.get('text')
    print("email_to: %s" % email_to)
    print("local_portion: %s" % local_portion)
    print("from_address: %s" % from_address)
    print("subject: %s" % subject)
    print("text: %s" % text)

    relay_address = get_object_or_404(RelayAddress, address=local_portion)

    # Store in local DB
    message = Message.objects.create(
        relay_address=relay_address,
        from_address=from_address,
        subject=subject,
        message=text
    )

    # Forward to real email address
    try:
        message = Mail(
            from_email='inbound@privaterelay.groovecoder.com',
            to_emails=relay_address.user.email,
            subject='Forwarding email from %s sent to %s' % (
                from_address, local_portion
            ),
            html_content=text
        )
        sendgrid_client = SendGridAPIClient(config('SENDGRID_API_KEY'))
        response = sendgrid_client.send(message)
        print(response)
    except Exception as e:
        print(e.message)

    return HttpResponse(message)


import sys
import os
import re
import xlwt
# import pandas as pd
import xlwt, xlrd
from xlutils.copy import copy

def default_process():
    if (len(sys.argv) == 3):
        path = sys.argv[1]
        filename = sys.argv[2]
    elif (len(sys.argv) == 2):
        if (sign in sys.argv[1] for sign in ['/', '\\']):
            path = sys.argv[1]
            filename = 'All'
        else:
            path = './'
            filename = sys.argv[1]
    else:
        path = './'
        filename = 'All'

    return path, filename

def write_file(filepath, filename, mode='r'):
    if filename == 'All':
        listdir = os.listdir(filepath)
        print(listdir)
        for name in listdir:
            # if name == '...': continue
            process_file(filepath, name)
    else:
        process_file(filepath, filename)
    return

def write_txt(filename, data, data_header):
    with open(filename, 'a+') as w:
        for key in data.keys():
            data_line = ' '.join(data[key])
            data_line = data_header + ' ' + key + ' ' + data_line

        w.writelines([data_line, '\n'])
        w.flush()

def write_excel_xls(path, data, tablename, sec, opcs, avg_opc):
    # index = len(value)  # 获取需要写入数据的行数
    # print('1')
    workbook = xlwt.Workbook()  # 新建一个工作簿
    # sheet_name = []
    for key in data.keys():
        name = key
        # sheet_name.append(name)
        sheet = workbook.add_sheet(name)  # 在工作簿中新建一个表格
        for i, n in enumerate(tablename):
            sheet.write(0, i , n)

        sheet.write(1, 0, sec)
        sheet.write(1, 1, opcs)
        sheet.write(1, 2, avg_opc)

        for i in range(len(data[key])):
            sheet.write(1, i + 3, data[key][i])
    workbook.save(path)  # 保存工作簿
    # print("xls格式表格写入数据成功！")

def write_excel_xls_append(path, data, tablename, sec, opcs, avg_opc):
    # index = len(value)  # 获取需要写入数据的行数

    # print(sheets)
    for key in data.keys():
        workbook = xlrd.open_workbook(path)  # 打开工作簿
        sheets = workbook.sheet_names()  # 获取工作簿中的所有表格
        if key not in sheets:
            # print(key)
            new_workbook = copy(workbook)
            # new_worksheet = new_workbook.add_sheet(key)

            sheet = new_workbook.add_sheet(key)
            for i, n in enumerate(tablename):
                sheet.write(0, i, n)

            sheet.write(1, 0, sec)
            sheet.write(1, 1, opcs)
            sheet.write(1, 2, avg_opc)

            for i in range(len(data[key])):
                sheet.write(1, i + 3, data[key][i])

            new_workbook.save(path)
        else:
            # print("1")
            # print(key)
            worksheet = workbook.sheet_by_name(key)
            rows_old = worksheet.nrows
            new_workbook = copy(workbook)
            new_worksheet = new_workbook.get_sheet(key)

            new_worksheet.write(rows_old, 0, sec)
            new_worksheet.write(rows_old, 1, opcs)
            new_worksheet.write(rows_old, 2, avg_opc)

            for i in range(len(data[key])):
                new_worksheet.write(rows_old, i + 3, data[key][i])
            new_workbook.save(path)
            # print("xls格式表格【追加】写入数据成功！")

def process_file(filepath, filename):
    # print("filepath : %s\nfilename : %s" % (filepath, filename))
    with open(filepath + filename, 'r') as f:
        # print('1')
        flag = 0  # excel第一次写入标志位

        for line in f:
            # print('2',line)
            if (line.find('Command line') != -1):
                # print('3', line)
                DB_pat = re.compile('com\.yahoo\.ycsb\.db\.(.*)Client')
                Thread_pat = re.compile('workloads/workload(.) -s -threads (.*) -t')
                mbj1 = DB_pat.search(line)
                mbj2 = Thread_pat.search(line)
                if mbj1 and mbj2:
                    wname = mbj1.group(1)
                    postx_txt = ".%s.%s.txt" % (mbj2.group(1), mbj2.group(2))
                    postx_xls = ".%s.%s.xls" % (mbj2.group(1), mbj2.group(2))
                # print(wname+postx_xls)
            elif 'wname' and 'postx_txt' in locals().keys():
                opc_pat = re.compile('\[([^\[\]]*)?\]')
                list_opc = opc_pat.findall(line)
                # print(line, len(list_opc), list_opc)
                if len(list_opc) > 1:
                    sec_pat = re.compile(' ([^ ]*) sec:')
                    pat_opc = re.compile('sec: (.*) operations; (.*) current ops/sec;')

                    sec = sec_pat.search(line).group(1)
                    opcs = pat_opc.search(line).group(1)
                    avg_opc = pat_opc.search(line).group(2)

                    data = {}

                    for i in range(len(list_opc)):
                        opc = list_opc[i].split()
                        opc_name = opc[0][:-1]
                        opc_count = opc[1][6:-1]
                        opc_maxL = opc[2][4:-1]
                        opc_minL = opc[3][4:-1]
                        opc_avg = opc[4][4:-1]
                        opc_90 = opc[5][3:-1]
                        opc_99 = opc[6][3:-1]
                        opc_999 = opc[7][5:-1]
                        opc_9999 = opc[8][6:]

                        data[opc_name] = [opc_count, opc_maxL,
                                          opc_minL, opc_avg,
                                          opc_90, opc_99, opc_999, opc_9999]

                    data_header = ' '.join([sec, opcs, avg_opc])

                    tablename = ['SEC(m)', 'TOTAL_OPC', 'QPS(ops/sec)', 'OPC_COUNT', 'OPC_MAXL(us)',
                                 'OPC_MINL(us)','OPC_AVG', '90%(us)', '99%(us)', '99.9%(us)', '99.99%(us)']

                    #  去掉注释 输出txt文件
                    #  write_txt(wname+postx_txt, data, data_header)

                    if flag == 0:
                        # print("1")
                        # print(avg_opc)
                        write_excel_xls(wname + postx_xls, data, tablename, sec, opcs, avg_opc)
                        # print("2")
                        flag = 1
                    else:
                        write_excel_xls_append(wname + postx_xls, data, tablename, sec, opcs, avg_opc)

            else:
                continue

if __name__ == '__main__':
    filepath, filename = default_process()
    print("filepath : %s\nfilename : %s" % (filepath, filename))
    # print(filepath,filename)
    write_file(filepath, filename)


import pytest

import bmds

def test_dataset_validation():
    # make dummy datasets
    dummy3 = [1, 2, 3]
    dummy4 = [1, 2, 3, 4]

    # these should be valid
    bmds.DichotomousDataset(
        doses=dummy3, ns=dummy3, incidences=dummy3)
    bmds.ContinuousDataset(
        doses=dummy3, ns=dummy3, responses=dummy3, stdevs=dummy3)

    # these should raise errors
    with pytest.raises(ValueError):
        # different sized lists
        bmds.DichotomousDataset(
            doses=dummy4, ns=dummy3, incidences=dummy3)
        bmds.ContinuousDataset(
            doses=dummy4, ns=dummy3, responses=dummy3, stdevs=dummy3)

        # 2 remaining after dropping-doses
        bmds.DichotomousDataset(
            doses=dummy3, ns=dummy3, incidences=dummy3,
            doses_dropped=1)
        bmds.ContinuousDataset(
            doses=dummy3, ns=dummy3, responses=dummy3, stdevs=dummy3,
            doses_dropped=1)

def test_dfile_outputs():
    dummy4 = [1, 2, 3, 4]

    # check dichotomous
    ds = bmds.DichotomousDataset(
        doses=dummy4, ns=[5, 5, 5, 5], incidences=[0, 1, 2, 3],
        doses_dropped=1)
    dfile = ds.as_dfile()
    expected = 'Dose Incidence NEGATIVE_RESPONSE\n1.000000 0 5\n2.000000 1 4\n3.000000 2 3'  # noqa
    assert dfile == expected

    # check continuous
    ds = bmds.ContinuousDataset(
        doses=dummy4, ns=dummy4, responses=dummy4, stdevs=dummy4,
        doses_dropped=1)
    dfile = ds.as_dfile()
    expected = 'Dose NumAnimals Response Stdev\n1.000000 1 1.000000 1.000000\n2.000000 2 2.000000 2.000000\n3.000000 3 3.000000 3.000000'  # noqa
    assert dfile == expected

def test_doses_used():
    ds5 = [1, 2, 3, 4, 5]

    ds = bmds.DichotomousDataset(ds5, ds5, ds5)
    assert ds.doses_used == 5
    ds = bmds.DichotomousDataset(ds5, ds5, ds5, doses_dropped=2)
    assert ds.doses_used == 3

    ds = bmds.ContinuousDataset(ds5, ds5, ds5, ds5)
    assert ds.doses_used == 5
    ds = bmds.ContinuousDataset(ds5, ds5, ds5, ds5, doses_dropped=2)
    assert ds.doses_used == 3


'''
Creating to run Model D Conv1d_mfcc40
different layer system
full mfcc features
Created on 2020.12.08 at 1843

'''
# libraries
import numpy as np
import pandas as pd
import pickle
import seaborn as sns
import tensorflow as tf

from tensorflow.keras.layers import Dense, Dropout, Flatten, Input
from tensorflow.keras.layers import Activation, BatchNormalization
from tensorflow.keras.layers import Conv1D, Conv2D, LSTM
from tensorflow.keras.layers import AveragePooling1D, GlobalAveragePooling2D, MaxPooling1D, MaxPool2D
from tensorflow.keras.models import Model, model_from_json, Sequential

# added this to plot
import matplotlib.pyplot as plt

from sklearn.metrics import confusion_matrix, accuracy_score, classification_report

# assigning the pickle files
with open('./Data_Array_Storage/male_pol_us_X_train.pkl', 'rb') as f:
    X_train = pickle.load(f)

with open('./Data_Array_Storage/male_pol_us_X_test.pkl', 'rb') as f:
    X_test = pickle.load(f)

with open('./Data_Array_Storage/male_pol_us_y_train.pkl', 'rb') as f:
    y_train = pickle.load(f)

with open('./Data_Array_Storage/male_pol_us_y_test.pkl', 'rb') as f:
    y_test = pickle.load(f)

input_shape = (X_train.shape[1], X_train.shape[2],1)

learning_rate=0.0001
nclass = 3

def get_2d_conv_model(input_shape):

    inp = Input(shape=input_shape)
    x = Conv2D(32, (4,10), padding="same")(inp)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Conv2D(32, (4,10), padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Conv2D(32, (4,10), padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Conv2D(32, (4,10), padding="same")(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    x = MaxPool2D()(x)

    x = Flatten()(x)
    x = Dense(64)(x)
    x = BatchNormalization()(x)
    x = Activation("relu")(x)
    out = Dense(nclass, activation='softmax')(x)

    model = Model(inputs=inp, outputs=out)
    opt = tf.keras.optimizers.Adam(learning_rate)

    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['acc'])
    return model

model = get_2d_conv_model(input_shape)
model.summary()

# callback list: ModelCheckpoint, reduceLROnPlat, EarlyStopping
checkpoint_path = "./models_saved/male_pol_conv2d.h5"

# Create a callback that saves the model's weights
callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=False,
                                                 save_best_only=True,
                                                 verbose=1), # 1 tells your which epoch is saving
               tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',
                                                patience=5,
                                                restore_best_weights=True),
               tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
                                                    patience=2,
                                                    factor=0.5,
                                                    min_lr=0.00001,
                                                    verbose=1)]

batch_size = 32

model_history=model.fit(X_train,
                        y_train,
                        batch_size=batch_size,
                        epochs=150,
                        validation_data=(X_test, y_test),
                        verbose=2,
                        callbacks=callbacks)

print('model training complete')

plt.plot(model_history.history['loss'])
plt.plot(model_history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

score,acc = model.evaluate(X_test, y_test, verbose = 2, batch_size = batch_size)
print("score: %.2f" % (score))
print("acc: %.2f" % (acc))

with open('./Data_Array_Storage/male_pol_us_labels.pkl', 'rb') as f:
    lb = pickle.load(f)

preds = model.predict(X_test,
                        batch_size=batch_size,
                        verbose=1)

preds=preds.argmax(axis=1)
preds

# predictions
preds = preds.astype(int).flatten()
preds = (lb.inverse_transform((preds)))
preds = pd.DataFrame({'predictedvalues': preds})

# Actual labels
actual = y_test.argmax(axis=1)
actual = actual.astype(int).flatten()
actual = (lb.inverse_transform((actual)))
actual = pd.DataFrame({'actualvalues': actual})

# Lets combined both of them into a single dataframe
finaldf = actual.join(preds)
finaldf[150:180]

def print_confusion_matrix(confusion_matrix, class_names, figsize = (10,7), fontsize=14):
    """Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap.

    Arguments
    ---------
    confusion_matrix: numpy.ndarray
        The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix.
        Similarly constructed ndarrays can also be used.
    class_names: list
        An ordered list of class names, in the order they index the given confusion matrix.
    figsize: tuple
        A 2-long tuple, the first value determining the horizontal size of the ouputted figure,
        the second determining the vertical size. Defaults to (10,7).
    fontsize: int
        Font size for axes labels. Defaults to 14.

    Returns
    -------
    matplotlib.figure.Figure
        The resulting confusion matrix figure
    """
    df_cm = pd.DataFrame(
        confusion_matrix, index=class_names, columns=class_names,
    )
    fig = plt.figure(figsize=figsize)
    try:
        heatmap = sns.heatmap(df_cm, annot=True, fmt="d")
    except ValueError:
        raise ValueError("Confusion matrix values must be integers.")

    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)
    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

    # Get the predictions file
# finaldf = pd.read_csv("Predictions.csv")
classes = finaldf.actualvalues.unique()
classes.sort()

# Confusion matrix
c = confusion_matrix(finaldf.actualvalues, finaldf.predictedvalues)
print(accuracy_score(finaldf.actualvalues, finaldf.predictedvalues))
print_confusion_matrix(c, class_names = classes)

# Classification report
classes = finaldf.actualvalues.unique()
classes.sort()
print(classification_report(finaldf.actualvalues, finaldf.predictedvalues, target_names=classes))


'''
Advanced Data Analysis in Python

    ----Drinks data----
    Downloaded from: https://github.com/fivethirtyeight/data/tree/master/alcohol-consumption
    Article Analyzing the Dataset: http://fivethirtyeight.com/datalab/dear-mona-followup-where-do-people-drink-the-most-beer-wine-and-spirits/

    ----UFO data----
    Scraped from: http://www.nuforc.org/webreports.html
    Article Analyzing the Dataset: http://josiahjdavis.com/

'''

# Import pandas package
import pandas as pd
import numpy as np
import matplotlib.pylab as plt

'''
Plotting

(Modified from excellent examples created by Kevin Markham)
'''

# Read drinks.csv into a DataFrame called 'drinks'
drinks = pd.read_csv('../data/drinks.csv')

# bar plot of number of countries in each continent
drinks.continent.value_counts().plot(kind='bar', title='Countries per Continent')
plt.xlabel('Continent')
plt.ylabel('Count')
plt.show()                                  # show plot window (if it doesn't automatically appear)
plt.savefig('countries_per_continent.png')  # save plot to file

# histogram of beer servings (shows the distribution of a numeric column)
drinks.beer_servings.hist(bins=20)
plt.xlabel('Beer Servings')
plt.ylabel('Frequency')

# density plot of beer servings (smooth version of a histogram)
drinks.beer_servings.plot(kind='density', xlim=(0,500))
plt.xlabel('Beer Servings')

# grouped histogram of beer servings (shows the distribution for each group)
drinks.beer_servings.hist(by=drinks.continent)
drinks.beer_servings.hist(by=drinks.continent, sharex=True)
drinks.beer_servings.hist(by=drinks.continent, sharex=True, sharey=True)
drinks.beer_servings.hist(by=drinks.continent, layout=(1, 5))   # change layout (new in pandas 0.15.0)

# boxplot of beer servings by continent (shows five-number summary and outliers)
drinks.boxplot(column='beer_servings', by='continent')

# scatterplot of beer servings versus wine servings
drinks.plot(kind='scatter', x='beer_servings', y='wine_servings', alpha=0.3)

# same scatterplot, except point color varies by 'spirit_servings'
# note: must use 'c=drinks.spirit_servings' prior to pandas 0.15.0
drinks.plot(kind='scatter', x='beer_servings', y='wine_servings', c=drinks.spirit_servings, colormap='Blues')

# same scatterplot, except all European countries are colored red
colors = np.where(drinks.continent=='EU', 'r', 'b')
drinks.plot(x='beer_servings', y='wine_servings', kind='scatter', c=colors)

# scatterplot matrix of all numerical columns
pd.scatter_matrix(drinks)

'''
Indexing
'''

# Read in ufo data
ufo = pd.read_csv('../data/ufo.csv')
ufo.rename(columns={'Colors Reported':'Colors', 'Shape Reported':'Shape'}, inplace=True)

# Create a new index
ufo.set_index('State', inplace=True)
ufo.head()
ufo.index
ufo.index.is_unique

ufo.set_index('City', inplace=True) # Replaces existing index
ufo.set_index('City', inplace=True, append=True) # Adds to existing index
ufo.sort_index(inplace=True)

# Slice using the index
ufo.loc['WY',:]
ufo.loc[['ND', 'WY'],:]
ufo.loc['ND':'WY',:]                # Error because of sorting
ufo.sort_index(inplace=True)
ufo.loc['ND':'WY',:]

# Reset the index
ufo.reset_index(level='City', inplace=True) # Remove a certain label from the index
ufo.reset_index(inplace=True)               # Remove all labels from the index

# Create a multi-index
ufo.set_index(['State', 'City'], inplace=True)
ufo.sort_index(inplace=True)
ufo.head()
ufo.index

# Slice using the multi-index
ufo.loc[['ND', 'WY'],:]
ufo.loc['ND':'WY',:]
ufo.loc[('ND', 'Bismarck'),:]
ufo.loc[('ND', 'Bismarck'):('ND','Casselton'),:]

'''
Analyzing across time
'''

# Reset the index
ufo.reset_index(inplace=True)

# Convert Time column to date-time format (defined in Pandas)
# transforming to time
# Formatting Time: https://docs.python.org/2/library/time.html#time.strftime
ufo.dtypes
ufo['Time'] = pd.to_datetime(ufo['Time'], format="%m/%d/%Y %H:%M")
ufo.dtypes

# Compute date range
ufo.Time.min()
ufo.Time.max()

# Slice using time
ufo[ufo.Time > pd.datetime(1995, 1, 1)]
ufo[(ufo.Time > pd.datetime(1995, 1, 1)) & (ufo.State =='TX')]

# Set the index to time
ufo.set_index('Time', inplace=True)
ufo.sort_index(inplace=True)
ufo.head()

# Access particular times/ranges
ufo.loc['1995',:]
ufo.loc['1995-01',:]
ufo.loc['1995-01-01',:]

# Access range of times/ranges
ufo.loc['1995':,:]
ufo.loc['1995':'1996',:]
ufo.loc['1995-12-01':'1996-01',:]

# Access elements of the timestamp
# Date Componenets: http://pandas.pydata.org/pandas-docs/stable/timeseries.html#time-date-components
ufo.index.year
ufo.index.month
ufo.index.weekday
ufo.index.day
ufo.index.time
ufo.index.hour

# Create a new variable with time element
ufo['Year'] = ufo.index.year
ufo['Month'] = ufo.index.month
ufo['Day'] = ufo.index.day
ufo['Weekday'] = ufo.index.weekday
ufo['Hour'] = ufo.index.hour

'''
Split-Apply-Combine
'''

# for each year, calculate the count of sightings
ufo.groupby('Year').City.count()
ufo.Hour.value_counts()             # Same as before

# for each Shape, calculate the first sighting, last sighting, and range of sightings.
ufo.groupby('Shape').Year.min()
ufo.groupby('Shape').Year.max()

# Specify the variable outside of the apply statement
ufo.groupby('Shape').Year.apply(lambda x: x.max())

# Specifiy the variable within the apply statement
ufo.groupby('Shape').apply(lambda x: x.Year.max() - x.Year.min())

# Specify a custom function to use in the apply statement
def get_max_year(df):
    try:
        return df.Year.max()
    except:
        return ''

ufo.groupby('Shape').apply(lambda x: get_max_year(x))

# split/combine can occur on multiple columns at the same time
# for each Weekday / Hour combination, determine first sighting
ufo.groupby(['Weekday','Hour']).City.count()

'''
Merging data
'''

# Read in population data

pop = pd.read_csv('../data/population.csv')
pop.head()

ufo.head()

# Merge the data together
ufo = pd.merge(ufo, pop, on='State', how = 'left')

# Specify keys if columns have different names
ufo = pd.merge(ufo, pop, left_on='State', right_on='State', how = 'left')

# Observe the new Population column
ufo.head()

# Check for values that didn't make it (length)
len(ufo[ufo.Population.isnull()])

# Check for values that didn't make it (values)
ufo[ufo.Population.isnull()]

# Change the records that didn't match up using np.where command
ufo['State'] = np.where(ufo['State'] == 'Fl', 'FL', ufo['State'])

# Alternatively, change the state using native python string functionality
ufo['State'] = ufo['State'].str.upper()

# Merge again, this time get all of the records
ufo = pd.merge(ufo, pop, on='State', how = 'left')

'''
Writing Data
'''
ufo.to_csv('ufo_new.csv')               # First column is an index
ufo.to_csv('ufo_new.csv', index=False)  # First column is no longer index

'''
Other useful features
'''

# map values to other values
ufo['Weekday'] = ufo.Weekday.map({  0:'Mon', 1:'Tue', 2:'Wed',
                                    3:'Thu', 4:'Fri', 5:'Sat',
                                    6:'Sun'})

# Pivot rows to columns
ufo.groupby(['Weekday','Hour']).City.count()
ufo.groupby(['Weekday','Hour']).City.count().unstack(0) # Make first row level a column
ufo.groupby(['Weekday','Hour']).City.count().unstack(1) # Make second row level a column

# randomly sample a DataFrame
idxs = np.random.rand(len(ufo)) < 0.66   # create a Series of booleans
train = ufo[idxs]                        # will contain about 66% of the rows
test = ufo[~idxs]                        # will contain the remaining rows

# replace all instances of a value (supports 'inplace=True' argument)
ufo.Shape.replace('DELTA', 'TRIANGLE')   # replace values in a Series
ufo.replace('PYRAMID', 'TRIANGLE')       # replace values throughout a DataFrame

'''
Fun Stuff
'''

# Plot the number of sightings over time
ufo.groupby('Year').City.count().plot(  kind='line',
                                        color='r',
                                        linewidth=2,
                                        title='UFO Sightings by year')
# -----Analysis-----
# Clearly, Aliens love the X-Files (which came out in 1993).
# Aliens are a natural extension of the target demographic so it makes sense.

# Plot the number of sightings over the day of week and time of day
ufo.groupby(['Weekday','Hour']).City.count().unstack(0).plot(   kind='line',
                                                                linewidth=2,
                                                                title='UFO Sightings by Time of Day')
# -----Analysis-----
# This may be speculative, but it appears that Aliens also enjoy Keggers.

# Plot the sightings in in July 2014
ufo[(ufo.Year == 2014) & (ufo.Month == 7)].groupby(['Day']).City.count().plot(  kind='bar',
                                                        color='b',
                                                        title='UFO Sightings in July 2014')

# Plot multiple plots on the same plot (plots neeed to be in column format)
ufo_fourth = ufo[(ufo.Year.isin([2011, 2012, 2013, 2014])) & (ufo.Month == 7)]
ufo_fourth.groupby(['Year', 'Day']).City.count().unstack(0).plot(   kind = 'bar',
                                                                    subplots=True,
                                                                    figsize=(7,9))
# -----Analysis-----
# Aliens are love the 4th of July. The White House is still standing. Therefore
# it follows that Aliens are just here for the party.


import supervisely_lib as sly
import sly_globals as g
import ui as ui
import sly_logger_hook  # to register hook
import sly_imgaugs  # to register first part of the pipeline
from sly_dataset import SlyDataset, SlyCocoDataset  # to register dataset class

def main():
    sly.logger.info("Script arguments", extra={
        "context.teamId": g.team_id,
        "context.workspaceId": g.workspace_id,
        "modal.state.slyProjectId": g.project_id,
    })

    g.my_app.compile_template(g.root_source_dir)

    data = {}
    state = {}
    data["taskId"] = g.task_id

    ui.init(data, state)  # init data for UI widgets
    g.my_app.run(data=data, state=state)

if __name__ == "__main__":
    sly.main_wrapper("main", main)


from active_users.keys import ActiveUserEntry

class PykafeActiveUserEntry(ActiveUserEntry):

    fields = ('service_id',) + ActiveUserEntry.fields

    @classmethod
    def create_from_request(cls, request):
        instance = super(PykafeActiveUserEntry, cls).create_from_request(request)
        instance.app_id = request.META.get('HTTP_SERVICE_ID', u'')
        return instance


from category_encoders import *
import pandas as pd
import numpy as np
from scipy.special import comb

import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn.utils import shuffle
from torch.autograd import Variable

class FeatureProcessor:
    def __init__(self,date_features, categorical_features,continous_features,label_column,max_cardinality=4,verbose=0):
        date_features.sort()
        categorical_features.sort()
        continous_features.sort()

        self.date_feats  = date_features
        self.cat_feats   = categorical_features
        self.conti_feats = continous_features
        self.label_column = label_column
        self.verbose      = verbose
        self.max_cardinality = max_cardinality

    def categorical_processor(self,train_X,train_y,cat_feats,label_column,test_X=None):

        """
        input : feature df and label column separtly
        output : do ohe for low cardinality features and catboost for the remaining.

        """
        if self.verbose>0:
            print('Me is here in categorical_processor')
        #ohe low cardinality
        low_cardinality_feats = [col for col in cat_feats if train_X[col].nunique() <= self.max_cardinality]

        if len(low_cardinality_feats)>0:
            print('doing ohe..............')
            if self.verbose>0:
                print(low_cardinality_feats)
            ohe_lb = OneHotEncoder(cols=low_cardinality_feats)
            train_X = ohe_lb.fit_transform(train_X)
            if test_X:
                test_X  = ohe_lb.transform(test_X)

            cat_feats = list(set(cat_feats)-set(low_cardinality_feats))
            if len(cat_feats)<1:
                if test_X:

                    return train_X,test_X
                else:
                    return train_X

        #catboost feats remaining
        print('cat boost cols')
        print(cat_feats)
        catboost_lb = CatBoostEncoder(cols=cat_feats)
        train_X = catboost_lb.fit_transform(train_X,train_y)
        if test_X:
            if self.verbose>0:
                print('both train and test datasets are passed')
            X_test = catboost_lb.transform(X_test)
            return train_X,test_X
        else:
            if self.verbose>0:
                print('only train dataset is passed')
            return train_X

    def date_parser(self,df,date_features):

        if self.verbose>0:
            print('Me is here in date_parser')
        initial_no_columns = df.shape[1]
        combination_tracker = []
        for col in date_features:
            df[col] = pd.to_datetime(df[col])

        for col in date_features:
            df['day_{}'.format(col)] = df[col].dt.day
            df['month_{}'.format(col)] = df[col].dt.month
            df['year_{}'.format(col)] = df[col].dt.month
            for second_col in set(date_features)-set([col]):
                if set([second_col,col]) in combination_tracker:
                    continue
                else:
                    combination_tracker.append(set([second_col,col]))

                df['diff_{a}_{b}'.format(a=col,b=second_col)] = (df[col] - df[second_col]).dt.days

        #sanity check
        no_date_cols = len(date_features)
        new_cols_added_by_diff = comb(no_date_cols,2)
        day_month_type_features = 3*no_date_cols
        total_new_cols = new_cols_added_by_diff + day_month_type_features

        if df.shape[1]-initial_no_columns != total_new_cols:
            exit('somthing went wrong in date parsing')
        else:
            if self.verbose>0:
                print('date parsing completed sucessfully')
        if self.verbose>0:
            print('no of new columns added for {a} date cols {b}'.format(a=len(date_features),b=total_new_cols))
        return df

    def continuous_categorical_maskup(self,train_X,cat_feats,conti_feats,test_X=None):
        """
        """
        if self.verbose>0:
            print('Me is here in cocontinuous_categorical_maskup')

        initial_columns = train_X.columns.tolist()
        for i,col in enumerate(cat_feats):
            if self.verbose>0:
                print('checking out {a}th ,ie {b} out of {c}'.format(a=i,b=col,c=len(cat_feats)))
            temp = train_X.groupby(by=col)[conti_feats].median(skipna=True).reset_index()
            temp.columns = [col+'_median_'+i if i!=col else col for i in temp.columns]
            train_X = pd.merge(train_X,temp,on=col,how='left')
            if test_X:
                test_X  = pd.merge(test_X,temp,on=col,how='left')

            temp = train_X.groupby(by=col)[conti_feats].max(skipna=True)
            temp.columns = [col+'_max_'+i if i!=col else col for i in temp.columns]
            train_X = pd.merge(train_X,temp,on=col,how='left')
            if test_X:
                test_X  = pd.merge(test_X,temp,on=col,how='left')

            temp = train_X.groupby(by=col)[conti_feats].min(skipna=True)
            temp.columns = [col+'_max_'+i if i!=col else col for i in temp.columns]
            train_X = pd.merge(train_X,temp,on=col,how='left')
            if test_X:
                test_X  = pd.merge(test_X,temp,on=col,how='left')

        #sanity check
        count_cols_added = 3*len(cat_feats)*len(conti_feats)
        new_cols_added = list(set(train_X.columns.tolist())-set(initial_columns))
        if new_cols_added !=count_cols_added:
            exit('Booyakasha! Check Dis! Column mismatch ')
        if self.verbose > 1:
            print('{a} columns added in continuous_categorical_maskup. They are {b}'.format(a=len(new_cols_added),b=new_cols_added))

        return train_X,test_X

    def arithmetic_mashup(self,train_X,train_y,conti_feats,test_X=None):
        """
        """
        for col in train_X.columns:
            med_val = train_X[col].median(skpna=True)
            train_X[col] = train_X[col].fillna(med_val)

            if test_X:
                test_X[col] = test_X[col].fillna(med_val)

    def deep_feat_generator(self,train_X,train_y,conti_feats,test_X=None):

        class Net(nn.Module):

            def __init__(self,continous_features):
                super(Net, self).__init__()
                n1 = len(continous_features)
                print(n1)
                self.fc1 = nn.Linear(n1,800)
                self.fc2 = nn.Linear(800,800)

            def forward(self, x):
                x = self.fc1(x)
                x = F.relu(x)
                x = self.fc2(x)
                x = F.dropout(x, p=0.1)
                x = F.relu(x)
                x = self.fc2(x)

                return x

        x_train,y_train,x_test = train_X,train_y,test_X

        #preprocessing for neural net

        for col in x_train.columns:
            med_val = train_X[col].median(skpna=True)
            x_train[col] = x_train[col].fillna(med_val)
            x_train[col] = x_train[col].fillna(med_val)

            train_min = x_train[col].min()
            train_max = x_train[col].max()
            x_train[col] -= train_min
            x_train[col] /= train_max

            if x_test:
                x_test[col] = x_test[col].fillna(med_val)
                x_test[col] -= train_min
                x_test[col] /= train_max

        #network training
        net = Net(continous_features=conti_feats)

        batch_size = 50
        num_epochs = 5
        learning_rate = 0.001
        batch_no = len(x_train) // batch_size

        criterion = nn.CrossEntropyLoss()
        optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

        for epoch in range(num_epochs):

            # x_train, y_train = shuffle(x_train, y_train)
            # Mini batch learning
            for i in range(batch_no):
                start = i * batch_size
                end = start + batch_size

                x_var = Variable(torch.FloatTensor(x_train[start:end].values))
                y_var = Variable(torch.LongTensor(y_train[start:end].values))
                # Forward + Backward + Optimize
                optimizer.zero_grad()
                ypred_var = net(x_var)
                loss =criterion(ypred_var, y_var)
                loss.backward()
                optimizer.step()
            # if epoch % 100 == 0:
            print('Epoch {}'.format(epoch+1))
            print(loss.detach().numpy())

        #getting features
        train_var = Variable(torch.FloatTensor(x_train.values), requires_grad=True)
        with torch.no_grad():
            result_train = net(train_var)

        out_size = 800
        train = pd.DataFrame(data=result_train.numpy(),columns=['deep_feat_{}'.format(i) for i in range(1,out_size+1)])
        # train['target'] = y_train
        print(train.head(2))

        if test_X:
            test_var = Variable(torch.FloatTensor(x_test.values), requires_grad=True)
            with torch.no_grad():
                result_test = net(test_var)

            test = pd.DataFrame(data=result_test.numpy(),columns=['deep_feat_{}'.format(i) for i in range(1,out_size+1)])
            # test['target'] = y_val
            print(test.head(2))
            exit('Not implemented')
        else:
            return train

    def fit_transform(self,train_X,train_y,test_X=None):
        # #all arithmetic operations
        # if test_X:
        #     pass
        # else:
        #     pass
        # #deep feat
        # if test_X:
        #     pass

        # else:
        #     print('only train')
        #     train_X_conti = self.deep_feat_generator(train_X=train_X[self.conti_feats],train_y=train_y,conti_feats=self.conti_feats,test_X=None)
        #     train_X = pd.concat([train_X,train_X_conti],axis=1)

        if test_X:
            train_X_cat, test_X_cat = self.categorical_processor(train_X=train_X[self.cat_feats],train_y=train_y,cat_feats=self.cat_feats,label_column=self.label_column,test_X=test_X[self.cat_feats])
            train_X = train_X.drop(self.cat_feats,axis=1)
            train_X = pd.concat([train_X,train_X_cat],axis=1)

            test_X = test_X.drop(self.cat_feats,axis=1)
            test_X = pd.concat([test_X,test_X_cat],axis=1)
        else:
            train_X_cat = self.categorical_processor(train_X=train_X[self.cat_feats],train_y=train_y,cat_feats=self.cat_feats,label_column=self.label_column,test_X=None)
            train_X = train_X.drop(self.cat_feats,axis=1)
            train_X = pd.concat([train_X,train_X_cat],axis=1)
        # print(train_X[self.cat_feats].head())
        # if test_X:
        #     train_X, test_X = self.continuous_categorical_maskup(train_X,self.cat_feats,self.conti_feats,test_X=test_X)
        # else:
        #     train_X = self.continuous_categorical_maskup(train_X,self.cat_feats,self.conti_feats)

        if test_X:
            df = pd.concat([train_X,test_X])
        else:
            train_X[self.label_column[0]]=train_y

        df = self.date_parser(df=train_X,date_features=self.date_feats)

        return df


import falcon

from pascua.db import pascuadb
from pascua.framework import ModelResource, PascuaError, pasqua_error_types, pasqua_error_codes
from pascua.resources.api.user.user_model import UserModel
from pascua.mails import send_mail

class NewUserResource(ModelResource):
    def __init__(self):
        super(NewUserResource, self).__init__(
            ('\nNew User:\n'
             '   - Insert a user into database.\n'), model=UserModel)

    def process(self, req, resp, data=None, errors=[]):
        user = UserModel(data, errors=errors)

        insert_user = user.copy()
        # check_duplied_mail = pascuadb.register.find_one({ 'email': insert_user['email'] })
        # if check_duplied_mail is not None:
        #     errors.append(PascuaError(
        #         type=pasqua_error_types.QUERY_ERROR,
        #         field='email',
        #         description='Duplicated email.',
        #         code=pasqua_error_codes['DUPLICATED_USER_EMAIL']
        #     ))
        #     resp.status = falcon.HTTP_409
        #     return

        pascuadb.register.insert_one(insert_user)
        user['_id'] = str(insert_user['_id'])

        # Send an email
        self.send_registration_mail(user)

        resp.status = falcon.HTTP_201
        return user

    @staticmethod
    def send_registration_mail(user):
        send_mail("register", user['email'], subject='Registro completado')

class GetUsersResource(ModelResource):
    def __init__(self):
        super(GetUsersResource, self).__init__(
            ('\nGet Users:\n'
             '   - Get all users from the database. Login needed.\n'), content_type=None)

    def process(self, req, resp, data=None, errors=[]):
        users = []
        docs = pascuadb.register.find()
        self.logger.debug('There are "' + str(docs.count()) + '" users registered')
        for doc in docs:
            user = UserModel(doc, errors=errors)
            user['_id'] = str(doc['_id'])
            users.append(user)

        return users


import pandas as pd
import requests
import logging
import sentry_sdk
from functools import lru_cache

from libs.datasets.combined_datasets import build_us_timeseries_with_all_fields
from libs.enums import Intervention
from libs.us_state_abbrev import US_STATE_ABBREV
from libs.us_state_abbrev import abbrev_us_state
from libs.us_state_abbrev import us_fips

from libs.datasets import FIPSPopulation
from libs.datasets import JHUDataset
from libs.datasets import CovidTrackingDataSource
from libs.datasets import CDSDataset
from libs.datasets.common_fields import CommonFields
from libs.datasets.dataset_utils import AggregationLevel
from libs.functions.calculate_projections import (
    get_state_projections_df,
    get_county_projections_df,
)
from libs.datasets.projections_schema import OUTPUT_COLUMN_REMAP_TO_RESULT_DATA
from libs.datasets.results_schema import (
    RESULT_DATA_COLUMNS_STATES,
    RESULT_DATA_COLUMNS_COUNTIES,
    CUMULATIVE_POSITIVE_TESTS,
    CUMULATIVE_NEGATIVE_TESTS,
)
from libs.constants import NULL_VALUE

_logger = logging.getLogger(__name__)

@lru_cache(None)
def _get_interventions_df():
    # TODO: read this from a dataset class
    interventions_url = "https://raw.githubusercontent.com/covid-projections/covid-data-public/master/data/misc/interventions.json"
    interventions = requests.get(interventions_url).json()
    columns = [CommonFields.STATE, "intervention"]
    return pd.DataFrame(list(interventions.items()), columns=columns)

TEST_FIELDS = [
    CommonFields.NEGATIVE_TESTS,
    CommonFields.POSITIVE_TESTS,
    CommonFields.STATE,
    CommonFields.DATE,
]

def get_testing_timeseries_by_state(state):
    testing_df = (
        build_us_timeseries_with_all_fields()
        .get_data(aggregation_level=AggregationLevel.STATE, state=state)
        .loc[:, (CommonFields.NEGATIVE_TESTS, CommonFields.POSITIVE_TESTS, CommonFields.DATE)]
    )
    testing_df.rename(
        columns={
            CommonFields.POSITIVE_TESTS: CovidTrackingDataSource.Fields.POSITIVE_TESTS,
            CommonFields.NEGATIVE_TESTS: CovidTrackingDataSource.Fields.NEGATIVE_TESTS,
        },
        inplace=True,
    )
    testing_df["date"] = testing_df.date.apply(lambda x: x.strftime("%m/%d/%y"))
    return testing_df

def get_testing_timeseries_by_fips(fips):
    """Called by generate_api"""
    testing_df = build_us_timeseries_with_all_fields().get_data(
        None, fips=fips, columns_slice=CDSDataset.COMMON_TEST_FIELDS
    )
    testing_df[CDSDataset.Fields.TESTED] = (
        testing_df[CommonFields.NEGATIVE_TESTS] + testing_df[CommonFields.POSITIVE_TESTS]
    )
    testing_df.drop(columns=[CommonFields.NEGATIVE_TESTS], inplace=True)
    all_fields = dict(**CDSDataset.INDEX_FIELD_MAP, **CDSDataset.COMMON_FIELD_MAP)
    testing_df.rename(columns=all_fields, inplace=True)
    testing_df["date"] = testing_df.date.apply(lambda x: x.strftime("%m/%d/%y"))
    testing_df.set_index([CDSDataset.Fields.FIPS, CDSDataset.Fields.DATE], inplace=True)
    return testing_df

county_replace_with_null = {"Unassigned": NULL_VALUE}

def _get_usa_by_county_df():
    # TODO: read this from a dataset class
    latest_path = JHUDataset.latest_path()
    _logger.info(f"Loading latest JHU data from {latest_path}")
    raw_df = pd.read_csv(latest_path, dtype={"FIPS": str})
    raw_df["FIPS"] = raw_df["FIPS"].astype(str).str.zfill(5)

    column_mapping = {
        "Province_State": CommonFields.STATE_FULL_NAME,
        "Country_Region": "Country/Region",
        "Last_Update": "Last Update",
        "Lat": "Latitude",
        "Long_": "Longitude",
        "Combined_Key": "Combined Key",
        "Admin2": "County",
        "FIPS": CommonFields.FIPS,
    }
    remapped_df = raw_df.rename(columns=column_mapping)
    # USA only
    us_df = remapped_df[(remapped_df["Country/Region"] == "US")]
    jhu_column_names = [
        CommonFields.STATE_FULL_NAME,
        "Country/Region",
        "Last Update",
        "Latitude",
        "Longitude",
        "Confirmed",
        "Recovered",
        "Deaths",
        "Active",
        "County",
        CommonFields.FIPS,
        "Combined Key",
        # Incident rate and people tested do not seem to be available yet
        # "Incident Rate",
        # "People Tested",
    ]
    final_df = pd.DataFrame(us_df, columns=jhu_column_names)
    final_df["Last Update"] = pd.to_datetime(final_df["Last Update"])
    final_df["Last Update"] = final_df["Last Update"].dt.strftime("%-m/%-d/%Y %H:%M")

    final_df["County"] = final_df["County"].replace(county_replace_with_null)
    final_df["Combined Key"] = final_df["Combined Key"].str.replace("Unassigned, ", "")

    final_df[CommonFields.STATE] = final_df[CommonFields.STATE_FULL_NAME].map(US_STATE_ABBREV)

    final_df = final_df.fillna(NULL_VALUE)
    # note this is a hack, 49053 is dupped in JHU data :(
    final_df = final_df.drop_duplicates(CommonFields.FIPS)
    final_df.index.name = "OBJECTID"
    # assert unique key test
    assert final_df["Combined Key"].value_counts().max() == 1
    assert final_df[CommonFields.FIPS].value_counts().max() == 1

    return final_df

def get_usa_by_county_with_projection_df(input_dir, intervention_type):
    us_only = _get_usa_by_county_df()
    fips_df = FIPSPopulation.local().data  # used to get interventions
    interventions_df = _get_interventions_df()
    projections_df = get_county_projections_df(input_dir, intervention_type, interventions_df)
    counties_decorated = (
        us_only.merge(projections_df, on=CommonFields.FIPS, how="inner")
        .merge(fips_df[[CommonFields.STATE, CommonFields.FIPS]], on=CommonFields.FIPS, how="inner",)
        .merge(interventions_df, on=CommonFields.STATE, how="inner")
    )
    counties_remapped = counties_decorated.rename(columns=OUTPUT_COLUMN_REMAP_TO_RESULT_DATA)
    counties = pd.DataFrame(counties_remapped)[RESULT_DATA_COLUMNS_COUNTIES]
    counties = counties.fillna(NULL_VALUE)
    counties.index.name = "OBJECTID"

    if counties["Combined Key"].value_counts().max() != 1:
        combined_key_max = counties["Combined Key"].value_counts().max()
        raise Exception(
            "counties['Combined Key'].value_counts().max() = "
            f"{combined_key_max}, at input_dir {input_dir}."
        )
    return counties

def get_usa_by_states_df(input_dir: str, intervention: Intervention):
    us_only = _get_usa_by_county_df()
    interventions_df = _get_interventions_df()
    projections_df = get_state_projections_df(input_dir, intervention.value, interventions_df)
    testing_df = build_us_timeseries_with_all_fields().get_data(
        aggregation_level=AggregationLevel.STATE,
        columns_slice=[
            CommonFields.STATE,
            CommonFields.POSITIVE_TESTS,
            CommonFields.NEGATIVE_TESTS,
        ],
    )
    test_max_df = (
        testing_df.groupby(CommonFields.STATE, as_index=False)[
            [CommonFields.POSITIVE_TESTS, CommonFields.NEGATIVE_TESTS]
        ]
        .max()
        .rename(
            columns={
                CommonFields.POSITIVE_TESTS: CovidTrackingDataSource.Fields.POSITIVE_TESTS,
                CommonFields.NEGATIVE_TESTS: CovidTrackingDataSource.Fields.NEGATIVE_TESTS,
            }
        )
    )
    states_group = us_only.groupby([CommonFields.STATE])
    states_agg = states_group.aggregate(
        {
            "Last Update": "max",
            "Confirmed": "sum",
            "Recovered": "sum",
            "Deaths": "sum",
            "Active": "sum",
            "Country/Region": "first",
            "Latitude": "first",
            "Longitude": "first"
            # People tested is currently null
            #'People Tested': 'sum'
        }
    )

    states_abbrev = (
        states_agg.merge(test_max_df, on=CommonFields.STATE, how="left")
        .merge(interventions_df, on=CommonFields.STATE, how="inner", suffixes=["", "_dropcol"],)
        .merge(projections_df, on=CommonFields.STATE, how="left")
    )
    STATE_COLS_REMAP = {
        CovidTrackingDataSource.Fields.POSITIVE_TESTS: CUMULATIVE_POSITIVE_TESTS,
        CovidTrackingDataSource.Fields.NEGATIVE_TESTS: CUMULATIVE_NEGATIVE_TESTS,
        **OUTPUT_COLUMN_REMAP_TO_RESULT_DATA,
    }

    states_remapped = states_abbrev.rename(columns=STATE_COLS_REMAP)
    states_remapped[CommonFields.STATE_FULL_NAME] = states_remapped[CommonFields.STATE].map(
        abbrev_us_state
    )
    states_final = pd.DataFrame(states_remapped, columns=RESULT_DATA_COLUMNS_STATES)

    # Keep nulls as nulls
    states_final = states_final.fillna(NULL_VALUE)
    states_final["Combined Key"] = states_final[CommonFields.STATE_FULL_NAME]
    states_final[CommonFields.FIPS] = states_final[CommonFields.STATE_FULL_NAME].map(us_fips)

    states_final.index.name = "OBJECTID"

    assert states_final["Combined Key"].value_counts().max() == 1
    return states_final


from keras.models import Model
from keras.layers import Dense, Input, multiply, concatenate, Dropout
import matplotlib.pyplot as plt
import numpy as np

# Load data
train_wic1 = np.array(np.load('../../processed_data/train_wic1.npy'))
train_wic2 = np.array(np.load('../../processed_data/train_wic2.npy'))
dev_wic1 = np.array(np.load('../../processed_data/dev_wic1.npy'))
dev_wic2 = np.array(np.load('../../processed_data/dev_wic2.npy'))
train_labels = np.load('../../processed_data/train_labels.npy')
dev_labels = np.load('../../processed_data/dev_labels.npy')

# Build model
layers = [1024, 512]
num_layer = len(layers)  # Number of layers in the MLP
dropouts = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
max_length = 1024
acc = []
loss = []

for dropout in dropouts:
    # Input variables
    wic1_input = Input(shape=(max_length,), dtype='float32')
    wic2_input = Input(shape=(max_length,), dtype='float32')

    wic1_latent = wic1_input
    wic2_latent = wic2_input

    # GMF part
    gmf_vector = multiply([wic1_latent, wic2_latent])
    gmf_vector = Dropout(dropout)(gmf_vector)

    # MLP part
    mlp_vector = concatenate([wic1_latent, wic2_latent])
    mlp_vector = Dropout(dropout)(mlp_vector)
    for idx in range(0, num_layer):
        layer = Dense(layers[idx], activation='relu')
        mlp_vector = layer(mlp_vector)
        mlp_vector = Dropout(dropout)(mlp_vector)

    # Concatenate MF and MLP parts
    neumf_vector = concatenate([gmf_vector, mlp_vector])

    # Output layer
    output = Dense(1, activation='sigmoid', init='lecun_uniform', name="prediction")(neumf_vector)

    model = Model(input=[wic1_input, wic2_input],
                  output=output)
    # Compile the model
    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    # print(model.summary())

    # Fit the model
    history = model.fit(
        [train_wic1, train_wic2],
        train_labels, epochs=10, verbose=2)

    los, ac = model.evaluate([dev_wic1, dev_wic2], dev_labels)

    acc.append(ac)
    loss.append(los)

np.save('acc.npy', acc)
np.save('loss.npy', loss)

xi = list(range(len(dropouts)))
plt.plot(acc)
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Dropout')
plt.xticks(xi, dropouts)
plt.show()

plt.plot(loss)
plt.title('Model loss')
plt.xticks(xi, dropouts)
plt.ylabel('Loss')
plt.xlabel('Dropout')
plt.show()


from builtins import super

from ..abstracts import Regionalizer, Mogrifyer
from ..metaheuristics.tabu import Tabu
from .. import utils

class AZP(Tabu, Regionalizer, Mogrifyer):
    def __init__(self, objective, constraints=None, remember=False):
        Regionalizer.__init__(self, objective=objective,
                                    constraints=constraints,
                                    remember=remember)
        Tabu.__init__(self, remember=remember)

    def fit(self, n_clusters, data,
            alist=None, w=None, nx=None,
            affinity_matrix=None, **kw):
        ...


#coding:utf-8
import random

# def add(x,y):
#     return x + y
#
# def sub(x,y):
#     return x - y

def exam():
    cmds = {'+':lambda x,y: x + y,'-':lambda x,y: x - y}
    nums = [random.randint(1,100) for i in range(2)]
    nums.sort(reverse=True)
    op = random.choice('+-')
    result = cmds[op](*nums)
    prompt = '%s %s %s = ' %(nums[0],op,nums[1])
    counter = 0
    while counter < 3:
        try:
            answer = int(input(prompt))
        except:     #except为空表示匹配所有异常,不推荐
            print()
            continue
        if answer == result:
            print('\nVery good!')
            break
        # else:
        print('\nWrong answer!')
        counter += 1
    else:
        print('%s%s' %(prompt,result))

def main():
    while True:
        exam()
        try:
            yn = input('Contine(y/n)?').strip()[0]
        except IndexError:
            continue
        except (KeyboardInterrupt,EOFError):
            yn = 'n'
        if yn in 'nN':
            print('\nByeBye!')
            break

if __name__ == '__main__':
    main()


'''
Write a script that finds the first vowel in a a user inputted string.

'''

string = 'qwerty'

vowels = 'aeiou'

def find_vowel(word):
    i = 0
    j = 0
    while i < len(word):
        j = 0
        while j < len(vowels):
            if word[i] == vowels[j]:
                return i
            j += 1
        i += 1

print(find_vowel(string))


from citylearn import  CityLearn, building_loader, auto_size
from energy_models import HeatPump, EnergyStorage, Building
import matplotlib.pyplot as plt
import torch
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import collections
import gym
from gym.utils import seeding

from gym import core, spaces

import os
import ptan
import time
import argparse

import model, common
from matplotlib.pyplot import figure
import numpy as np

class AgentD4PG(ptan.agent.BaseAgent):
    """
    Agent implementing noisy agent
    """
    def __init__(self, net, device="cpu", epsilon=1.0):
        self.net = net
        self.device = device
        self.epsilon = epsilon

    def __call__(self, states, agent_states):
        states_v = ptan.agent.float32_preprocessor(states).to(self.device)
        mu_v = self.net(states_v)
        actions = mu_v.data.cpu().numpy()
        actions += self.epsilon * np.random.normal(size=actions.shape)
        actions = np.clip(actions, -1, 1)
        return actions, agent_states

class DDPGActor(nn.Module):
    def __init__(self, obs_size, act_size):
        super(DDPGActor, self).__init__()

        self.net = nn.Sequential(
            nn.Linear(obs_size, 4),
            nn.ReLU(),
            nn.Linear(4, 4),
            nn.ReLU(),
            nn.Linear(4, act_size),
            nn.Tanh()
        )

    def forward(self, x):
        return self.net(x)

class DDPGCritic(nn.Module):
    def __init__(self, obs_size, act_size):
        super(DDPGCritic, self).__init__()

        self.obs_net = nn.Sequential(
            nn.Linear(obs_size, 8),
            nn.BatchNorm1d(8),
            nn.ReLU(),
        )

        self.out_net = nn.Sequential(
            nn.Linear(8 + act_size, 6),
            nn.BatchNorm1d(6),
            nn.ReLU(),
            nn.Linear(6, 1)
        )

    def forward(self, x, a):
        obs = self.obs_net(x)
        return self.out_net(torch.cat([obs, a], dim=1))

from pathlib import Path

data_folder = Path("data/")

demand_file = data_folder / "AustinResidential_TH.csv"
weather_file = data_folder / 'Austin_Airp_TX-hour.csv'

#building_ids = [4, 5, 9, 16, 21, 26, 33, 36, 49, 59]
building_ids = [4]

heat_pump, heat_tank, cooling_tank = {}, {}, {}

#Ref: Assessment of energy efficiency in electric storage water heaters (2008 Energy and Buildings)
loss_factor = 0.19/24
buildings = {}
for uid in building_ids:
    heat_pump[uid] = HeatPump(nominal_power = 9e12, eta_tech = 0.22, t_target_heating = 45, t_target_cooling = 10)
    heat_tank[uid] = EnergyStorage(capacity = 9e12, loss_coeff = loss_factor)
    cooling_tank[uid] = EnergyStorage(capacity = 9e12, loss_coeff = loss_factor)
    buildings[uid] = Building(uid, heating_storage = heat_tank[uid], cooling_storage = cooling_tank[uid], heating_device = heat_pump[uid], cooling_device = heat_pump[uid])
    buildings[uid].state_action_space(np.array([24.0, 40.0, 1.001]), np.array([1.0, 17.0, -0.001]), np.array([0.5]), np.array([-0.5]))

building_loader(demand_file, weather_file, buildings)

auto_size(buildings, t_target_heating = 45, t_target_cooling = 10)

env = {}
for uid in building_ids:
    env[uid] = CityLearn(demand_file, weather_file, buildings = {uid: buildings[uid]}, time_resolution = 1, simulation_period = (3500,6000))
    env[uid](uid)

if __name__ == "__main__":
    N_AGENTS = 2
    GAMMA = 0.99
    BATCH_SIZE = 5000
    LEARNING_RATE_ACTOR = 1e-4
    LEARNING_RATE_CRITIC = 1e-3
    REPLAY_SIZE = 5000
    REPLAY_INITIAL = 100
    TEST_ITERS = 120
    EPSILON_DECAY_LAST_FRAME = 1000
    EPSILON_START = 1.2
    EPSILON_FINAL = 0.02

    device = torch.device("cpu")

    act_net, crt_net, tgt_act_net, tgt_crt_net, agent, exp_source, buffer, act_opt, crt_opt, frame_idx = {}, {}, {}, {}, {}, {}, {}, {}, {}, {}
    rew_last_1000, rew, track_loss_critic, track_loss_actor = {}, {}, {}, {}
#     for uid in buildings:
#         env[uid].reset()
    for uid in building_ids:
        #Create as many actor and critic nets as number of agents
        #Actor: states_agent_i -> actions_agent_i
        act_net[uid] = DDPGActor(buildings[uid].observation_spaces.shape[0], buildings[uid].action_spaces.shape[0]).to(device)

        #Critic: states_all_agents + actions_all_agents -> Q-value_agent_i [1]
        crt_net[uid] = DDPGCritic(buildings[uid].observation_spaces.shape[0], buildings[uid].action_spaces.shape[0]).to(device)

        tgt_act_net[uid] = ptan.agent.TargetNet(act_net[uid])
        tgt_crt_net[uid] = ptan.agent.TargetNet(crt_net[uid])

        agent[uid] = model.AgentD4PG(act_net[uid], device=device)
        exp_source[uid] = ptan.experience.ExperienceSourceFirstLast(env[uid], agent[uid], gamma=GAMMA, steps_count=1)
        buffer[uid] = ptan.experience.ExperienceReplayBuffer(exp_source[uid], buffer_size=REPLAY_SIZE)
        act_opt[uid] = optim.Adam(act_net[uid].parameters(), lr=LEARNING_RATE_ACTOR)
        crt_opt[uid] = optim.Adam(crt_net[uid].parameters(), lr=LEARNING_RATE_CRITIC)

        frame_idx[uid] = 0

        rew_last_1000[uid], rew[uid], track_loss_critic[uid], track_loss_actor[uid] = [], [], [], []

    batch, states_v, actions_v, rewards_v, dones_mask, last_states_v, q_v, last_act_v, q_last_v, q_ref_v, critic_loss_v, cur_actions_v, actor_loss_v = {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}
    cost, price_list, buffer_reward = {},{},{}
    for uid in buildings:
        cost[uid] = []
        price_list[uid] = []
        buffer_reward[uid] = []
    while not env[building_ids[-1]]._terminal():
        if frame_idx[4]%100 == 0:
            print(frame_idx[uid])
        for uid in buildings:
#             print(env[uid].time_step)
            agent[uid].epsilon = max(EPSILON_FINAL, EPSILON_START - frame_idx[uid] / EPSILON_DECAY_LAST_FRAME)
            frame_idx[uid] += 1
            buffer[uid].populate(1)
#             print(buffer[uid].buffer[-1])
#             print(env[uid].buildings[uid].time_step)

        price = env[uid].total_electric_consumption[-1]*3e-5 + 0.045
        price_list[uid].append(price)
        for uid in buildings:
            buffer_reward[uid].append(buffer[uid].buffer[-1].reward)
            electricity_cost = buffer[uid].buffer[-1].reward*price
            cost[uid].append(-electricity_cost)
            buffer[uid].buffer[-1] = buffer[uid].buffer[-1]._replace(reward=electricity_cost)

        if len(buffer[uid]) < REPLAY_INITIAL:
            continue

        for uid in buildings:
            for k in range(6):
                batch[uid] = buffer[uid].sample(BATCH_SIZE)
                states_v[uid], actions_v[uid], rewards_v[uid], dones_mask[uid], last_states_v[uid] = common.unpack_batch_ddqn(batch[uid], device)

                # TRAIN CRITIC
                crt_opt[uid].zero_grad()
                #Obtaining Q' using critic net with parameters teta_Q'
                q_v[uid] = crt_net[uid](states_v[uid], actions_v[uid])

                #Obtaining estimated optimal actions a|teta_mu from target actor net and from s_i+1.
                last_act_v[uid] = tgt_act_net[uid].target_model(last_states_v[uid]) #<----- Actor to train Critic

                #Obtaining Q'(s_i+1, a|teta_mu) from critic net Q'
                q_last_v[uid] = tgt_crt_net[uid].target_model(last_states_v[uid], last_act_v[uid])
                q_last_v[uid][dones_mask[uid]] = 0.0

                #Q_target used to train critic net Q'
                q_ref_v[uid] = rewards_v[uid].unsqueeze(dim=-1) + q_last_v[uid] * GAMMA
                critic_loss_v[uid] = F.mse_loss(q_v[uid], q_ref_v[uid].detach())
                critic_loss_v[uid].backward()
                crt_opt[uid].step()

                # TRAIN ACTOR
                act_opt[uid].zero_grad()
                #Obtaining estimated optimal current actions a|teta_mu from actor net and from s_i
                cur_actions_v[uid] = act_net[uid](states_v[uid])

                #Actor loss = mean{ -Q_i'(s_i, a|teta_mu) }
                actor_loss_v[uid] = -crt_net[uid](states_v[uid], cur_actions_v[uid]) #<----- Critic to train Actor
                actor_loss_v[uid] = actor_loss_v[uid].mean()
                #Find gradient of the loss and backpropagate to perform the updates of teta_mu
                actor_loss_v[uid].backward()
                act_opt[uid].step()

                if frame_idx[uid] % 1 == 0:
                    tgt_act_net[uid].alpha_sync(alpha=1 - 0.1)
                    tgt_crt_net[uid].alpha_sync(alpha=1 - 0.1)

from matplotlib.pyplot import figure

#Plotting all the individual actions
print(env)

figure(figsize=(18, 6))
for uid in buildings:
    print(env[uid].buildings[uid].time_step)
    plt.plot(env[uid].action_track[uid][2400:2500])
plt.show()


import tensorflow as tf
import numpy as np
import os
import copy

MODEL_PATH = "code/ML/myModel"

TestFolder = "data/jppahl/"

MaxSamples = 600 # Num Gx Samples for Model

def getClass(file):
    c = (file[:-4].split("_")[1])
    if c == "c":
        return 0
    elif c =="o":
        return 1
    else:
        raise Exception("Unknown class")

# Read the gx data from start:stop
def readData(file):
    data = []
    with open(file, "r") as f:
        lines = f.readlines()
        start = int(lines[4].split(",")[1])
        stop = int(lines[5].split(",")[1])
        myclass = getClass(file)
        gx = lines[9].split(",")
        assert gx[0] == "gx"
        for val in gx[start:stop]:
            data.append(float(val))
    return data, myclass

# Create a list of n lists
def nList(n):
    myList = []
    for i in range(n):
        myList.append([])
    return myList

# Down sample a list of data into factor lists
def downsample(data, factor):
    downData = nList(factor)
    nValsPerSplit = int(len(data) / factor)
    for i in range(nValsPerSplit):
        for j in range(factor):
            index = i * factor + j
            downData[j].append(data[index])
    return downData

def downsampleToNPoints(data, n):
    if len(data) > n:
        factor = 2
        while True:
            dVals = downsample(data,factor)[0]
            if len(dVals) < MaxSamples:
                break
            else:
                factor += 1
        return dVals, factor
    return data, 1

def normalize(data):
    mean = np.mean(data)
    sd = np.std(data)
    data2 = (data - mean) / sd
    return list(data2)

# Fills data list up until the length matches n
def fill(data, n, val=0):
    data2 = copy.deepcopy(data)
    diff = n - len(data2)
    for i in range(diff):
        data2.append(0)
    return data2

# Normalizes data and puts it into the correct shape for the model
def preprocessData(data):
    data, _ = downsampleToNPoints(data, MaxSamples)
    data = normalize(data)
    data = fill(data, MaxSamples)
    return np.dstack([data])

def predict(model, data):
    predictRaw = model.predict(data)
    return np.argmax(predictRaw)

# This is to test out loading a saved model and having it predict against data
# defined in the files in MyFolder.
def main():
    myModel = tf.keras.models.load_model(MODEL_PATH)
    myModel.summary()
    files = os.listdir(TestFolder)
    badClasses = 0
    for file in files:
        data, myclass = readData(TestFolder + file)
        data = preprocessData(data)
        predictedClass = predict(myModel, data)
        if not myclass == predictedClass:
            badClasses+=1
            print(file)
    print("Misclassifications: ", badClasses)

if __name__ == "__main__":
    main()


# Boston Housing Study (Python)
# using data from the Boston Housing Study case
# as described in "Marketing Data Science: Modeling Techniques
# for Predictive Analytics with R and Python" (Miller 2015)

# Here we use data from the Boston Housing Study to evaluate
# regression modeling methods within a cross-validation design.

# program revised by Thomas W. Milller (2017/09/29)
# program revised by John Kiley (2018/07/14)
# program revised by John Kiley (2018/07/21)

# Scikit Learn documentation for this assignment:
# http://scikit-learn.org/stable/modules/model_evaluation.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.model_selection.KFold.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.LinearRegression.html
# http://scikit-learn.org/stable/auto_examples/linear_model/plot_ols.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.Ridge.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.Lasso.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.linear_model.ElasticNet.html
# http://scikit-learn.org/stable/modules/generated/
#   sklearn.metrics.r2_score.html

# Textbook reference materials:
# Geron, A. 2017. Hands-On Machine Learning with Scikit-Learn
# and TensorFlow. Sebastopal, Calif.: O'Reilly. Chapter 3 Training Models
# has sections covering linear regression, polynomial regression,
# and regularized linear models. Sample code from the book is
# available on GitHub at https://github.com/ageron/handson-ml

# prepare for Python version 3x features and functions
# comment out for Python 3.x execution
# from __future__ import division, print_function
# from future_builtins import ascii, filter, hex, map, oct, zip

# seed value for random number generators to obtain reproducible results
RANDOM_SEED = 1

# although we standardize X and y variables on input,
# we will fit the intercept term in the models
# Expect fitted values to be close to zero
SET_FIT_INTERCEPT = True

# import base packages into the namespace for this program
import numpy as np
import pandas as pd

# read data for the Boston Housing Study
# creating data frame restdata
boston_input = pd.read_csv('boston.csv')

# check the pandas DataFrame object boston_input
print('\nboston DataFrame (first and last five rows):')
print(boston_input.head())
print(boston_input.tail())

print('\nGeneral description of the boston_input DataFrame:')
print(boston_input.info())

# drop neighborhood from the data being considered
boston = boston_input.drop('neighborhood', 1)
print('\nGeneral description of the boston DataFrame:')
print(boston.info())

print('\nDescriptive statistics of the boston DataFrame:')
print(boston.describe())

# set up preliminary data for data for fitting the models
# the first column is the median housing value response
# the remaining columns are the explanatory variables
prelim_model_data = np.array([boston.mv,\
    boston.crim,\
    boston.zn,\
    boston.indus,\
    boston.chas,\
    boston.nox,\
    boston.rooms,\
    boston.age,\
    boston.dis,\
    boston.rad,\
    boston.tax,\
    boston.ptratio,\
    boston.lstat]).T

print(prelim_model_data)
# dimensions of the polynomial model X input and y response
# preliminary data before standardization
print('\nData dimensions:', prelim_model_data.shape)

# standard scores for the columns... along axis 0
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
print(scaler.fit(prelim_model_data))
# show standardization constants being employed
print(scaler.mean_)
print(scaler.scale_)

# the model data will be standardized form of preliminary model data
model_data = scaler.fit_transform(prelim_model_data)

# dimensions of the polynomial model X input and y response
# all in standardized units of measure
print('\nDimensions for model_data:', model_data.shape)

###############################################################################
# exploratory data analysis
###############################################################################

# import visualization packages
import matplotlib.pyplot as plt

# create a list of plots to make
cols = list(boston)
print(cols)

# loop through columns to produce histograms of the various metrics
for i in cols:
     fig, axes = plt.subplots()
     x = axes.hist(x=boston[i], alpha=0.75)
     axes.set_xlabel('Frequency')
     axes.set_ylabel(i, labelpad=20)
     axes.set_title('Histogram of ' + i,
             fontsize=13)
     fig.savefig('boston_details_'+ i + '.pdf',
                     bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b',
                     orientation='portrait', papertype=None, format=None,
                     transparent=True, pad_inches=0.25, frameon=None)
     plt.show()

     fig, axes = plt.subplots()
     x = axes.scatter(x= boston['mv'], y=boston[i], alpha=0.25)
     axes.set_xlabel('Median Home Value in 1970$s')
     axes.set_ylabel(i, labelpad=20)
     axes.set_title('Scatter MV vs ' + i,
             fontsize=13)
     fig.savefig('boston_scatter'+ i + '.pdf',
                     bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b',
                     orientation='portrait', papertype=None, format=None,
                     transparent=True, pad_inches=0.25, frameon=None)
     plt.show()

###############################################################################
# select models |||| linear models
###############################################################################

# modeling routines from Scikit Learn packages
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.metrics import mean_squared_error, r2_score
from math import sqrt  # for root mean-squared error calculation

# Set list of names and hyper parameters
names = ["Linear Regression", "Ridge Regression",
         "Lasso Regression", "Elastic Net"]
models = [LinearRegression(), Ridge(alpha=1, solver='cholesky'),
          Lasso(alpha=0.1), ElasticNet(alpha=0.1, l1_ratio=0.5)]

###############################################################################
# kfolds validation |||| ten folds
###############################################################################

# --------------------------------------------------------
# specify the k-fold cross-validation design
from sklearn.model_selection import KFold

# ten-fold cross-validation employed here
N_FOLDS = 5

# set up numpy array for storing results
cv_results = np.zeros((N_FOLDS, len(names)))
cv_results2 = np.zeros((N_FOLDS, len(names)))

kf = KFold(n_splits = N_FOLDS, shuffle=False, random_state = RANDOM_SEED)
# check the splitting process by looking at fold observation counts
index_for_fold = 0  # fold count initialized
for train_index, test_index in kf.split(model_data):
    print('\nFold index:', index_for_fold,
          '------------------------------------------')
#   note that 1:model_data.shape[1] slices for explanatory variables
#   and model_data.shape[0] is the index for the response variable
    X_train = model_data[train_index, 0:model_data.shape[1]-1]
    X_test = model_data[test_index,  0:model_data.shape[1]-1]
    y_train = model_data[train_index, model_data.shape[1]-1]
    y_test = model_data[test_index, model_data.shape[1]-1]
    print('\nShape of input data for this fold:',
          '\nData Set: (Observations, Variables)')
    print('X_train:', X_train.shape)
    print('X_test:',X_test.shape)
    print('y_train:', y_train.shape)
    print('y_test:',y_test.shape)

    index_for_method = 0  # initialize
    for name, clf in zip(names, models):
        print('\nClassifier evaluation for:', name)
        print('  Scikit Learn method:', clf)
        clf.fit(X_train, y_train)  # fit on the train set for this fold
        # evaluate on the test set for this fold
        y_test_predict = clf.predict(X_test)
        fold_method_result = sqrt(mean_squared_error(y_test,y_test_predict))
        fold_method_result2 = r2_score(y_test,y_test_predict)
        print('MSE:', fold_method_result)
        print('R2 Score:', fold_method_result2)
        cv_results[index_for_fold, index_for_method] = fold_method_result
        cv_results2[index_for_fold, index_for_method] = fold_method_result2
        index_for_method += 1

    index_for_fold += 1

# set up the R2 and MSE score outputs
cv_results_df = pd.DataFrame(cv_results)
cv_results_df2 = pd.DataFrame(cv_results2)
cv_results_df.columns = names
cv_results_df2.columns = names

cv_results_df2

print('\n----------------------------------------------')
print('Average results from ', N_FOLDS, '-fold cross-validation\n',
      '\nMethod                 MSE', sep = '')
print(cv_results_df.mean())
print('\n----------------------------------------------')
print('Average results from ', N_FOLDS, '-fold cross-validation\n',
      '\nMethod                 R2', sep = '')
print(cv_results_df2.mean())
print('\n----------------------------------------------')
###############################################################################
# Select method, train a model, and create predicted regression line
###############################################################################

# fit regression to full data set
clf = models[3]
X_train = model_data[:, 0:model_data.shape[1]-1]
y_train = model_data[:, model_data.shape[1]-1]
clf.fit(X_train, y_train)

# view the shape of the data
X_train.shape
y_train.shape

# predict
y_predict = clf.predict(X_train)

###############################################################################
# View relevant statistics for the specific model
###############################################################################
# print coefficents

print('Coefficients: \n', list(zip(cols[:][:12],clf.coef_)))
print('\n----------------------------------------------')
# print intercept
print('Intercept: \n', clf.intercept_)

###############################################################################
# View Graph
###############################################################################
from matplotlib.lines import Line2D

fig, axes = plt.subplots()
axes.scatter(y_train,y_predict, color='blue', alpha=0.25)
axes.plot(axes.get_xlim(), axes.get_ylim(), ls="--", c="red")
axes.set_xlabel('actual values')
axes.set_xlim(-4,4)
axes.set_ylabel('predicted values', labelpad=20)
axes.set_ylim(-4,4)
axes.set_title('Regression ',
             fontsize=13)

legend_elements = [Line2D([0], [0], color='R', lw=1, label='Perfect Prediction'),
                   Line2D([0], [0], marker='o', color='w', label='Actual v Predicted',
                          markerfacecolor='b', markersize=15),]

axes.legend(handles=legend_elements)
fig.savefig('regression_output'+ '.pdf',
                     bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b',
                     orientation='portrait', papertype=None, format=None,
                     transparent=True, pad_inches=0.25, frameon=None)


#Даны два числа. Найти их сумму, разность, произведение, а также частное от деления первого числа на второе.
from random import randint
import math
a = randint(1, 50)
b = randint(1, 50)

S = a + b
R = a - b
P = a * b
C = 2 * (a+b) / (a-b)

print(f'Сумма: {a} + {b} =', round(S, 2))
print(f'Разность: {a} - {b} =', round(R, 2))
print(f'Произведение: {a} * {b} =', round(P, 2))
print(f'Частное от деления: 2 * ({a}+{b}) / ({a}-{b}) =', round(C, 2))


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def maxPathSum(self, root: TreeNode) -> int:
        def max_gain(node):
            nonlocal max_sum
            if node == None:
                return 0

            left_gain = max(0, max_gain(node.left))
            right_gain = max(0, max_gain(node.right))
            max_sum = max(max_sum, left_gain + right_gain + node.val)

            return node.val + max(left_gain, right_gain)

        max_sum = -math.inf
        max_gain(root)
        return max_sum

124. Binary Tree Maximum Path Sum/solution1.py