name = 'Water Temperature'
location = 'Mokauea Island'
note = 'Dual temperature probes (surface + bottom)'
latitude = 21.307778
longitude = -157.891389
deployment_status = 'decommissioned'

conf = [
    {
        'dbtag':'idx',
        'description':'Sample Number',
        'unit':'-',
        'lb':0,
        'plot_range':7*24,
        'interval':16*8,
    },
    {
        'dbtag':'t76',
        'description':'Temperature (ch76)',
        'unit':'Deg.C',
        'lb':0,
        'ub':40,
        'plot_range':7*24,
        'interval':16*8,
    },
    {
        'dbtag':'t77',
        'description':'Temperature (ch77)',
        'unit':'Deg.C',
        'lb':0,
        'ub':40,
        'plot_range':7*24,
        'interval':16*8,
    },
    {
        'dbtag':'Vb',
        'unit':'V',
        'description':'Battery voltage',
        'lb':2.7,
        'ub':4.2,
        'interval':16*8,
    },
    {
        'dbtag':'Vs',
        'unit':'V',
        'description':'Solar panel voltage',
        'lb':0,
        'ub':7,
        'interval':16*8,
    },
]

if '__main__' == __name__:
    for c in conf:
        print('- - -')
        for k, v in c.items():
            print(k, ':', v)

    import sys
    sys.path.append('../..')
    from os.path import basename
    from storage.storage2 import create_table
    create_table(conf, basename(__file__).split('.')[0].replace('_', '-'))


import pandas as pd
import psycopg2
from sqlalchemy import create_engine

def create_sqlalchemy_connection(conn_str_file):
    sqlalchemy_conn_str = open(conn_str_file,'r').read()
    sqlalchemy_conn = create_engine(sqlalchemy_conn_str)
    return sqlalchemy_conn

def query_best_parameters(conn_str_file, model_number=None):
    sqlalchemy_conn = create_sqlalchemy_connection(conn_str_file)
    if model_number != None:
        best_parameters = pd.read_sql('SELECT * FROM validation_metrics WHERE model_number = {} ORDER BY auc DESC LIMIT 1'.format(str(model_number)),
                                  sqlalchemy_conn).to_dict(orient='records')[0]
    if model_number == None:
        best_parameters = pd.read_sql('SELECT * FROM validation_metrics ORDER BY auc DESC LIMIT 1'.format(str(model_number)),
                                  sqlalchemy_conn).to_dict(orient='records')[0]
    del best_parameters['index']
    del best_parameters['auc']
    del best_parameters['epochs']
    del best_parameters['model_number']
    del best_parameters['free']
    return best_parameters


import matplotlib.pyplot as plt
import copy
import numpy as np

import argparse

def get_eucld_error(x1, x2, y1, y2, z1, z2):
    error = []
    for i in range(len(x1)):
        p1 = np.array([x1[i], y1[i], z1[i]])
        p2 = np.array([x2[i], y2[i], z2[i]])

        error.append(float(np.sum(np.subtract(p1, p2))))

    return error

def get_data(f, ignore_count=[]):
    count = 0
    idx = []
    time = []
    x = []
    y = []
    z = []
    roll = []
    pitch = []
    yaw = []
    for line in f:
        idx.append(count)
        count += 1
        time.append(line.split(',')[0])
        x.append(float(line.split(',')[1]))
        y.append(float(line.split(',')[2]))
        z.append(float(line.split(',')[3]))
        roll.append(float(line.split(',')[4]))
        pitch.append(float(line.split(',')[5]))
        yaw.append(float(line.split(',')[6]))

    return idx, time, x, y, z, roll, pitch, yaw

def check_val(l1, l2, idx, ind, max_val, min_val):
    if l1[ind] > max_val or l2[ind] > max_val:
        l1.pop(ind)
        l2.pop(ind)
        idx.pop(ind)
    elif l1[ind] < min_val or l2[ind] < min_val:
        l1.pop(ind)
        l2.pop(ind)
        idx.pop(ind)
    else:
        ind += 1

    return l1, l2, idx, ind

def check_error_val(error, idx, ind, max_val, min_val):
    if error[ind] > max_val:
        error.pop(ind)
        idx.pop(ind)
    elif error[ind] < min_val:
        error.pop(ind)
        idx.pop(ind)
    else:
        ind += 1

    return error, idx, ind

def plot(idx_list, v1, v2, v3=None, title=" ",
         label1="1", label2="2", label3="3", dump=False, y_min=None,
         y_max=None):
    fig1, ax1 = plt.subplots()
    ax1.scatter(idx_list, v1, c='k', label=label1)
    ax1.scatter(idx_list, v2, c='g', label=label2)
    if v3 is not None:
        ax1.scatter(idx_list, v3, c='b', label=label3)
    ax1.set_title(title)
    if y_min is not None and y_max is not None:
        ax1.set_ylim((y_min, y_max))
    ax1.legend()
    if dump:
        fig1.savefig(title + ".png")

def plot_error(idx_list, error, title=" ",
               dump=False, y_min=None, y_max=None):
    fig1, ax1 = plt.subplots()
    ax1.scatter(idx_list, error, c='k')
    ax1.set_title(title)
    if y_min is not None and y_max is not None:
        ax1.set_ylim((y_min, y_max))
    if dump:
        fig1.savefig(title + ".png")

def integrate(lst):
    total = 0
    for v in lst:
        total += v

    return total

def main(fname1, fname2, display_integration=False, fname3 = None):
    print(fname1, fname2, fname3)
    f1 = open(fname1, 'r')
    f2 = open(fname2, 'r')
    if fname3 is not None:
        f3 = open(fname3, 'r')
    idx1, time1, x1, y1, z1, roll1, pitch1, yaw1 = get_data(f1)
    idx2, time2, x2, y2, z2, roll2, pitch2, yaw2 = get_data(f2)
    if fname3 is not None:
        idx3, time3, x3, y3, z3, roll3, pitch3, yaw3 = get_data(f3)
    error = get_eucld_error(x1, x2, y1, y2, z1, z2)

    idx_x = copy.deepcopy(idx1)
    idx_y = copy.deepcopy(idx1)
    idx_z = copy.deepcopy(idx1)
    idx_error = copy.deepcopy(idx1)

    # i = 0
    # while i < len(x1):
    #     x1, x2, idx_x, i = check_val(x1, x2, idx_x, i, 100.0, -100.0)
    # i = 0
    # while i < len(y1):
    #     y1, y2, idx_y, i = check_val(y1, y2, idx_y, i, 100.0, -100.0)
    # i = 0
    # while i < len(z1):
    #     z1, z2, idx_z, i = check_val(z1, z2, idx_z, i, 100.0, -100.0)
    # i = 0
    # while i < len(error):
    #     error, idx_error, i = check_error_val(error,
    #                                           idx_error, i, 100.0, -100.0)

    if display_integration:
        print('X')
        print(integrate(x1), integrate(x2))
        print('Y')
        print(integrate(y1), integrate(y2))
        print('Z')
        print(integrate(z1), integrate(z2))
        print('Error')
        print(integrate(error)/len(error))

    plot(idx_x, x1, x2, v3=x3, title="x",
         label1=fname1.split('/')[-1].replace('.txt', ''),
         label2=fname2.split('/')[-1].replace('.txt', ''),
         label3=fname3.split('/')[-1].replace('.txt', ''),
         dump=True)
    plot(idx_y, y1, y2, v3=y3, title="y",
         label1=fname1.split('/')[-1].replace('.txt', ''),
         label2=fname2.split('/')[-1].replace('.txt', ''),
         label3=fname3.split('/')[-1].replace('.txt', ''),
         dump=True)
    plot(idx_z, z1, z2, v3=z3, title="z",
         label1=fname1.split('/')[-1].replace('.txt', ''),
         label2=fname2.split('/')[-1].replace('.txt', ''),
         label3=fname3.split('/')[-1].replace('.txt', ''),
         dump=True)
    plot_error(idx_error, error)

    plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser('Display twist')
    parser.add_argument('file1')
    parser.add_argument('file2')
    parser.add_argument('--file3')
    parser.add_argument('--show_final_pose', type=bool, default=False)

    args = parser.parse_args()

    main(args.file1, args.file2, args.show_final_pose, fname3=args.file3)


import pandas as pd
import numpy as np
from lmfit import report_fit  # Minimizer, Parameters,
from lmfit.models import GaussianModel
import matplotlib.pyplot as plt
import pickle
import calendar
from scipy import signal

# UGRHI SP
# tagname = '58220000'
# -------------------------
# tagname = '3D-001'
# -------------------------
# tagname = '4C-007'
# -------------------------
# tagname = '4B-015'
# -------------------------
tagname = '5B-011'
# -------------------------

dirInput = '/home/evandro/lcbiag/ProcessoOtimizacaoModelos/' \
    'calibracaoBalagua/InstrucaoModBalagua/dados/ugrhi_sp/'

dirMR = '/home/evandro/lcbiag/ProcessoOtimizacaoModelos/' \
    'calibracaoBalagua/InstrucaoModBalagua/resultados/'

dirplot = '/home/evandro/lcbiag/ProcessoOtimizacaoModelos/' \
    'calibracaoBalagua/InstrucaoModBalagua/plots/'

# CRU
arqvminres = tagname+'_ugrhi_leastsqMinimizerResult.pkl'
input_df = pd.read_pickle(dirInput+tagname+'_ugrhi_sp.pkl')

# Xavier
# arqvminres = tagname+'_xavier_ugrhi_leastsqMinimizerResult.pkl'
# input_df = pd.read_pickle(dirInput+tagname+'_xavier_ugrhi_sp.pkl')

out_leastsq = pickle.load(open(dirMR + arqvminres, "rb"))

"""
Modelagem do balanco de agua com os parametros otimos
"""
a1 = out_leastsq.params['a1']
a2 = out_leastsq.params['a2']
a22 = out_leastsq.params['a22']
a3 = out_leastsq.params['a3']

etp = np.asarray(input_df.etp)
p2 = np.asarray(input_df.p)
q2 = np.asarray(input_df.q)

# where
posval = np.asarray(~np.isnan(etp) &
                    ~np.isnan(p2) &
                    ~np.isnan(q2)).nonzero()
posval = posval[0]

datatempo = input_df.index

nxts = len(input_df)

ts_mt = np.empty(nxts)
ts_dt = np.empty(nxts)
ts_u = np.empty(nxts)
ts_Dm = np.empty(nxts)
ts_r = np.empty(nxts)
ts_mt.fill(np.nan)
ts_dt.fill(np.nan)
ts_u.fill(np.nan)
ts_Dm.fill(np.nan)
ts_r.fill(np.nan)

m1 = np.float64(500.)
r2 = np.float64(0.)
s2 = np.float64(0.)
n2 = np.float64(0.)
d2 = np.float64(0.)
m2 = np.float64(0.)

m_func = len(posval)

for kount in range(0, m_func):
    """
    Modelo de balanço de agua

    r2 => evapotranpiracao real
    s2 => escoamento lento
    n2 => precipitacao ativa
    f2 => escoamento rapido
    d2 => vazao com filtro
    """
    posck = posval[kount]

    r2 = min(
        etp[posck]*(1.-a1**((p2[posck]+max(m1, 0.))/etp[posck])),
        (p2[posck]+max(m1, 0.))
    )
    s2 = a2*(max(m1, 0.)**a22)
    n2 = p2[posck]-etp[posck]*(1-np.exp(-p2[posck]/etp[posck]))
    f2 = a3*max(m1, 0.)*n2
    d2 = s2+f2
    m2 = m1 + p2[posck] - r2 - d2
    # print(d2, s2, f2, m2)
    ts_mt[posck] = m2
    ts_dt[posck] = d2
    ts_u[posck] = (np.sqrt(q2[posck]) - np.sqrt(d2))
    ts_Dm[posck] = m2 - m1
    ts_r[posck] = r2

    m1 = m2

'''
Visualicao dos dados
'''

x_eixo = datatempo

# histograma de u
minbin = -5
maxbin = 5
nx = 30.
step = (maxbin - minbin) / nx

binarr = np.arange(nx+1)*step + minbin
hist_u, edges_u = np.histogram(ts_u, bins=binarr, density='True')

xhist = np.arange(nx)*step + minbin + step

# ajuste gaussiano
gmodel = GaussianModel()
print('parameter names: {}'.format(gmodel.param_names))
print('independent variables: {}'.format(gmodel.independent_vars))

gparams = gmodel.make_params()
gresult = gmodel.fit(hist_u, gparams, x=xhist)
print(gresult.fit_report())

# Para agrupamentos mensais/anuais
# d_mes = dict(enumerate(calendar.month_abbr))
dados = {'ETP': etp, 'ET': ts_r, 'P': p2,
         'Qm': q2, 'Qc': ts_dt, 'S': ts_mt, 'DeltaS': ts_Dm,
         'mes': x_eixo.month}
toanual_df = pd.DataFrame(data=dados, index=x_eixo)
# totanual = toanual_df.resample('Y').sum()
totanual = toanual_df.resample('Y').agg(np.nansum)
medianual = toanual_df.resample('Y').agg(np.nanmean)
nantotanual = toanual_df.isnull().resample('Y').agg(np.sum)

'''
Plota paineis de verificacao
'''

'''
Figura series temporais
'''

# where
posval2 = np.asarray(~np.isnan(etp) &
                     ~np.isnan(p2) &
                     ~np.isnan(q2)).nonzero()
posval2 = posval2[0]

xi = x_eixo[posval2[0]]
xf = x_eixo[posval2[-1]]

# xi = np.datetime64('2014-01-01')
# xf = np.datetime64('2016-08-01')

pngfigplot = dirplot+tagname+'_pltBalagua_SerieTemporal.png'

# report_fit(out_leastsq)
# print(gresult.fit_report())

fig = plt.figure()
fig.set_figwidth(25)
fig.set_figheight(18)
# fig.suptitle(tagname)

plt.style.use('bmh')  # ggplot dark_background classic bmh seaborn-bright

plt.subplot(7, 1, 1)
plt.title(tagname, fontsize=10)
plt.plot(x_eixo, ts_mt, '.-')
plt.ylabel('S\n(mm/mês)')
plt.xlim(xi, xf)

plt.subplot(7, 1, 2)
plt.plot(x_eixo, ts_Dm, '.-')
plt.ylabel('$\Delta$S\n(mm/mês)')
plt.xlim(xi, xf)

# considera os anos com falhas nas medidas menores do que 3 meses
posval9 = np.asarray((nantotanual.DeltaS < 3.)).nonzero()
posval9 = posval9[0]

x_eixo_ano_ts = totanual.index
nxano = len(x_eixo_ano_ts)
DeltaS_ano_ts = np.empty(nxano)
DeltaS_ano_ts.fill(np.nan)

DeltaS_ano_ts[posval9] = totanual.DeltaS[posval9]

plt.subplot(7, 1, 3)
#plt.plot(x_eixo, ts_mt, '.-')
plt.plot(x_eixo_ano_ts, DeltaS_ano_ts, '.-')
plt.ylabel('$\Delta$S\n(mm/ano)')
plt.xlim(xi, xf)

ax1 = plt.subplot(7, 1, 4)
ax1.plot(x_eixo, q2, 'black', label='Q$_m$', linewidth=0.8)
ax1.plot(x_eixo, ts_dt, 'red', label='Q$_c$', linewidth=0.8)
plt.ylabel('Q$_{m,c}$\n(mm/mês)')
plt.xlim(xi, xf)

ax2 = ax1.twinx()
ax2.plot(x_eixo, p2, 'b', label='P', linewidth=0.8)
plt.ylabel('P\n(mm/mês)', axes=ax2)
plt.xlim(xi, xf)

ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

# Anual
qm_anual = np.asarray(medianual.Qm)
qc_anual = np.asarray(medianual.Qc)
p_anual = np.asarray(medianual.P)

ax1 = plt.subplot(7, 1, 5)
ax1.plot(x_eixo_ano_ts, qm_anual, 'black', label='Q$_m$', linewidth=0.8)
ax1.plot(x_eixo_ano_ts, qc_anual, 'red', label='Q$_c$', linewidth=0.8)
plt.ylabel('Q$_{m,c}$\n(mm/ano)')
plt.xlim(xi, xf)

ax2 = ax1.twinx()
ax2.plot(x_eixo_ano_ts, p_anual, 'b', label='P', linewidth=0.8)
plt.ylabel('P\n(mm/ano)', axes=ax2)
plt.xlim(xi, xf)

ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

plt.subplot(7, 1, 6)
plt.plot(x_eixo, ts_r, 'g', label='ET', linewidth=0.8)
plt.plot(x_eixo, etp, 'black', label='ETP', linewidth=0.8)
plt.ylabel('ET,ETP\n(mm/mês)')
plt.xlim(xi, xf)
plt.legend()

# Anual
ET_anual = np.asarray(medianual.ET)
ETP_anual = np.asarray(medianual.ETP)
plt.subplot(7, 1, 7)
plt.plot(x_eixo_ano_ts, ET_anual, 'g', label='ET', linewidth=0.8)
plt.plot(x_eixo_ano_ts, ETP_anual, 'black', label='ETP', linewidth=0.8)
plt.ylabel('ET,ETP\n(mm/ano)')
plt.xlim(xi, xf)
plt.legend()

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

'''
Figura - erro do modelamento
'''

pngfigplot = dirplot+tagname+'_pltBalagua_ErroModelo.png'
fig = plt.figure()
fig.set_figwidth(7)
fig.set_figheight(4)

# semivariancia
plt.subplot(2, 1, 1)

ts_u_svar = ts_u[~np.isnan(ts_u)]

semivar = signal.correlate(ts_u_svar, ts_u_svar)
# https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.correlate.html
semivarplt = semivar[np.int(semivar.size/2):]
plt.plot(semivarplt)
plt.ylabel('Autocorrelação')
plt.xlabel('Lag')

plt.subplot(2, 1, 2)
plt.scatter(xhist, hist_u)
plt.plot(xhist, gresult.best_fit, 'r-', label='Gaussiana')
plt.ylabel('(%)')
plt.xlabel('u')
plt.legend()
corrperson = np.corrcoef(hist_u, gresult.best_fit)
corrperson = corrperson[0, 1]
plt.text(2.5, np.max(hist_u)/2.,
         'C(prob, gauss) = '+'{:5.3f}'.format(corrperson), wrap=True)
print('correlacao de pearson')
print(np.corrcoef(hist_u, gresult.best_fit))

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

# '''
# Figura anual
# '''

# pngfigplot = dirplot+tagname+'_pltBalagua_ts_anual.png'

# # considera os anos com falhas nas medidas menores do que 3 meses
# posval9 = np.asarray((nantotanual.DeltaS < 3.)).nonzero()
# posval9 = posval9[0]

# x_eixo_ano_ts = totanual.index
# nxano = len(x_eixo_ano_ts)
# DeltaS_ano_ts = np.empty(nxano)
# DeltaS_ano_ts.fill(np.nan)

# DeltaS_ano_ts[posval9] = totanual.DeltaS[posval9]

# fig = plt.figure()
# fig.set_figwidth(25)
# fig.set_figheight(18)
# # fig.suptitle(tagname)

# plt.style.use('bmh')  # ggplot dark_background classic bmh seaborn-bright

# plt.subplot(6, 1, 1)
# plt.title(tagname, fontsize=10)
# #plt.plot(x_eixo, ts_mt, '.-')
# plt.plot(x_eixo_ano_ts, DeltaS_ano_ts, '.-')
# plt.ylabel('S\n(mm/ano)')
# plt.xlim(xi, xf)

# # dados = {'ETP': etp, 'ET': ts_r, 'P': p2,
# #          'Qm': q2, 'Qc': ts_dt, 'S': ts_mt, 'DeltaS': ts_Dm,
# #          'mes': x_eixo.month}
# # toanual_df = pd.DataFrame(data=dados, index=x_eixo)
# # # totanual = toanual_df.resample('Y').sum()
# # totanual = toanual_df.resample('Y').agg(np.nansum)
# # medianual = toanual_df.resample('Y').agg(np.nanmean)
# # nantotanual = toanual_df.isnull().resample('Y').agg(np.sum)

# qm_anual = np.asarray(medianual.Qm)
# qc_anual = np.asarray(medianual.Qc)
# p_anual = np.asarray(medianual.P)

# ax1 = plt.subplot(6, 1, 2)
# ax1.plot(x_eixo_ano_ts, qm_anual, 'black', label='Q$_m$', linewidth=0.8)
# ax1.plot(x_eixo_ano_ts, qc_anual, 'red', label='Q$_c$', linewidth=0.8)
# plt.ylabel('(mm/ano)')
# plt.xlim(xi, xf)

# ax2 = ax1.twinx()
# ax2.plot(x_eixo_ano_ts, p_anual, 'b', label='P', linewidth=0.8)
# plt.ylabel('P (mm/ano)', axes=ax2)
# plt.xlim(xi, xf)

# ax2.legend(loc='upper right')
# ax1.legend(loc='upper left')

# ET_anual = np.asarray(medianual.ET)
# ETP_anual = np.asarray(medianual.ETP)

# plt.subplot(6, 1, 3)
# plt.plot(x_eixo_ano_ts, ET_anual, 'g', label='ET', linewidth=0.8)
# plt.plot(x_eixo_ano_ts, ETP_anual, 'black', label='ETP', linewidth=0.8)
# plt.ylabel('(mm/ano)')
# plt.xlim(xi, xf)
# plt.legend()

# plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

'''
Figura - ciclo anual
'''

pngfigplot = dirplot+tagname+'_pltBalagua_CicloAnual.png'

canual = toanual_df.groupby('mes').agg(np.nanmean)

d_mes = dict(enumerate(calendar.month_abbr))
canual.index = canual.index.map(d_mes)  # .str.slice(stop=3)

fig = plt.figure()
fig.set_figwidth(6)
fig.set_figheight(6)

plt.style.use('bmh')  # ggplot dark_background classic bmh seaborn-bright
# plt.rcParams.update({'font.size': 10})

ax1 = plt.subplot(311)
ax1.plot(canual.Qm, label='Q$_m$')
ax1.plot(canual.Qc, label='Q$_c$')
plt.ylabel('(mm/mês)')

ax2 = ax1.twinx()
ax2.plot(canual.P, 'black', label='P')
plt.ylabel('(mm/mês)')
ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

ax1 = plt.subplot(312)
ax1.plot(canual.ET, label='ET')
ax1.plot(canual.ETP, label='ETP')
plt.ylabel('(mm/mês)')

ax2 = ax1.twinx()
ax2.plot(canual.P, 'black', label='P')
plt.ylabel('(mm/mês)')
ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

ax1 = plt.subplot(313)
ax1.plot(canual.S, label='S')
plt.ylabel('(mm/mês)')

ax2 = ax1.twinx()
ax2.plot(canual.P, 'black', label='P')
plt.ylabel('(mm/mês)')
ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

'''
Terceira figura
'''

pngfigplot = dirplot+tagname+'_pltBalagua_Dispersao.png'

fig = plt.figure()
fig.set_figwidth(10)
fig.set_figheight(9)

plt.subplot(221)

#  where
q2 = np.asarray(q2)
posval3 = np.asarray(~np.isnan(q2) |
                     ~np.isnan(ts_dt)).nonzero()
posval3 = posval3[0]
q2_plt = q2[posval3]
ts_dt_plt = ts_dt[posval3]

minv = np.nanmin(np.concatenate((q2_plt, ts_dt_plt)))
maxv = np.nanmax(np.concatenate((q2_plt, ts_dt_plt)))
xx = np.arange(minv, maxv)
yy = xx
plt.plot(xx, yy, c='green', linewidth=0.8)
plt.scatter(q2_plt, ts_dt_plt, c='black', marker='+', linewidth=0.8)
plt.ylabel('Q$_c$ (mm/mês)')
plt.xlabel('Q$_m$ (mm/mês)')

plt.subplot(222)
posval4 = np.asarray(~np.isnan(ts_r) |
                     ~np.isnan(etp)).nonzero()
posval4 = posval4[0]
ts_r_plt = ts_r[posval4]
etp_plt = etp[posval4]

minv = np.nanmin(np.concatenate((ts_r_plt, etp_plt)))
maxv = np.nanmax(np.concatenate((ts_r_plt, etp_plt)))
xx = np.arange(minv, maxv)
yy = xx
plt.plot(xx, yy, c='green', linewidth=0.8)
plt.scatter(etp_plt, ts_r_plt, c='black', marker='+', linewidth=0.8)
plt.xlabel('ETP (mm/mês)')
plt.ylabel('ET (mm/mês)')

plt.subplot(223)
posval5 = np.asarray((nantotanual.Qm < 1.) & (nantotanual.P < 1.)).nonzero()
posval5 = posval5[0]
Qm_ano_plt = totanual.Qm[posval5]
P_ano_plt = totanual.P[posval5]

posval6 = np.asarray((nantotanual.Qc < 1.) & (nantotanual.P < 1.)).nonzero()
posval6 = posval6[0]
Qc_ano_plt = totanual.Qc[posval6]
P_ano_plt = totanual.P[posval6]

posval7 = np.asarray((nantotanual.ET < 1.) & (nantotanual.P < 1.)).nonzero()
posval7 = posval7[0]
ET_ano_plt = totanual.ET[posval7]

minv = np.nanmin(np.concatenate((
    P_ano_plt, Qm_ano_plt, Qc_ano_plt, ET_ano_plt)))
maxv = np.nanmax(np.concatenate((
    P_ano_plt, Qm_ano_plt, Qc_ano_plt, ET_ano_plt)))
xx = np.arange(minv, maxv)
yy = xx

plt.plot(xx, yy, c='green', linewidth=0.8)
plt.scatter(P_ano_plt, ET_ano_plt, c='red', marker='1',
            linewidth=0.8, label='ET')
plt.scatter(P_ano_plt, Qc_ano_plt, c='blue', marker='.',
            linewidth=0.8, label='Q$_c$')
plt.scatter(P_ano_plt, Qm_ano_plt, c='black', marker='+',
            linewidth=0.8, label='Q$_m$')
plt.ylabel('(mm/ano)')
plt.xlabel('P (mm/ano)')
plt.legend()

plt.subplot(224)

posval8 = np.asarray((nantotanual.DeltaS < 1.)
                     & (nantotanual.P < 1.)).nonzero()
posval8 = posval8[0]
DeltaS_ano_plt = totanual.DeltaS[posval8]
P_ano_plt = totanual.P[posval8]

plt.scatter(P_ano_plt, DeltaS_ano_plt, c='black', marker='+', linewidth=0.8)
plt.ylabel('$\Delta$S (mm/ano)')
plt.xlabel('P (mm/ano)')

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

plt.show()

# # Salva resultados para plots

# pickle.dump((out_leastsq,
#              x_eixo,
#              ts_mt,
#              ts_dt,
#              ts_u,
#              ts_Dm,
#              ts_r,
#              q2,
#              p2,
#              etp,
#              xhist,
#              hist_u,
#              gresult),
#             # open(dirR+tagname+'_ugrhi_leastsqMinimizerResult.pkl',
#             # 'wb'))
#             open(dirR+tagname+'_xavier_ugrhi_leastsqMinimizerResult.pkl',
#                  'wb'))


# The QUBO formulation for clustering is specified in https://arxiv.org/pdf/1708.05753.pdf,
# with intensity / RGB difference as the distance metric.

from dwave_qbsolv import *
import cv2
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
from os import path
from itertools import permutations

def ising_k(vectors, labels, clusters=3, hardware_precision=7, scale_range=np.arange(0, 1, 0.1), show_plot=True):
    if len(vectors.shape) != 2:
        print("Specify a valid dataset first.")
        exit()

    N, C = vectors.shape # Number of vectors and their dimensionality (C for "channels").
    K = clusters # cluster counts
    n = hardware_precision # hardware precision of annealer coupling / bias
    LAMBDA = (2 ** (n-1) - 1) / (2 * (K - 2 if K > 2 else K)) # calculate lambda cluster constraint

    peaks = [] # record all peak accuracies
    scales = [] # record associated scales
    best_assignments = [] # record best clusterings

    for SCALE in scale_range:
        print("Scaling all loss to %f" % SCALE)

        diffs = np.zeros((N, N)) # store pairwise differences
        sums = np.zeros(N) # store sum of all pairwise differences relative to i-th pixel

        print("Calculating pairwise differences...")

        for idx, vec in tqdm(enumerate(vectors)): # for all pixels in the image...
            diffs[idx] = np.linalg.norm((vectors - vec), axis=1) * SCALE # assign pairwise difference for all pixels, scaled accordingly
            sums[idx] = np.sum(diffs[idx]) # assign sum of all pairwise differences relative to i-th pixel

        h = {} # intrinsic weight matrix
        J = {} # coupling weight matrix

        print("Assigning pairwise clustering weights...")

        # assign general pairwise clustering weights - both biases and couplings
        for k in tqdm(range(K)): # for every cluster we have...
            for i in range(N): # for every pixel in the image...
                for j in range(N): # and every pixel it might be paired with...
                    # give each binary variable an unique number
                    var_i = 10 * i + k # s_ik = x_(10i+k)
                    var_j = 10 * j + k # s_jk = x_(10j+k)
                    if i == j: # intrinsic weights (bias) indexing - for every cluster assignment (s_ik for all k in K)
                        h[var_i] = sums[i] + (2 * LAMBDA * (K - 2 if K > 2 else K)) # assign weight
                    else:
                        J[(var_i, var_j)] = 0.5 * diffs[i][j] # the weighting for a cluster assignment for two vars is half their "distance"

        print("Assigning pairwise cluster possibility weights...")

        # assign pairwise clustering variables per pixel
        for i in tqdm(range(N)):
            for a in range(K):
                for b in range(K):
                    if a != b:
                        var_a = 10 * i + a
                        var_b = 10 * i + b
                        J[(var_a, var_b)] = LAMBDA

        # TODO: Guarantee that for all s_ik for each i, only one is "true"

        print("%d couplings defined between %d variables." % (len(J.keys()), N * K))

        response = QBSolv().sample_ising(h, J, verbosity=2) # sample / solve Ising problem

        print(response)

        clusters = list(response.samples()) # store clustering assignments
        clustered = np.zeros(N)

        accuracies = [] # record accuracies of every given solution
        assignments = [] # record corresponding assignments

        possible_labels = set(permutations(range(K))) # generate all possible permutations of clusters

        for i, clustering in enumerate(clusters): # for every low-energy solution...
            curr_max = float('-inf') # current max accuracy achieved with permutation
            clustered = np.zeros(N) # initialize empty array

            # mark cluster assignments for visualization
            for key in clustering.keys():
                # extract pixel number and cluster assignments (var. number format is {pixel}{cluster})
                key_str = str(key)
                # since pixel #0 cluster assignments will break this format (only {cluster}), fix manually
                if len(key_str) == 1:
                    key_str = "0%s" % key_str
                # extract all parameters from single number!
                key_pix, key_cluster, key_status = int(key_str[:-1]), int(key_str[-1:]), clustering[key]

                for label_set in possible_labels:
                    # if spin / state is +1, fill in cluster indicated by variable
                    if key_status == 1:
                        clustered[key_pix] = label_set[key_cluster] # replace with permuted cluster label possibility

                    # Count matches
                    matches = np.sum(np.where(clustered == labels, 1, 0))
                    percent = matches / N

                    # Update maximum accuracy if this permutation is better
                    if percent > curr_max: curr_max = percent

            # Record accuracy
            accuracies.append(curr_max)

            # Record assignment
            assignments.append(clustered)

        accuracies = np.array(accuracies)

        print(accuracies)

        SNR = np.linalg.norm(np.mean(np.mean(vectors, axis=0), axis=0)) / np.linalg.norm(np.std(np.std(vectors, axis=0), axis=0))
        print("Found input SNR to be %f." % SNR)

        peak = np.max(accuracies) * 100
        best = assignments[np.argmax(accuracies)]

        print("Highest accuracy: %f percent" % peak)
        print("Associated clustering: %s" % str(best))

        peaks.append(peak)
        scales.append(SCALE)
        best_assignments.append(best)

    print(peaks)
    print(scales)

    # Find best-fit polynomial for data
    # poly = np.poly1d(np.polyfit(scales, peaks, 10))
    # param_x = np.linspace(PARAM_MIN, PARAM_MAX, 100)
    # plt.plot(param_x, poly(param_x), 'r--')

    # Plot accuracy points and line of best fit
    plt.scatter(scales, peaks)
    plt.ylim(0,100)

    for x, y in zip(scales, peaks):
        plt.annotate(
            "%d" % y,
            (x, y),
            textcoords="offset points",
            xytext=(0,10),
            ha="center"
        )

    plt.xticks(scale_range)
    plt.yticks(np.arange(0, 100, 10))

    plt.xlabel("Scale")
    plt.ylabel("Accuracy")

    if show_plot: plt.show()

    return {
        'bests': best_assignments,
        'peaks': peaks,
        'scales': scales,
        # 'plot': plt.gcf()
    }


import click

from scaraplate.rollup import rollup as _rollup

@click.group()
@click.version_option()
def main():
    pass

@main.command()
@click.argument("template_dir", type=click.Path(exists=True, file_okay=False))
@click.argument("target_project_dir", type=click.Path(file_okay=False))
@click.option(
    "--no-input",
    is_flag=True,
    help=(
        "Do not prompt for missing data. Cookiecutter will use "
        "the defaults provided by the `cookiecutter.json` "
        "in the TEMPLATE_DIR."
    ),
)
def rollup(template_dir: str, target_project_dir: str, no_input: bool) -> None:
    """Rollup (apply) the cookiecutter template.

    The template from TEMPLATE_DIR is applied on top of TARGET_PROJECT_DIR.

    TEMPLATE_DIR must be a local path to the location of a git repo with
    the cookiecutter template to apply. It must contain `scaraplate.yaml`
    file in the root. The TEMPLATE_DIR must be in a git repo, because
    some strategies require the commit hash at HEAD and the git remote URL.

    TARGET_PROJECT_DIR should point to the directory where the template
    should be applied. Might not exist -- it will be created then.
    For monorepos this should point to the subproject inside the monorepo.
    """
    _rollup(
        template_dir=template_dir,
        target_project_dir=target_project_dir,
        no_input=no_input,
    )

if __name__ == "__main__":
    main(prog_name="scaraplate")  # pylint: disable=unexpected-keyword-arg


import re

class HtmlContextParser:
    """
        Control *valid* text data in HTML document
        Options:
          set TAG_PARAMS and TAG_PARAMS_DEFAULT to parse tag parameters for *valid* text
    """

    __INVALID_DATA_TAGS: set[str] = {'script', 'style'}
    __SELF_CLOSE_TAGS: set[str] = {'meta', 'link', 'hr', 'img', 'input', 'br'}
    __INLINE_TAGS: set[str] = {'b', 'u', 'i', 'strong'}   # TODO: Mb as items...
    __TAGS_PARAMS: dict[str, set[str]] = {}  # Params list of tag to parse as *valid* text
    __TAGS_PARAMS_DEFAULT: set[str] = set()  # Params that contain *valid* text for any tag

    def __init__(self, parse_options: dict[str, set[str]] = None, default_parameters: set[str] = None):
        self.__left_data: str = ''        # Not parsed data
        # Class props to return
        self.__dom_data: list[dict[str, str]] = []  # List of DOM-tree items (item where is *valid* text)
        self.__dom_tree: list[str] = []   # DOM tree of elements with context
        self.__warnings: list[str] = []   # DOM errors or parsing warnings
        # Element props while parsing
        self.__elem_dom: list[any] = [1, ]  # Element DOM tree *Unique* (index is always last item)
        self.__elem_prefix: str = ''        # Element technical text before *valid* text
        self.__elem_warning: str = ''       # Element warning (DOM tree errors like 'tag not closed')
        # Update tags parsing options
        if default_parameters:
            self.__set_tag_default_parse_options(default_parameters)
        if parse_options:
            self.__set_tag_parse_options(parse_options)

    @property
    def data(self) -> list[dict[str, str]]:
        """ Dict of DOM-tree items. Item is a tag where *valid* text (context without close tag). """
        return self.__dom_data

    @data.setter
    def data(self, value: str):
        """ Set data to parse as html (from file) and parse it """
        self.__left_data: str = value
        # Parse file while not EOF
        while self.__next_parse():
            pass

    @property
    def tree(self) -> list[str]:
        """ Return DOM-tree """
        return self.__dom_tree

    @property
    def warnings(self) -> list[str]:
        return self.__warnings

    def __next_parse(self) -> bool:
        """ Parse next part of file to find *valid* text """
        # Check for EOF
        if self.__end_of_file():
            return False
        # Check for invisible symbols at start
        if self.__cut_non_visible_start():
            return True
        # Try to parse next data as *valid* text
        _tag_data = re.match(r'[^<]+', self.__left_data)
        if not _tag_data:  # Data not found
            self.__tag_open_or_close()
        else:
            _text: str = _tag_data.group()
            _cut_idx: int = _tag_data.end()
            self.__elem_validate_and_add(_text, _cut_idx)
        return True

    def __elem_validate_and_add(self, text: str, cut_index: int, dom_suffix: str = '') -> None:
        """ Add *valid* text as elem or add *invalid* as prefix """
        if self.__is_text_valid(text):  # URL check
            _prefix = self.__elem_prefix_return(cut_index)  # Will null prefix
            _warning = self.__elem_warning_return()  # Will null warning
            _dom_tree = self.__elem_dom_string_return()  # Save element DOM position to DOM tree
            _dom_tree = f'{_dom_tree}:+{dom_suffix}' if dom_suffix else _dom_tree  # DOM tree for tag parameter
            _text = self.__cut_non_visible_end(text)  # Add non visible symbols to next prefix
            self.__dom_data.append({'dom': _dom_tree, 'text': _text, 'prefix': _prefix, 'warning': _warning})
        else:
            self.__cut_to_elem_prefix(cut_index)

    def __elem_prefix_return(self, cut_index: int) -> str:
        """ Return prefix and refresh data """
        _prefix = f'{self.__elem_prefix}'
        self.__elem_prefix = ''
        self.__left_data = self.__left_data[cut_index:]  # refresh not parsed data
        return _prefix

    def __elem_warning_return(self) -> str:
        """ Return warning and null it """
        _warning = f'{self.__elem_warning}'
        self.__elem_warning = ''
        return _warning

    def __elem_dom_string_return(self) -> str:
        """ Get tag and DOM tree of last element  """
        item_tree = ':'.join([str(tag) for tag in self.__elem_dom])
        self.__dom_tree.append(item_tree)
        return item_tree

    def __tag_open_or_close(self) -> None:
        """ Find what kind of tag is next """  # TODO: (check tag parameters for open tag)
        if self.__left_data.startswith('</'):
            _cut_idx = self.__left_data.find('>')
            if _cut_idx == -1:
                self.__data_broken()
            else:
                _close_tag: str = self.__left_data[2:_cut_idx]
                self.__cut_to_elem_prefix(_cut_idx + 1)
                # Validate tag name
                if re.match(r'[a-z]', _close_tag, re.IGNORECASE):
                    self.__tag_dom_close(_close_tag)
                else:
                    self.__warnings.append(f'close tag not html format - {_close_tag}')
        else:
            _html_tag = re.match(r'<([a-z]+)(>|/>|\s*)', self.__left_data, re.IGNORECASE)
            if _html_tag:
                _open_tag = _html_tag.group(1)
                _after_tag = _html_tag.group(2)
                _cut_idx = _html_tag.end()
                self.__cut_to_elem_prefix(_cut_idx)
                # When open any tag - add tag and it's first index to dom tree
                self.__elem_dom += [_open_tag, 1]
                # Control tag parameters, auto-close tag, get cut index
                if _after_tag == '/>':  # Close tag if self closed
                    self.__tag_dom_close(_open_tag)
                    if _open_tag not in self.__SELF_CLOSE_TAGS:
                        self.__warnings.append(f'tag {_open_tag} not in known self-close tags but ends with /')
                elif _open_tag in self.__INVALID_DATA_TAGS:
                    # Cut data inside non *valid* tags (leave </tag_name> to close tag on next iteration)
                    self.__cut_or_eof(f'</{_open_tag}')
                elif _open_tag in self.__SELF_CLOSE_TAGS:  # No need / for *self-close* tag (like <br>)
                    self.__tag_dom_close(_open_tag)
                    if _after_tag != '>':
                        self.__tag_parse_params(_open_tag, ignore_params=True)
                        self.__warnings.append(f'self-close tag {_open_tag} have parameters')
                elif _after_tag != '>':
                    self.__tag_parse_params(_open_tag)
            else:
                # tags like <!doctype> or unknown tags <?> ignored in DOM tree
                self.__cut_or_eof('>')

    def __tag_parse_params(self, tag_name: str, ignore_params: bool = False) -> None:
        """ Parse tag parameters till end tag """
        while self.__tag_param_next_or_end(tag_name, ignore_params):
            pass

    def __tag_param_next_or_end(self, tag_name: str, ignore_params) -> bool:
        """ Parse next tag parameter or end tag """
        _param = re.match(r'([a-z-_]+)\s*=\s*', self.__left_data, re.IGNORECASE)
        if _param:
            # common parameter
            _param_name = _param.group(1)
            _cut_idx = _param.end()
            self.__cut_to_elem_prefix(_cut_idx)
            # self.__tag_param_add_or_cut(tag_name, _param_name, ignore_params)
            return self.__tag_param_add_or_cut(tag_name, _param_name, ignore_params)
        elif self.__left_data.startswith('>') or self.__left_data.startswith('/>'):
            # common end of tag
            self.__cut_or_eof('>')
            self.__cut_non_visible_start()
        else:
            # data broken or unknown template - but try to find tag end '>'
            self.__warnings.append(f'wrong html parameter in {tag_name}')
            self.__elem_warning = 'unknown tag parameters structure'
            self.__cut_or_eof('>')
        return False

    def __tag_param_add_or_cut(self, tag_name: str, param_name: str, ignore_params: bool) -> bool:
        """ Check parameter and add it as elem or add to prefix (return True if no errors) """
        # Find parameter value and it's cut index
        _next_char = self.__left_data[0]
        if _next_char == '"':  # double quote
            _value = re.match(r'"\s*([^"]*?)"', self.__left_data)
        elif _next_char == "'":  # single quote
            _value = re.match(r"'\s*([^']*?)'", self.__left_data)
        else:  # space as quote/delimiter or tag ends
            _value = re.match(r'([^\s]+?)[\s>]', self.__left_data)

        if _value:
            # check parameter to add as elem (don't change order !)
            if self.__left_data.startswith('<'):
                self.__warnings.append(f'wrong parameter value in tag {tag_name}')
                self.__cut_or_eof('>')
            elif not ignore_params and param_name in self.__TAGS_PARAMS_DEFAULT or \
                    tag_name in self.__TAGS_PARAMS.keys() and param_name in self.__TAGS_PARAMS[tag_name]:
                # cut quote and invisible symbols at start
                _cut_idx = _value.start(1)
                self.__cut_to_elem_prefix(_cut_idx)
                # add item
                _end_idx = _value.end(1) - _cut_idx
                _param_value: str = _value.group(1)
                self.__elem_validate_and_add(text=_param_value, cut_index=_end_idx, dom_suffix=param_name)
                # cut quote and invisible symbols at end (of parameter)
                if _next_char in ['\'', '"']:
                    self.__cut_to_elem_prefix(1)
            else:
                _cut_idx = _value.end() if _next_char in ['\'', '"'] else _value.end(1)
                self.__cut_to_elem_prefix(_cut_idx)
            self.__cut_non_visible_start()
            return True
        else:
            self.__warnings.append(f'tag {tag_name} parameters quote error')
            self.__cut_or_eof('>')
        return False

    def __tag_dom_close(self, tag_name: str) -> None:
        """ Handle closing tag - update tag tree """
        if tag_name in self.__elem_dom:
            if self.__elem_dom[-2] == tag_name:
                self.__elem_dom = self.__elem_dom[:-2]
            else:
                # tag not at the end of a tree (tree cut error)
                _fix_dom_tree_cut_deep = len(self.__elem_dom) - 1 - self.__elem_dom[::-1].index(tag_name)
                self.__elem_dom = self.__elem_dom[:_fix_dom_tree_cut_deep]  # Cut DOM deep
                self.__elem_warning = 'previous tag was not closed'
                self.__warnings.append(f'dom tree error - tag before {tag_name} was not closed')
            self.__elem_dom[-1] += 1  # increment index for next child
        else:
            # close tag not found in tree (tree error)
            self.__warnings.append(f'dom tree error - close tag {tag_name} have no open tag')

    def __data_broken(self):
        """ For critical errors - save left data to EOF """
        _cut_idx = len(self.__left_data)
        self.__cut_to_elem_prefix(_cut_idx)
        _prefix = self.__elem_prefix_return(_cut_idx)
        _data_tail = {'dom': 'EOF', 'text': _prefix, 'prefix': _prefix, 'warning': 'data broken'}
        self.__dom_data.append(_data_tail)  # Add left data to return data as 'EOF'
        self.__warnings.append(f'data broken - {_cut_idx} bytes left not parsed')

    def __end_of_file(self) -> bool:
        """ Check for EOF """
        if not self.__left_data:  # no more data to parse
            if self.__dom_data and self.__dom_data[-1]['dom'] != 'EOF':
                _prefix = self.__elem_prefix_return(0)
                _data_tail = {'dom': 'EOF', 'text': '', 'prefix': _prefix, 'warning': 'end of file'}
                self.__dom_data.append(_data_tail)  # Add left data to return data as 'EOF'
            return True
        return False

    def __cut_to_elem_prefix(self, cut_index: int) -> None:
        """ Update prefix and left data by cut index (of left data) - it's primary cut function """
        self.__elem_prefix += self.__left_data[:cut_index]
        self.__left_data = self.__left_data[cut_index:]  # refresh not parsed data

    def __cut_or_eof(self, value: str) -> None:
        """ Find substring index or data broken """
        _index: int = self.__left_data.find(value)
        if _index == -1:
            self.__data_broken()
        elif len(value) == 1:  # Add '>' to prefix
            self.__cut_to_elem_prefix(_index + 1)
        else:
            self.__cut_to_elem_prefix(_index)

    def __cut_non_visible_start(self) -> bool:
        """ Add invisible symbols at start to prefix """
        _start_with_non_visible = re.match(r'\s+(.*)', self.__left_data)
        if _start_with_non_visible:
            _cut_idx = _start_with_non_visible.start(1)
            self.__cut_to_elem_prefix(_cut_idx)
            return True
        return False

    def __cut_non_visible_end(self, text: str) -> str:
        """ Add invisible symbols at end to next elem prefix """
        _validate = re.fullmatch(r'(.*?)\s+$', text)
        if _validate:
            _index = _validate.end(1)
            self.__elem_prefix += text[_index:]  # Prefix for next elem (elem_prefix must be empty at this moment)
            return text[:_index]
        return text

    # ==============================
    # TODO: SETTER - params_default
    def __set_tag_default_parse_options(self, parameters: set[str]) -> None:
        """ Update what parameters to parse as *valid* text for any tag """
        if self.__TAGS_PARAMS:
            raise ValueError('Call order error: change default options before tag options were set')
        self.__TAGS_PARAMS_DEFAULT |= parameters

    # TODO: SETTER - params_tags
    def __set_tag_parse_options(self, tags_options: dict[str, set[str]]) -> None:
        """ Update in what tags parse parameters as *valid* text """
        if isinstance(tags_options, dict):
            _untestable_tags = self.__INVALID_DATA_TAGS | self.__SELF_CLOSE_TAGS
            for _tag_name in tags_options.keys():
                if _tag_name not in _untestable_tags:
                    _params = tags_options[_tag_name] - self.__TAGS_PARAMS_DEFAULT   # Remove tag params if in defaults
                    if _params:
                        self.__TAGS_PARAMS |= {_tag_name: _params}

    @staticmethod
    def __is_text_valid(text: str) -> bool:
        """ Validate text """
        try:  # Float check
            float(text)
            return False
        except ValueError:
            pass
        _text = text.strip()
        if re.match(r'[0-9 ,./<>?;\'\\":|\[\]{}!@#$%^&*()+=-_]+$', _text):  # only numbers and symbols
            return False
        if ' ' in _text:  # Text with space is *valid*
            return True
        elif _text and re.match(r'[^.:/\s]+.$', _text):  # empty, URL like or technical
            return True
        else:
            return False


from collections import defaultdict

class Solution:
    def findOrder(self, numCourses: int, prerequisites: List[List[int]]) -> List[int]:
        node_list = defaultdict(set)
        for pair in prerequisites:
            if pair[0] in node_list:
                node_list[pair[0]].add(pair[1])
            else:
                node_list[pair[0]] = set([pair[1]])

        explored = [False] * numCourses
        current_label = numCourses
        top_sort = {}

        def dfs(course: int):
            nonlocal current_label
            explored[course] = True
            if course in node_list:
                for next_course in node_list[course]:
                    if not explored[next_course]:
                        dfs(next_course)
            top_sort[course] = current_label
            current_label -= 1

        for course in range(numCourses):
            if not explored[course]:
                dfs(course)

        for pair in prerequisites:
            if top_sort[pair[0]] >= top_sort[pair[1]]:
                return []

        return sorted(top_sort, key=top_sort.get, reverse=True)


import csv
import requests
import re
import time
from bs4 import BeautifulSoup

def csv_dict_writer(path, fieldnames, data):
    """
    Writes a CSV file using DictWriter
    """
    with open(path, "w", encoding='utf8', newline='') as out_file:
        writer = csv.DictWriter(out_file, delimiter='#', fieldnames=fieldnames)
        writer.writeheader()
        for row in data:
            writer.writerow(row)

# var 1
def requester_with_soup_first(base_url, amount, delay_time):
    r = requests.get(BASE_URL)
    soap = BeautifulSoup(r.text, features="html.parser")
    ases = soap.find_all("a")
    data = ["id#url#date#header#content"]
    regexp = re.compile('[\/][n][e][w][s][\/].*')
    id = 0
    for i in ases:
        href = i.get("href")
        if regexp.match(str(href)) != None:
            id = id + 1
            url = "https://lenta.ru" + href
            if delay_time != 0:
                time.sleep(delay_time)
            r_page = requests.get(url)
            page = BeautifulSoup(r_page.text, features="html.parser")
            page_content = BeautifulSoup(str(page.find("div", {"class": "b-topic__content"})), features="html.parser")
            ppp = str(page_content.find_all("p"))
            content = BeautifulSoup(ppp, features="html.parser").text[1:-1].replace(".,", ".")
            date = page_content.time.contents[0]
            header = str(page_content.h1.contents[0]).replace("\\xa0", " ")
            data.append("{}#{}#{}#{}#{}".format(id, url, date, header, content))
            print(id)
            if id == amount: break
    return data

#var 2
def requester_with_soup_second_var(base_url, amount, delay_time):
    r = requests.get(BASE_URL)
    soap = BeautifulSoup(r.text, features="html.parser")
    div1 = str(soap.find("div", {"class": "touchcarousel-container"}))
    soap_div1 = BeautifulSoup(str(div1), features="html.parser")
    div2 = soap_div1.find_all("div", {'class':'touchcarousel-item ctg-slider__wrap top_goods'})
    #print(div2)
    data = ["id#url#category#name#description"]
    id = 0
    for i in div2:
        id = id + 1
        print(id)
        url = "https://www.e-katalog.ru" + i.a.get("href")
        if delay_time != 0:
            time.sleep(delay_time)
        r_page = requests.get(url)
        page = BeautifulSoup(r_page.text, features="html.parser")
        header = str.split(page.h1.text)
        categ = header[0]
        name = " ".join(header[1:])
        desc = page.find("div", {'class':'conf-desc-ai-title'}).text
        data.append("{}#{}#{}#{}#{}".format(id, url, categ, name, desc))
        if id == amount: break
    return data

#var 1
#BASE_URL = 'https://lenta.ru/rubrics/economics/'
BASE_URL = 'https://www.e-katalog.ru/k265.htm'
data = requester_with_soup_second_var(BASE_URL, 70, 0)

if __name__ == "__main__":
    data = [i.replace("\\xa0", " ").split("#") for i in data]
    print("OK")
    my_list = []
    print(data)
    fieldnames = data[0]
    for values in data[1:]:
        inner_dict = dict(zip(fieldnames, values))
        my_list.append(inner_dict)
    path = "dict_output.csv"
    csv_dict_writer(path, fieldnames, my_list)


from sys import stdin, setrecursionlimit
from heapq import heapify, heappop, heappush

def main():
    input = stdin.buffer.readline
    n, m = map(int, input().split())
    a = list(map(int, input().split()))
    bc = [list(map(int, input().split())) for _ in range(m)]
    bc.sort(key=lambda x: -x[1])
    heapify(a)

    for i in range(m):
        b, c = bc[i]
        for _ in range(b):
            ai = heappop(a)
            if ai < c:
                heappush(a, c)
            else:
                heappush(a, ai)
                break

    print(sum(a))

if __name__ == "__main__":
    setrecursionlimit(10000)
    main()


"""
Example of a neighbor MARL w/ electron spin
(1D representation w/ 2 neighbors: left and right)
- each agent has its own set of parameters, can be either spin up (1) or down (0)
- greedy policy: always takes Qmax action
- reward is calculated as -1 + sum(0.9 * neighbor's value)
- target value just r_t+1 without the discounted q-val of the next step
- in effect, circular grid, leftmost and rightmost electrons treated as neighbors
"""
import numpy as np
import torch
from matplotlib import pyplot as plt

def init_grid(size=(10,)):
    # pylint: disable=E1103
    """ Creates a random grid of 1/0s representing electron spin """
    grid_ = torch.randn(*size)
    grid_[grid_ > 0] = 1
    grid_[grid_ <= 0] = 0
    grid_ = grid_.byte()  # boolean
    return grid_

def get_rewards(list_binary, value_rewarded):
    """ Reward an electron agent based on its neighbors """
    total_reward = -1
    for neighbor_spin in list_binary:
        if neighbor_spin == value_rewarded:
            total_reward += 0.9
    total_reward *= 2.
    return total_reward

def gen_params(num_agents_, size):
    # pylint: disable=E1103
    """
    Create randomly inited parameters for N agents
    of dimensions <size>  (flattened) each
    """
    ret = []
    for _ in range(num_agents_):
        vec = torch.randn(size) / 10.
        vec.requires_grad = True
        ret.append(vec)
    return ret

def qfunc(state_input, theta, layers=((4, 20), (20, 2)), afn=torch.tanh):
    # pylint: disable=E1101
    """
    Q-model w/ 2 layers + elu, tanh activations
    - network dimensions adjustable as function input
    """
    l1n = layers[0]
    l1s = np.prod(l1n)  # get indices of theta for 1st layer
    theta_1 = theta[0:l1s].reshape(l1n)
    l2n = layers[1]
    l2s = np.prod(l2n)  # get indices of theta for 2nd layer
    theta_2 = theta[l1s:l2s + l1s].reshape(l2n)
    bias = torch.ones((1, theta_1.shape[1]))
    layer1_out = state_input @ theta_1 + bias
    layer1_out = torch.nn.functional.elu(layer1_out)
    layer2_out = afn(layer1_out @ theta_2)
    return layer2_out.flatten()

def get_substate(binary_value):
    # pylint: disable=E1101
    """ Encode binary to OHE """
    ohe_vector = torch.zeros(2)
    if binary_value > 0:
        ohe_vector[1] = 1
    else:
        ohe_vector[0] = 1
    return ohe_vector

def joint_state(neighbors):
    """ Get OH-encode states for neighbors <s>  as single vector """
    s1_ = get_substate(neighbors[0])
    s2_ = get_substate(neighbors[1])
    ret = (s1_.reshape(2, 1) @ s2_.reshape(1, 2)).flatten()
    return ret

def train_and_show():
    """ Run training loop and chart out grid before and after """
    # pylint: disable=E1101
    plt.figure(figsize=(8, 5))
    num_agents = (20,)
    hid_layer = 20
    params = gen_params(num_agents[0], 4 * hid_layer + hid_layer * 2)
    grid = init_grid(size=num_agents)
    shadow_grid = grid.clone()
    print(grid)
    plt.imshow(np.expand_dims(grid, 0))
    plt.show()

    # train
    epochs = 200
    learning_rate = 0.001
    losses = [[] for i in range(num_agents[0])]
    for i in range(epochs):
        for curr_agent in range(num_agents[0]):  # go thru each agent
            # get left neighbor
            left_neighbor = curr_agent - 1 if curr_agent - 1 >= 0 else num_agents[0] - 1
            # get right neighbor
            right_neighbor = curr_agent + 1 if curr_agent + 1 < num_agents[0] else 0
            state_ = grid[[left_neighbor, right_neighbor]]
            state = joint_state(state_)
            q_vals = qfunc(state.float().detach(),
                           params[curr_agent],
                           layers=[(4, hid_layer),
                                   [hid_layer, 2]])
            # policy - just take highest q-value action
            q_max = torch.argmax(q_vals, dim=0).detach().item()
            action = int(q_max)
            # only take action in cloned grid - update only when all agents have taken actions
            shadow_grid[curr_agent] = action
            reward = get_rewards(state_.detach(), action)
            # target is Q-value vector with action taken replaced by reward observed
            with torch.no_grad():
                target = q_vals.clone()
                target[action] = reward
            loss = torch.sum(torch.pow(q_vals - target, 2))
            losses[curr_agent].append(loss.detach().numpy())
            loss.backward()

            with torch.no_grad():  # manual backprop
                params[curr_agent] = params[curr_agent] - learning_rate * params[curr_agent].grad
            params[curr_agent].requires_grad = True

        # update grid state once all agent have taken actions
        with torch.no_grad():
            grid.data = shadow_grid.data

    _, axis = plt.subplots(2, 1)
    for i in range(num_agents[0]):
        axis[0].scatter(np.arange(len(losses[i])), losses[i])
    print(grid, grid.sum())
    axis[1].imshow(np.expand_dims(grid, 0))
    plt.show()

if __name__ == '__main__':
    train_and_show()


import codecs
import locale
import os.path
import grokcore.view
import docutils.core
import grokcore.component

from StringIO import StringIO
from docutils.writers.html4css1 import Writer
from docutils.writers.html4css1 import HTMLTranslator
from zope.app.renderer.rest import (
    ReStructuredTextSourceFactory, ReStructuredTextToHTMLRenderer)
from zope.app.appsetup.appsetup import getConfigContext

class RSTToHTMLRenderer(ReStructuredTextToHTMLRenderer):
    """ """

    def render(self, settings_overrides={}):
        # default settings for the renderer
        overrides = {
            'halt_level': 6,
            'input_encoding': 'unicode',
            'output_encoding': 'unicode',
            'initial_header_level': 3,
            }
        overrides.update(settings_overrides)
        writer = Writer()
        writer.translator_class = HTMLTranslator
        html = docutils.core.publish_file(
            source_path=self.context,
            writer=writer,
            destination=StringIO(), # prevents output to console!
            settings_overrides=overrides,
            )
        return html

class RSTTemplateBase(grokcore.view.components.GrokTemplate):

    def __init__(self, string=None, filename=None, _prefix=None):
        super(RSTTemplateBase, self).__init__(string=string, filename=filename, _prefix=_prefix)
        self.string=string
        self.filename=filename
        self._prefix=_prefix
        self._cooked=False

    def setFromFilename(self, filename, _prefix=None):
        filename = os.path.join(_prefix, filename)
        self._template = ReStructuredTextSourceFactory(filename)

    def render(self, view):
        view.response.setHeader('Content-Type','text/html')
        settings_overrides=self.getNamespace(view).get('settings_overrides',{})
        if not(self._cooked) or getConfigContext().hasFeature('devmode'):
            self.setFromFilename(self.filename, self._prefix)
            self._cooked=RSTToHTMLRenderer(self._template, view.request).render(settings_overrides=settings_overrides)
        return self._cooked

class RSTTemplateFactory(grokcore.component.GlobalUtility):

    grokcore.component.implements(grokcore.view.interfaces.ITemplateFileFactory)
    grokcore.component.name('rst')

    def __call__(self, filename, _prefix=None):
        return RSTTemplateBase(filename=filename, _prefix=_prefix)


import matplotlib
matplotlib.use("TkAgg")
import matplotlib as mpl
from matplotlib import dates, ticker
import matplotlib.pyplot as plt
import mpl_finance
import tkinter as tk
from tocka import Tocka
from pravac import Pravac
from trendBaseFibExtension import TrendBaseFibExtension
from coinPair import Pair
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk
from matplotlib.figure import Figure
from data import CSV
import tkinter as tk
from tkinter import ttk
LARGE_FONT= ("Verdana", 12)

def setPlot(coin):
    dayFormattter = dates.DateFormatter('%d-%b-%y %H:%M')
    # Shrink current axis by 20%
    box = ax1.get_position()
    ax1.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    # Put a legend to the right of the current axis
    ax1.legend(loc='center left', bbox_to_anchor=(1, 0.5))
    ax1.xaxis.set_major_formatter(dayFormattter)
    ax1.xaxis.set_major_locator(ticker.MaxNLocator(10))
    plt.xticks(rotation=30)
    plt.grid()
    plt.xlabel('Date')
    plt.ylabel('Price')
    plt.title("Histrical Data for {0} \nPeriod {1} to {2}, interval: 30min".format(coin.symbol, coin.start, coin.end))

def onclick(event, title):
        plt.close(title)

def pronadjiTBFE(coin):
    global fig, ax1, ohlc_data
    trends = []

    file = CSV(coin)
    file.getData(coin)
    file.saveCSV(coin)
    ohlc_data = file.fixData()

    fig, ax1 = plt.subplots()
    mpl_finance.candlestick_ohlc(ax1, ohlc_data, width=0.005, colorup='g', colordown='r', alpha=0.8)

    fib = TrendBaseFibExtension(ohlc_data)
    fib.postaviTockeABC()
    fib.trend()
    trends.append(fib)
    setPlot(coin)
"""
    for i in range(1, 1):
        fib = TrendBaseFibExtension(ohlc_data[trends[i-1].C.index:])
        fib.A.index += trends[i-1].A.index
        fib.B.index += trends[i-1].B.index
        fib.C.index += trends[i-1].C.index
        trends.append(fib)
        fib.postaviTockeABC() """

valute = []
#valute.append(Pair("ADABTC", "20 Jul, 2018", "30 Jul, 2018"))
#valute.append(Pair("WTCBTC", "20 Jul, 2018", "30 Jul, 2018"))
valute.append(Pair("ADAUSDT", '30m', "29 Jul, 2018", "30 Jul, 2018"))
#valute.append(Pair("BTCUSDT", "1 Jul, 2018", "30 Jul, 2018"))
for coin in valute:
    pronadjiTBFE(coin)

plt.show()


from random import randint
# 1.Напишите программу, которая находит  максимальный и минимальный из элементов массива с четными положительными значениями.
# Если в массиве нет элементов с четными положительными значениями, нужно вывести сообщения об этом.

# Задаем случайный список
print("Введите число элементов в массиве")
d = int(input())
A = [randint(-100, 100) for i in range(d)]
print(A)

# Ищем максимальное число
max = -1
for c in range(d):
    k = A[c]
    if (k > max) and (k % 2 == 0):
        max = k

if max == -1:
    print('Четных положительных чисел нет')
else:
    print("Максимальное положительное четное число: " , max)

# Ищем минимальное число
min = 101
for c in range(d):
    k = A[c]
    if (k < min) and (k > 0) and (k % 2 == 0):
        min = k

if min == 101:
    print('Четных положительных чисел нет')
else:
    print("Минимальное положительное четное число: " , min)


import random
import matplotlib.pyplot as plt
import numpy as np

#random.normalvariate(0,1)
f=[]
f10=[]
f100=[]
fdiff=[]
for i in range(500):
	f.append(float(1000))
for i in range(1000,1500,1):
	f.append(float(i))
for i in range(500):
	f.append(float(1500))

#add gaussian noise
for i in range(1,len(f)):
	f[i]=f[i] + random.normalvariate(0,20)

alpha10=0.05
alpha100=0.025

f10.append(f[0])
f100.append(f[0])
for i in range(1,len(f)):
	f10.append((alpha10*f[i])+((1-alpha10)*f10[i-1]))
	f100.append((alpha100*f[i])+((1-alpha100)*f100[i-1]))

for i in range(1,len(f10)):
	fdiff.append((f10[i] - f100[i])*10)

plt.plot(f)
plt.plot(f10)
plt.plot(f100)
plt.plot(fdiff)
plt.grid(True)
plt.show()


# We want to compute
#
# I = int_0^1 x^{-1/2} / (exp(x) + 1) dx
#
# here we just use uniform random numbers -- we'll have problems near
# the origin, and our answer will be unreliable

import random
import matplotlib.pyplot as plt
import numpy as np

def f(x):
    return x**(-1.0/2.0)/(np.exp(x) + 1.0)

# plot our function
eps = 1.e-16
x = np.linspace(0.0+eps, 1.0, 1000)
plt.plot(x, f(x))
plt.ylim(0,10)
plt.xlabel("x")
plt.ylabel("f(x)")
plt.savefig("integrand-importance.png", dpi=150)

# do multiple runs
for t in range(5):

    I = 0.0
    N = 10000

    for n in range(N):
        x = np.random.uniform()
        I += f(x)

    I *= 1.0/N

    print("N: {}, I = {}".format(N, I))


#A4-Q3
import re
from scapy.all import *
from scapy.utils import PcapReader
#input the file directory and name
f=input('Enter a file:')
#input the words which we want to find
users=input("Enter a string or more strings(seperated by space):")
userlist=users.split(' ')
for i in userlist:
    packets=rdpcap(f)
    for data in packets:
        if IP in data:
            if data.payload:
                payload_str=repr(data.payload)
                #remove the unprintable letter
                s=re.sub(r'\\x..',r'',payload_str)
                w=s.find(i)
                if w>0:
                    #print the source IP and destination IP
                    print(data[IP].src,'  ',data[IP].dst)
                    #print the search word and 20 characters on each side
                    print(s[w-20:w],s[w:w+len(users)],s[w+len(users):w+len(users)+20])
    print('\n')


#
#docker build -t pytest .
#docker run pytest python ./randomNumberSort.py param1 param2
#

import random
import sys

def random_sort(n):
    return sorted([random.random() for i in range(n)])

if __name__ == "__main__":
	i=0
	if(len(sys.argv)!=3):
		print("Usage:\n python3 randomNumberSort.py repetitions arrayLenght\n\nEx:\n python randomNumberSort.py 10 100")

	else:
		while(i<int(sys.argv[1])):
			random_sort(int(sys.argv[2]))
			i+=1
			print(str(i)+".random array is sorting...")


from .Scale import Scale
from .Rotate import Rotate
from .ARKImage import ARKImage
import cv2
import numpy as np
import os

class Overlay:
    def __init__(self):
        self.__actions = []
        self.__results = []
        self.__count = 1

    def addActions(self, *actions):
        # Add transformation action to self.__actions
        self.__actions += actions

    def augment(self, image, iteration=1, linear=False):
        # Repeat the augmentation process "iteration" number of times
        if not linear:
            for i in range(0, iteration):
                for action in self.__actions:
                    self.__results.append(action.process(image))
        else:
            for i in range(0, iteration):
                tmp = image
                for action in self.__actions:
                    tmp = action.process(tmp)
                self.__results.append(tmp)

    def getResults(self):
        # Return the result of the augmentation process
        return self.__results

    def saveResults(self, outputPath="output"):
        # Save the results
        if not os.path.exists(outputPath):
            os.mkdir(outputPath)
        for result in self.__results:
            outputFileName = "%.6i.png" % (self.__count)
            outputFilePath = os.path.join(outputPath, outputFileName)
            cv2.imwrite(outputFilePath, result.data)
            self.__count += 1
            print("[+] Writing Image: %s" % (outputFilePath))


from wxpy import *
import time
import schedule
import holiday as holiday, DingDing as Ding
import configparser
import check_holiday

config = configparser.ConfigParser(allow_no_value=False)
config.read("dingding.cfg", encoding='utf-8')
directory = config.get("ADB","directory")
gowork_time=config.get("time","go_time")
offwork_time=config.get("time","off_time")

#target "self_"为微信助手 或者输入用户名发送给其他人。
#target="self_"
target=config.get("weixin","target")
sex=config.get("weixin","sex")
city=config.get("weixin","city")
#print(target,sex,city)

now = int(time.time())
timeStruct = time.localtime(now)
year = timeStruct.tm_year
month = timeStruct.tm_mon
day = timeStruct.tm_mday

#工作日对应结果为 0, 休息日对应结果为 1, 节假日对应的结果为 2；
date="{0}{1:02d}{2:02d}".format(year,month,day)
choliday=check_holiday.checkholiday(date)
holiday_status=choliday['work_status']
#print(choliday)

bot = Bot(qr_path="qr.png",cache_path=True)
print('微信启动')
if target=='self_':
    bot_target=bot.file_helper
else:
    if sex=="MALE":
        bot_target = bot.friends().search(target, sex=MALE, city=city)[0]
    if sex=="FEMALE":
        bot_target = bot.friends().search(target, sex=FEMALE, city=city)[0]

#延迟是防止打卡的机子在扫微信时候触发事件
#time.sleep(10)

def job_gowork():
    if holiday_status == 0:
        print('今天是：{0}-{1:02d}-{2:02d},{3},准备打卡'.format(year, month, day, choliday['wday']))
        bot_target.send('今天是：{0}-{1:02d}-{2:02d},{3},准备打卡'.format(year, month, day, choliday['wday']))
        try:
            dingding = Ding.dingding(directory)
            dingding.goto_work()
            png = dingding.filename
            print('D:\\dingding\\' +png)
            bot_target.send('上班打卡')
            bot_target.send_image('D:\\dingding\\' + png)
        except Exception as e:
            print(e)
            bot_target.send('打卡失败:' + e)
            exit()
    else:
        print('今天是：{0}-{1:02d}-{2:02d},{3},今天不用打卡'.format(year, month, day, choliday['wday']))
        bot_target.send('今天是：{0}-{1:02d}-{2:02d},{3},今天不用打卡'.format(year, month, day, choliday['wday']))

def job_offwork():
    if holiday_status == 0:
        print('今天是：{0}-{1:02d}-{2:02d},{3},准备打卡'.format(year, month, day, choliday['wday']))
        bot_target.send('今天是：{0}-{1:02d}-{2:02d},{3},准备打卡'.format(year, month, day, choliday['wday']))
        try:
            dingding = Ding.dingding(directory)
            dingding.after_work()
            png = dingding.filename

            print('D:\\dingding\\' +png)
            bot_target.send('下班打卡')
            bot_target.send_image('D:\\dingding\\' + png)
        except Exception as e:
            print(e)
            bot_target.send('打卡失败:' + e)
            exit()
    else:
        print('今天是：{0}-{1:02d}-{2:02d},{3},今天不用打卡'.format(year, month, day, choliday['wday']))
        bot_target.send('今天是：{0}-{1:02d}-{2:02d},{3},今天不用打卡'.format(year, month, day, choliday['wday']))

schedule.every().day.at(gowork_time).do(job_gowork)
schedule.every().day.at(offwork_time).do(job_offwork)

while True:
    schedule.run_pending()

embed()


# Flask app which accepts a github username and public access token,
# creates a new repository on the user's github page, and pushes its own
# code from the server to the github repository

from flask import Flask, request
import githubfuncs
import github

app = Flask(__name__, static_folder='', static_url_path='')
# app.debug = True

@app.route('/')
def display_form():
    # display form for input of the github username and token
    return app.send_static_file('index.html')

@app.route('/pushtogithub', methods=['POST'])
def push_to_github():
    # get the github username and token, create a new repository on github,
    # and push the files from the server to the github repository
    github_username = request.form['username']
    github_token = request.form['token']
    # test_message = 'Your username is ' + github_username + '.  Your token is ' + github_token + '.'
    try:
         githubfuncs.create_remote_repo(github_token)
    except github.GithubException as error:
        if error.__class__.__name__ == 'BadCredentialsException':
            return (
                'Authentication failed.  Please check your Github personal access '
                'token.'
                )
        elif error.data['message'] == 'Repository creation failed.':
            return(
                'The auto-repo-creation repository already exists on your Github '
                'page.'
                )
        else:
            # err_type = error.__class__.__name__
            return 'An error occurred.  Please try again.'
    try:
        githubfuncs.push_code(github_username, github_token)
    except Exception as error:
        # delete the auto-repo-creation repo that was created by this program
        # in the previous step to allow the user to re-run the program without getting the
        # 'repo already exists on your page' error
        g = github.Github("<token>")
        repo = g.get_repo(full_name="auto-repo-creation")
        repo.delete()
        return (
            'An error occurred while pushing the code with git.  Please try '
            'again.'
            )
    exit_message = 'The code has been pushed to '+ github_username + '! Please check your github page.'
    return exit_message


#!/usr/bin/python3
import sys
from socket import *
import threading

serverName = ''
serverPort = 10000
clientsocket = socket(AF_INET, SOCK_STREAM)
print("socket created")
clientsocket.connect((serverName, serverPort))
print("Connection achieved")
message = clientsocket.recv(1024)
message = message.decode()
print(message)
clientsocket.send('hello'.encode())


'''
for higher answer precision
concat 2 neighbouring tokens together
2d coordinates sort
up to down
left to right
'''

import json
from tqdm import tqdm
from matplotlib import pyplot as plt

def concat(test, ratio):
    score, number = 0, 0
    for key in tqdm(test.keys()):
        if key != "0":
            image = test[key]  # change it and write back
            ocr_tokens = image["ocr_tokens"]  # change it and write back
            ocr_info = image["ocr_info"]  # change it and write back
            answers = image["answers"]
            range = len(ocr_info)  # the last bbox_dict has no next

            if range <= 50:  # requested by gcy
                new_ocr_info = ocr_info
                new_tokens = ocr_tokens
            else:
                new_ocr_info = ocr_info[:50]
                new_tokens = ocr_tokens[:50]
            for index, bbox_dict in enumerate(ocr_info):
                word = bbox_dict["word"]
                bounding_box = bbox_dict["bounding_box"]
                top_left_x = bounding_box["top_left_x"]
                top_left_y = bounding_box["top_left_y"]
                width = bounding_box["width"]
                height = bounding_box["height"]
                bottom_right_x = top_left_x + width
                bottom_right_y = top_left_y + height
                center_x = bounding_box["exact_center_x"]
                center_y = bounding_box["exact_center_y"]

                if index < range-1:
                    # hyperparameter
                    # first try,too strict, not successful
                    # threshold = 0.5*height # ratio of height that determine overlap limit
                    # second try
                    threshold = (ratio*0.001 )*height
                    next_bbox_dict = ocr_info[index+1]
                    next_word = next_bbox_dict["word"]
                    next_bounding_box = next_bbox_dict["bounding_box"]
                    next_top_left_x = next_bounding_box["top_left_x"]
                    next_top_left_y = next_bounding_box["top_left_y"]
                    next_width = next_bounding_box["width"]
                    next_height = next_bounding_box["height"]
                    next_bottom_right_x = next_top_left_x + next_width
                    next_bottom_right_y = next_top_left_y + next_height
                    next_center_x = next_bounding_box["exact_center_x"]
                    next_center_y = next_bounding_box["exact_center_y"]

                    if next_bottom_right_x < top_left_x:  # change to a new line
                        continue
                    elif abs(next_center_y-center_y) > threshold:  # too far away in y axis
                        continue
                    elif next_top_left_x - bottom_right_x > threshold:  # too far away in x axis
                        continue
                    else:
                        new_bbox_dict = {}
                        new_word = word + " " + next_word
                        new_bbox_dict["word"] = new_word
                        new_bounding_box = {}
                        new_bounding_box["top_left_x"] = top_left_x
                        new_bounding_box["top_left_y"] = top_left_y
                        new_bounding_box["width"] = next_bottom_right_x - top_left_x
                        new_bounding_box["height"] = next_bottom_right_y - top_left_y
                        new_bounding_box["rotation"] = 0
                        new_bounding_box["yaw"] = 0
                        new_bounding_box["roll"] = 0
                        new_bounding_box["pitch"] = 0
                        new_bounding_box["exact_center_x"] = top_left_x + \
                            new_bounding_box["width"]/2
                        new_bounding_box["exact_center_y"] = top_left_y + \
                            new_bounding_box["height"]/2
                        new_bbox_dict["bounding_box"] = new_bounding_box

                        new_ocr_info.append(new_bbox_dict)
                        new_tokens.append(new_word)
                        if len(new_tokens)<=100:
                            cnt = 0
                            for answer in answers:
                                if(new_word==answer):cnt+=1
                            cnt = min(3,cnt)
                            cnt/=3
                            score+=cnt # score for each image
                            number+=1

            range = len(new_ocr_info)

            if range<=100:# requested by gcy
                pass
            else:
                new_ocr_info = new_ocr_info[:100]
                new_tokens = new_tokens[:100]

            # image["ocr_info"] = new_ocr_info
            # image["ocr_tokens"] = new_tokens
            # test[key] = image
    return score, number

if __name__=="__main__":
    # dir = './transformer/transformer_'
    # dir = './kid_gt/final_noempty_textvqa_'
    dir = "/media/trd/Elements/labeledTextVQA/kid_gt/gt_denormalize_"
    ratio_list = []
    scores_dist = []
    #for ratio in range(1200, 2000):
    for ratio in [10000000000]:
        scores = 0
        numbers = 0
        #for flag in ['train', 'val', 'test']:
        for flag in ['train', 'val']:# test has no answer
            with open(dir+flag+'_sort.json', 'r') as file:
                test = json.load(file)
                score, number = concat(test, ratio)
                scores += score
                numbers += number
        print('------------------------------------------')
        print(ratio*0.001)
        print(scores/numbers)
        ratio_list.append(ratio*0.001)
        scores_dist.append(scores/numbers)
    print(ratio_list)
    print(scores_dist)
    plt.plot(ratio_list, scores_dist)
    plt.savefig('plog1.png')
    plt.show()

        # with open(dir+flag+'_concat.json', 'w') as file:
        #     json.dump(test, file, indent=2)


# Copyright 2020 Red Hat Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""OpenStack transformer module."""
import logging

from mrack.errors import ProvisioningConfigError
from mrack.transformers.transformer import Transformer

logger = logging.getLogger(__name__)

CONFIG_KEY = "openstack"

class OpenStackTransformer(Transformer):
    """OpenStack transformer."""

    _config_key = CONFIG_KEY
    _required_config_attrs = ["flavors", "networks", "images", "keypair"]  # List[str]

    async def init_provider(self):
        """Initialize associate provider and transformer display name."""
        self.dsp_name = "OpenStack"
        await self._provider.init(
            image_names=self.config["images"].values(),
            networks=self.config["networks"],
        )

    def _get_network_type(self, host):
        """Get network type from host object definition.

        If host object doesn't have it defined, then get it from metadata or
        provisioning config.
        """
        network_type = host.get("network")
        default_network = self.config.get("default_network")
        if network_type is None:
            network_type = self._metadata.get("network", default_network)
        if not network_type:
            raise ProvisioningConfigError(
                "No network type specified and project doesn't have default "
                "network type (property 'default_network') specified in "
                "provisioning config."
            )
        return network_type

    def create_host_requirement(self, host):
        """Create single input for OpenStack provisioner."""
        required_image = host.get("image") or self._get_image(host["os"])
        req = {
            "name": host["name"],
            "flavor": self._get_flavor(host),
            "image": required_image,
            "key_name": self.config["keypair"],
            "network": self._get_network_type(host),
        }

        # do not add this requirement at all if it is not required
        config_drive_req = self.config.get("enable_config_drive")
        if config_drive_req:
            req.update({"config_drive": config_drive_req})

        return req


# -*- coding: utf-8 -*-
"""
Resources below '/<api_base>/task'
"""
import logging
import json

from flask import g, request, url_for
from flask_restful import Resource
from . import with_user_or_node, with_user, only_for
from ._schema import TaskSchema, TaskIncludedSchema, TaskResultSchema
from http import HTTPStatus
from flasgger import swag_from
from pathlib import Path

from vantage6.server import db
from vantage6.server import socketio
from vantage6.server.util import (
    bytes_to_base64s,
    base64s_to_bytes,
    log_full_request,
)
from vantage6.server.globals import STRING_ENCODING

module_name = __name__.split('.')[-1]
log = logging.getLogger(module_name)

def setup(api, API_BASE):
    path = "/".join([API_BASE, module_name])
    log.info('Setting up "{}" and subdirectories'.format(path))

    api.add_resource(
        Task,
        path,
        endpoint='task_without_id',
        methods=('GET', 'POST')
    )
    api.add_resource(
        Task,
        path + '/<int:id>',
        endpoint='task_with_id',
        methods=('GET', 'DELETE')
    )
    api.add_resource(
        TaskResult,
        path + '/<int:id>/result',
        endpoint='task_result',
        methods=('GET',)
    )

# ------------------------------------------------------------------------------
# Resources / API's
# ------------------------------------------------------------------------------
class Task(Resource):
    """Resource for /api/task"""

    task_schema = TaskSchema()
    task_result_schema = TaskIncludedSchema()

    @only_for(["user", "node", "container"])
    @swag_from(str(Path(r"swagger/get_task_with_id.yaml")), endpoint='task_with_id')
    @swag_from(str(Path(r"swagger/get_task_without_id.yaml")), endpoint='task_without_id')
    def get(self, id=None):
        task = db.Task.get(id)
        if not task:
            return {"msg": "task id={} is not found"}, HTTPStatus.NOT_FOUND

        s = self.task_result_schema if request.args.get('include') == 'results' else self.task_schema
        return s.dump(task, many=not id).data, HTTPStatus.OK

    @only_for(["user", "container"])
    @swag_from(str(Path(r"swagger/post_task_without_id.yaml")), endpoint='task_without_id')
    def post(self):
        """Create a new Task."""
        # log_full_request(request, log)

        if not request.is_json:
            return {"msg": "No JSON body found..."}, HTTPStatus.BAD_REQUEST

        data = request.get_json()

        collaboration_id = data.get('collaboration_id')
        collaboration = db.Collaboration.get(collaboration_id)

        if not collaboration:
            return {"msg": f"collaboration id={collaboration_id} not found"},\
            HTTPStatus.NOT_FOUND

        organizations_json_list = data.get('organizations')
        org_ids = [org.get("id") for org in organizations_json_list]
        db_ids = collaboration.get_organization_ids()

        # TODO what happens if user's organization does not have a node that
        # is part of a the collaboration
        # if the 'master'-flag is set to true the (master) task is executed on
        # a node in the collaboration from the organization to which the user
        # belongs. If also organization_ids are supplied, then these are ignored.
        if data.get("master", False) and g.user:
            org_ids = [g.user.organization_id]

        # Check that all organization ids are within the collaboration
        # if not all([org_id in db_ids for org_id in org_ids]):
        if not set(org_ids).issubset(db_ids):
            return {"msg": (
                f"At least one of the supplied organizations in not within "
                f"the collaboration"
            )}, HTTPStatus.BAD_REQUEST

        if g.user:
            initiator = g.user.organization
        elif g.container:
            initiator = db.Node.get(g.container["node_id"]).organization
        else:
            # FIXME: Who then? Shouldn't this raise an exception?
            initiator = None

        # Create the new task in the database
        task = db.Task(
            collaboration=collaboration,
            name=data.get('name', ''),
            description=data.get('description', ''),
            image=data.get('image', ''),
            database=data.get('database', ''),
            initiator=initiator
        )

        if g.user:
            if not self.__verify_user_permissions(g.user, task):
                return {"msg": "You lack the permission to do that!"}, HTTPStatus.UNAUTHORIZED

            # Users can only create top-level -tasks (they will not
            # have sub-tasks). Therefore, always create a new run_id.
            # FIXME: I can see no reason why users wouldn't be allowed to create
            #   subtasks? For iterative processes this would actually make a lot
            #   of sense.
            task.run_id = task.next_run_id()
            log.debug(f"New run_id {task.run_id}")

        elif g.container:
            # verify that the container has permissions to create the task
            if not self.__verify_container_permissions(g.container, task):
                return {"msg": "Container-token is not valid"}, HTTPStatus.UNAUTHORIZED

            # Tasks created by containers are always sub-tasks
            task.parent_task_id = g.container["task_id"]
            task.run_id = db.Task.get(g.container["task_id"]).run_id
            log.debug(f"Sub task from parent_task_id={task.parent_task_id}")

        # permissions ok, save to DB
        task.save()

        organizations = [db.Organization.get(id_) for id_ in org_ids]
        log.debug(f"Assigning task to {len(organizations)} nodes.")

        for organization in organizations:
            log.debug(f"Assigning task to '{organization.name}'.")

            # TODO make this cleaner...
            # get organization specific (encrypted) input
            input_ = [org.get("input") for org in organizations_json_list \
                if org.get("id")==organization.id].pop()

            if isinstance(input_, dict):
                input_ = json.dumps(input_).encode(STRING_ENCODING)

            result = db.Result(
                task=task,
                organization=organization,
                input=input_,
            )
            result.save()

        # notify nodes a new task available (only to online nodes)
        socketio.emit(
            'new_task',
            task.id,
            room=f'collaboration_{task.collaboration_id}',
            namespace='/tasks'
        )

        log.info(f"New task created for collaboration '{task.collaboration.name}'")

        if g.type == 'user':
            log.debug(f" created by: '{g.user.username}'")
        else:
            log.debug((f" created by container on node_id={g.container['node_id']}"
                       f" for (master) task_id={g.container['task_id']}"))

        log.debug(f" url: '{url_for('task_with_id', id=task.id)}'")
        log.debug(f" name: '{task.name}'")
        log.debug(f" image: '{task.image}'")

        return self.task_schema.dump(task, many=False)

    def handle_master_task(self):
        """Handle creation of a master task."""
        pass

    def handle_regular_task(self):
        """Handle creation of a regular task."""
        pass

    @staticmethod
    def __verify_user_permissions(user, task: db.Task):
        """Verify that user is permitted to create task"""

        # I have the power!
        if "root" in g.user.roles:
            return True

        # user is within organization that is part of the collaboration
        return g.user.organization_id in task.collaboration.get_organization_ids()

    @staticmethod
    def __verify_container_permissions(container, task):
        """Validates that the container is allowed to create the task."""

        # check that the image is allowed
        # if container["image"] != task.image:
        if not task.image.endswith(container["image"]):
            log.warning((f"Container from node={container['node_id']} "
                f"attempts to post a task using illegal image!?"))
            log.warning(f"  task image: {task.image}")
            log.warning(f"  container image: {container['image']}")
            return False

        # check master task is not completed yet
        if db.Task.get(container["task_id"]).complete:
            log.warning((f"Container from node={container['node_id']} attempts "
                f"to start sub-task for a completed task={container['task_id']}"))
            return False

        # check that node id is indeed part of the collaboration
        if not container["collaboration_id"] == task.collaboration.id:
            log.warning((f"Container attempts to create a task outside "
                f"collaboration_id={container['collaboration_id']} in "
                f"collaboration_id={task.collaboration_id}!"))
            return False

        return True

    @only_for(['user'])
    @swag_from(str(Path(r"swagger/delete_task_with_id.yaml")), endpoint='task_with_id')
    def delete(self, id):
        """Deletes a task"""
        # TODO we might want to delete the corresponding results also?

        task = db.Task.get(id)
        if not task:
            return {"msg": "task id={} not found".format(id)}, HTTPStatus.NOT_FOUND

        task.delete()
        return {"msg": "task id={} successfully deleted".format(id)}, HTTPStatus.OK

class TaskResult(Resource):
    """Resource for /api/task/<int:id>/result"""

    task_result_schema = TaskResultSchema()

    @only_for(['user', 'container'])
    @swag_from(str(Path(r"swagger/get_task_result.yaml")), endpoint='task_result')
    def get(self, id):
        """Return results for task."""
        task = db.Task.get(id)
        if not task:
            return {"msg": "task id={} not found".format(id)}, HTTPStatus.NOT_FOUND

        return self.task_result_schema.dump(task.results, many=True).data, HTTPStatus.OK


from storage.db import Guild

prefix_cache = dict()

def prefix_changed(gid):
    del prefix_cache[gid]

def command_prefix(bot, message):
    if not message.server:
        return [bot.user.mention+' ', '!']
    gid = message.server.id
    if gid not in prefix_cache:
        guild = bot.database.get(Guild, id=gid)
        if not guild:
            guild = bot.database.add(Guild(id=gid))
        prefix_cache[gid] = [bot.user.mention+' ', guild.prefix]
    return prefix_cache[gid]