##############################################################################################
# PURPOSE
#   Measures the flux of the emission lines of the spectra by integrating the adjusted Gaussian function on these lines
#
# CREATED BY:
#   Felícia Palacios
#
# ADAPTED BY:
#   Vitor Eduardo Buss Bootz
#
# CALLING SEQUENCE
#   python Felícia_Palácios_Flux_Calculator.py     --> In terminal
#
# INPUT PARAMETERS
#   output_dir            --> Output directory name
#   output_file           --> Output filename
#   starlight_input_dir   --> Directory of input files
#   starlight_output_dir  --> Directory of input files
#
# OUTPUT
#   sample_flux_lines.csv (flag_correction == 'off')
#   corrections.csv (flag_correction == 'on')
#   PNG images showing the adjusts of the function to check its quality
#
# REQUIRED SCRIPTS
#
#
# COMMENTS
#   Vacuum wavelengths used
#   The 'flag_correction' flag allows the modification of problematic spectra without modifying the well-ajusted spectra
#
##############################################################################################

##############################################################################################
# INPUTS
##############################################################################################

def interp(xin,xout,yin): #Tentar arrumar isso no futuro.. problemas na conversão entre R e Python
    Ninterpol = len(xout)
    yout = np.array([])

    for k in range(Ninterpol):
        t = xin[xin < xout[k]]
        tind = len(t)

        if tind <= 0: tind = 1
        if tind >= len(xin): tind = len(xin) - 1
        t1 = xin[tind - 1]
        t2 = xin[tind]
        t3 = xin[tind + 1]
        tx = xout[k]

        A = (tx - t1) / (t3 - t1)
        B = (tx - t2) / (t3 - t2)
        C = (tx - t3) / (t2 - t1)
        D = (tx - t1) / (t3 - t2)
        E = (tx - t2) / (t3 - t1)

        G = (tx - t2) / (t3 - t2)
        H = (tx - t1) / (t2 - t1)

        if (t1 != t2) & (t2 != t3):
            yout = np.append(yout, yin[tind+1] * A * B - yin[tind] * D * C + yin[tind-1] * E * C)

        if (t1 == t2):
            yout = np.append(yout, yin[tind+1] - yin[tind]) * G + yin[tind]

        if(t2 == t3):
            yout = np.append(yout, yin[tind] - yin[tind-1]) * H + yin[tind-1]

    return(yout)

def interp1(xin,xout,yin):
    Ninterpol = len(xout)
    yout = np.array([])

    for k in range(Ninterpol):
        t = xin[xin < xout[k]]
        tind = len(t) -1 #VERIFICAR SE ESTE -1 NÃO INTERFERE NOS RESULTADOS

        if tind <= 0: tind = 1
        if tind >= len(xin): tind = len(xin) - 1
        t1 = xin[tind - 1]
        t2 = xin[tind]
        t3 = xin[tind + 1]
        tx = xout[k]

        A = (tx - t1) / (t3 - t1)
        B = (tx - t2) / (t3 - t2)
        C = (tx - t3) / (t2 - t1)
        D = (tx - t1) / (t3 - t2)
        E = (tx - t2) / (t3 - t1)

        G = (tx - t2) / (t3 - t2)
        H = (tx - t1) / (t2 - t1)

        if (t1 != t2) & (t2 != t3):
            yout = np.append(yout, yin[tind+1] * A * B - yin[tind] * D * C + yin[tind-1] * E * C)

        if (t1 == t2):
            yout = np.append(yout, yin[tind+1] - yin[tind]) * G + yin[tind]

        if(t2 == t3):
            yout = np.append(yout, yin[tind] - yin[tind-1]) * H + yin[tind-1]

    return(yout)

def interp2(xin,xout,yin):
    Ninterpol = len(xout)
    yout = np.array([])

    for k in range(Ninterpol):
        t = xin[xin < xout[k]]
        tind = len(t) -2 #VERIFICAR SE ESTE -1 NÃO INTERFERE NOS RESULTADOS

        if tind <= 0: tind = 1
        if tind >= len(xin): tind = len(xin) - 1
        t1 = xin[tind - 1]
        t2 = xin[tind]
        if k != Ninterpol:
            t3 = xin[tind + 1]
        tx = xout[k]

        A = (tx - t1) / (t3 - t1)
        B = (tx - t2) / (t3 - t2)
        C = (tx - t3) / (t2 - t1)
        D = (tx - t1) / (t3 - t2)
        E = (tx - t2) / (t3 - t1)

        G = (tx - t2) / (t3 - t2)
        H = (tx - t1) / (t2 - t1)

        if (t1 != t2) & (t2 != t3):
            yout = np.append(yout, yin[tind+1] * A * B - yin[tind] * D * C + yin[tind-1] * E * C)

        if (t1 == t2):
            yout = np.append(yout, yin[tind+1] - yin[tind]) * G + yin[tind]

        if(t2 == t3):
            yout = np.append(yout, yin[tind] - yin[tind-1]) * H + yin[tind-1]

    return(yout)

import numpy as np
import pandas as pd
import astropy.io.fits as fits
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
import math
import copy
from matplotlib.backends.backend_pdf import PdfPages
import datetime

#declara as função gaussianas e os contínuos que serão ajustados aos dados
def gauss(x, a, b, c, d, cs, e, f, g, h, i, j, k, l, s1, s2):
    step_min_oiii = np.heaviside(lam_final - min_oiii, 0)
    step_max_oiii = np. heaviside(lam_final - max_oiii, 0)
    step_min_hbeta = np.heaviside(lam_final - min_hbeta, 0)
    step_max_hbeta = np.heaviside(lam_final - max_hbeta, 0)
    step_min_halpha = np.heaviside(lam_final - min_nii, 0)
    step_max_halpha = np.heaviside(lam_final - max_nii, 0)
    step_min_oii = np.heaviside(lam_final - min_oii, 0)
    step_max_oii = np.heaviside(lam_final - max_oii, 0)
    step_min_sii = np.heaviside(lam_final - min_sii, 0)
    step_max_sii = np.heaviside(lam_final - max_sii, 0)

    r1 = a*(step_min_oiii - step_max_oiii)
    r2 = b*(step_min_halpha - step_max_halpha)
    r3 = c*(step_min_oii - step_max_oii)
    r4 = d*(step_min_hbeta - step_max_hbeta)
    r5 = cs*(step_min_sii - step_max_sii)

    w_oiii1 = (x - e)/f
    alpha_oiii1 = (1./math.sqrt(2.*math.pi))*np.exp((-w_oiii1**2.)/2.)*g

    cenoiii2 = lam_oiii2 + (lam_oiii2/lam_oiii1)*(e - lam_oiii1)
    sigoiii2 = (lam_oiii2/lam_oiii1)*f
    ampoiii2 = g/3.
    w_oiii2 = (x - cenoiii2)/sigoiii2
    alpha_oiii2 = (1./math.sqrt(2.*math.pi))*np.exp((-w_oiii2**2.)/2.)*ampoiii2

    cenhbeta = lam_hbeta + (lam_hbeta/lam_oiii1)*(e - lam_oiii1)
    sighbeta = (lam_hbeta/lam_oiii1)*f
    w_hbeta = (x - cenhbeta)/sighbeta
    alpha_hbeta = (1./math.sqrt(2.*math.pi))*np.exp((-w_hbeta**2.)/2.)*h

    cenhalpha = lam_halpha + (lam_halpha/lam_oiii1)*(e - lam_oiii1)
    sighalpha = (lam_halpha/lam_oiii1)*f
    w_halpha = (x - cenhalpha)/sighalpha
    alpha_halpha = (1./math.sqrt(2.*math.pi))*np.exp((-w_halpha**2.)/2.)*i

    cennii1 = lam_nii1 + (lam_nii1/lam_oiii1)*(e - lam_oiii1)
    signii1 = (lam_nii1/lam_oiii1)*f
    w_nii1 = (x - cennii1)/signii1
    alpha_nii1 = (1./math.sqrt(2.*math.pi))*np.exp((-w_nii1**2.)/2.)*j

    cennii2 = lam_nii2 + (lam_nii2/lam_oiii1)*(e - lam_oiii1)
    signii2 = (lam_nii2/lam_oiii1)*f
    ampnii2 = j/3.
    w_nii2 = (x - cennii2)/signii2
    alpha_nii2 = (1./math.sqrt(2.*math.pi))*np.exp((-w_nii2**2.)/2.)*ampnii2

    cenoii1 = lam_oii1 + (lam_oii1/lam_oiii1)*(e - lam_oiii1)
    sigoii1 = (lam_oii1/lam_oiii1)*f
    w_oii1 = (x - cenoii1)/sigoii1
    alpha_oii1 = (1./math.sqrt(2.*math.pi))*np.exp((-w_oii1**2.)/2.)*k

    cenoii2 = lam_oii2 + (lam_oii2/lam_oiii1)*(e - lam_oiii1)
    sigoii2 = (lam_oii2/lam_oiii1)*f
    w_oii2 = (x - cenoii2)/sigoii2
    alpha_oii2 = (1./math.sqrt(2.*math.pi))*np.exp((-w_oii2**2.)/2.)*l

    censii1 = lam_sii1 + (lam_sii1/lam_oiii1)*(e - lam_oiii1)
    sigsii1 = (lam_sii1/lam_oiii1)*f
    w_sii1 = (x - censii1)/sigsii1
    alpha_sii1 = (1./math.sqrt(2.*math.pi))*np.exp((-w_sii1**2.)/2.)*s1

    censii2 = lam_sii2 + (lam_sii2/lam_oiii1)*(e - lam_oiii1)
    sigsii2 = (lam_sii2/lam_oiii1)*f
    w_sii2 = (x - censii2)/sigsii2
    alpha_sii2 = (1./math.sqrt(2.*math.pi))*np.exp((-w_sii2**2.)/2.)*s2

    return (r1 + r2 + r3 + r4 + r5 + (alpha_oiii1/f) + (alpha_oiii2/sigoiii2) + (alpha_hbeta/sighbeta)
+ (alpha_halpha/sighalpha) + (alpha_nii1/signii1) + (alpha_nii2/signii2) + (alpha_oii1/sigoii1)
+ (alpha_oii2/sigoii2) + (alpha_sii1/sigsii1) + (alpha_sii2/sigsii2))

#comprimentos de onda de laboratório
lam_oiii1 = 5008.239
lam_oiii2 = 4960.295
lam_hbeta = 4862.721
lam_halpha = 6564.61
lam_nii1 = 6585.27
lam_nii2 = 6549.86
lam_oii1 = 3727.092
lam_oii2 = 3729.875

lam_sii1 = 6718.29
lam_sii2 = 6732.68

###
flag_correction = 'off'
###

### FILES & FOLDERS ###
starlight_input_dir = '/home/vitorbootz/research/files_input_starlight/'
starlight_output_dir = '/home/vitorbootz/research/files_output_starlight/'
if flag_correction == 'on':  spectra_z_file = '/home/vitorbootz/research/aux_files/to_correct.csv'
if flag_correction == 'off': spectra_z_file = '/home/vitorbootz/research/aux_files/galaxy_list.csv'
output_dir = '/home/vitorbootz/research/flux_measurements/'

if flag_correction == 'on':  output_file = 'corrections.csv'
if flag_correction == 'off': output_file = 'sample_flux_lines.csv'

### OUTPUT FILE HEADER ###
header = pd.DataFrame(['lcgID','extension','ra','dec','OIII_5008','OIII_4959', 'Hb', 'Ha', 'NII_6583', 'NII_6548', 'OII1', 'OII2', 'SII_6718', 'SII_6732']).T
header.to_csv(output_dir+output_file, index=False, mode='w', header=False)

### SELECTION ###
data = pd.read_csv(spectra_z_file, sep=',')  # Lista de spectros e redshifts
selection = (data['onoff'] == 1) & (data['extension'] < 100) #Seleciona apenas as lcgs do sdss e as do gemini com abertura = 3 arcsec
data = data[selection]
data.index = range(len(data))

for i in range(len(data)):

    if data.flag_sdss[i] == 0:
        file_vis = (starlight_input_dir + 'input_starlight_gemini_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.csv')
        file_syn = (starlight_output_dir + 'output_starlight_gemini_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.csv')

    if data.flag_sdss[i] == 1:
        file_vis = (starlight_input_dir + 'input_starlight_sdss_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.csv')
        if (data.extension[i] > 10) & (data.extension[i] < 20):
            file_syn = (starlight_output_dir + 'output_starlight_sdss_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]-10) + '.csv')
        else:
            file_syn = (starlight_output_dir + 'output_starlight_sdss_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.csv')

    hdu = pd.read_csv(file_vis, delim_whitespace=True, engine='python', header = None)
    starlight = pd.read_csv(file_syn, skiprows = 306, delim_whitespace=True, engine='python', header = None)

### O range escolhido para o starlight pega o menor e o maior lambda entre os n espectros de cada grupo ###
### Portanto, tem casos de lambdas iniciais e finais maiores e menores nos inputs em relação ao output e vice e versa ###
### Por isso aplico estas correções, para que tanto o input quando o output do starlight fiquem com o mesmo lambda range ###
    if data.flag_sdss[i] == 0:
        if (hdu[0][0] <= starlight[0][0]) & (hdu[0][len(hdu[0])-1] >= starlight[0][len(starlight[0])-1]):
            hdu = hdu[(hdu[0] >= starlight[0][0]) & (hdu[0] <= starlight[0][len(starlight[0])-1])]
            hdu.index = range(len(hdu))
        elif (hdu[0][0] > starlight[0][0]) & (hdu[0][len(hdu[0])-1] <= starlight[0][len(starlight[0])-1]):
            starlight = starlight[(starlight[0] >= hdu[0][0]) & (starlight[0] <= hdu[0][len(hdu[0])-1])]
            starlight.index = range(len(starlight))
        elif (hdu[0][0] >= starlight[0][0]) & (hdu[0][len(hdu[0])-1] >= starlight[0][len(starlight[0])-1]):
            starlight = starlight[starlight[0] >= hdu[0][0]]
            starlight.index = range(len(starlight))
            hdu = hdu[hdu[0] <= starlight[0][len(starlight[0])-1]]
            hdu.index = range(len(hdu))
        elif (hdu[0][0] <= starlight[0][0]) & (hdu[0][len(hdu[0])-1] <= starlight[0][len(starlight[0])-1]):
            hdu = hdu[hdu[0] >= starlight[0][0]]
            hdu.index = range(len(hdu))
            starlight = starlight[starlight[0] <= hdu[0][len(hdu[0])-1]]
            starlight.index = range(len(starlight))

        l_obs = hdu[0]
        f_obs = hdu[1]
        sig = hdu[2]
        f_syn = starlight[2]

#### Os espectros de input do sloan estão bem feios, então aplico uma interpolação aqui ###
    if data.flag_sdss[i] == 1:

        lamb = hdu[0]
        flux = hdu[1]
        l_starlight = starlight[0]
        f_syn = starlight[2]
        sig   = hdu[2]

        try:
            f_obs = interp(lamb, l_starlight, flux)
            sig = interp(lamb, l_starlight, sig)
            l_obs = l_starlight
        except:
            try:
                f_obs = interp1(lamb, l_starlight, flux)
                sig = interp1(lamb, l_starlight, sig)
                l_obs = l_starlight
            except:
                try:
                    f_obs = interp2(lamb, l_starlight, flux)
                    sig = interp2(lamb, l_starlight, sig)
                    l_obs = l_starlight
                except:
                    print('Erro na interpolação: LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]))
                    continue
################
### Exceções: aqui foi necessário aplicar a correção feita nos arquivos do gemini E interpolar o espectro ###
    if (data.lcgID[i] == 2764) & (data.extension[i] == 1):
        hdu = pd.read_csv(file_vis, delim_whitespace=True, engine='python', header = None)
        starlight = pd.read_csv(file_syn, skiprows = 306, delim_whitespace=True, engine='python', header = None)
        starlight = starlight[starlight[0] >= hdu[0][0]]
        starlight.index = range(len(starlight))
        hdu = hdu[hdu[0] <= starlight[0][len(starlight[0])-1]]
        hdu.index = range(len(hdu))
        lamb = hdu[0]
        flux = hdu[1]
        l_starlight = starlight[0]
        f_syn = starlight[2]
        sig   = hdu[2]
        f_obs = interp1(lamb, l_starlight, flux)
        sig = interp1(lamb, l_starlight, sig)
        l_obs = l_starlight
  #  if (data.lcgID[i] == 2023) & (data.extension[i] == 1):
  #      hdu = pd.read_csv(file_vis, delim_whitespace=True, engine='python', header = None)
  #      starlight = pd.read_csv(file_syn, skiprows = 306, delim_whitespace=True, engine='python', header = None)
  #
  #      interp2023 = pd.read_csv('/home/vitorbootz/research/aux_files/interp_2023.csv')
  #      interp2023.columns = ['n','flux_out']
  #      interp2023 = interp2023[interp2023.n>118].dropna()
  #
  #      interpsig2023 = pd.read_csv('/home/vitorbootz/research/aux_files/interp_sig_2023.csv')
  #      interpsig2023.columns = ['n','sig_out']
  #      interpsig2023 = interpsig2023[interpsig2023.n>118].dropna()
  #
  #      starlight = starlight[starlight[0] >= hdu[0][0]]
  #      starlight.index = range(len(starlight))
  #      hdu = hdu[hdu[0] <= starlight[0][len(starlight[0])-1]]
  #      hdu.index = range(len(hdu))
  #      lamb = hdu[0]
  #      flux = hdu[1]
  #      l_starlight = starlight[0]
  #      f_syn = starlight[2]
  #      sig   = hdu[2]
  #      f_obs = interp2023.flux_out
  #      sig = interpsig2023.sig_out
  #      l_obs = l_starlight
################

    cube = (f_obs - f_syn)
    #cube = (f_obs)
    cube.dropna(inplace=True)

    n = len(l_obs)
    l0 = l_obs[0]     #primeiro valor de lambda
    step = l_obs[7] - l_obs[6]          #intervalo delta lambda
    unit = 'Angstrom' #unidade

    #cria um vetor com comprimentos de onda
    if unit == 'Angstrom':
        lam = np.arange(n)*step + l0
    else:
        if unit == 'nm':
            lam = np.arange(n)*step*10 + l0*10

    #centro das linhas observadas
    cen_oiii1 = 5006.843
    cen_oiii2 = lam_oiii2 + (lam_oiii2/lam_oiii1)*(cen_oiii1 - lam_oiii1)
    cen_hbeta = lam_hbeta + (lam_hbeta/lam_oiii1)*(cen_oiii1 - lam_oiii1)
    cen_halpha = lam_halpha + (lam_halpha/lam_oiii1)*(cen_oiii1 - lam_oiii1)
    cen_nii1 = lam_nii1 + (lam_nii1/lam_oiii1)*(cen_oiii1 - lam_oiii1)
    cen_nii2 = lam_nii2 + (lam_nii2/lam_oiii1)*(cen_oiii1 - lam_oiii1)
    cen_oii1 = lam_oii1 + (lam_oii1/lam_oiii1)*(cen_oiii1 - lam_oiii1)
    cen_oii2 = lam_oii2 + (lam_oii2/lam_oiii1)*(cen_oiii1 - lam_oiii1)

    cen_sii1 = lam_sii1 + (lam_sii1/lam_oiii1)*(cen_oiii1 - lam_oiii1)
    cen_sii2 = lam_sii2 + (lam_sii2/lam_oiii1)*(cen_oiii1 - lam_oiii1)

    #cria janela do OIII
    min_oiii = cen_oiii2 - 20
    max_oiii = cen_oiii1 + 20

    imin_oiii = (np.abs(lam - min_oiii)).argmin()
    imax_oiii = (np.abs(lam - max_oiii)).argmin()

    lam1 = lam[imin_oiii : imax_oiii]
    cube1 = cube[imin_oiii : imax_oiii]
    sig1 = sig[imin_oiii : imax_oiii]

    #cria janela do H beta
    min_hbeta = cen_hbeta - 20
    max_hbeta = cen_hbeta + 20

    imin_hbeta = (np.abs(lam - min_hbeta)).argmin()
    imax_hbeta = (np.abs(lam - max_hbeta)).argmin()

    lam2 = lam[imin_hbeta : imax_hbeta]
    cube2 = cube[imin_hbeta : imax_hbeta]
    sig2 = sig[imin_hbeta : imax_hbeta]

    #cria janela do NII
    min_nii = cen_nii2 - 20
    max_nii = cen_nii1 + 20

    imin_nii = (np.abs(lam - min_nii)).argmin()
    imax_nii = (np.abs(lam - max_nii)).argmin()

    lam3 = lam[imin_nii : imax_nii]
    cube3 = cube[imin_nii : imax_nii]
    sig3 = sig[imin_nii : imax_nii]

    #cria janela do SII
    min_sii = cen_sii1 - 30
    max_sii = cen_sii2 + 30

    imin_sii = (np.abs(lam - min_sii)).argmin()
    imax_sii = (np.abs(lam - max_sii)).argmin()

    lam5 = lam[imin_sii : imax_sii]
    cube5 = cube[imin_sii : imax_sii]
    sig5 = sig[imin_sii : imax_sii]

    #cria janela do OII
    min_oii = cen_oii1 - 20
    max_oii = cen_oii2 + 20

    #abre o arquivo do espectro visual se as linhas de OII cairem fora do infravermelho
    if min_oii < 10000:
        #head_vis = hdu_vis[0].header
        #data_vis = hdu_vis[0].data
        #sig_vis = hdu_vis[1].data/1e-19

        #cube_vis = data_vis/1e-19

        cube_vis = cube
        sig_vis = sig

        n_vis = len(cube_vis)
        l0_vis = l_obs[0]
        step_vis = l_obs[7] - l_obs[6]
        unit_vis = 'Angstrom'

        if unit_vis == 'Angstrom':
            lam_vis = np.arange(n_vis)*step_vis + l0_vis
        else:
            if unit_vis == 'nm':
                lam_vis = np.arange(n_vis)*step_vis*10 + l0_vis*10

        imin_oii = (np.abs(lam_vis - min_oii)).argmin()
        imax_oii = (np.abs(lam_vis - max_oii)).argmin()

        lam4 = lam_vis[imin_oii : imax_oii]
        cube4 = cube_vis[imin_oii : imax_oii]
        sig4 = sig_vis[imin_oii : imax_oii]

    else:

        imin_oii = (np.abs(lam - min_oii)).argmin()
        imax_oii = (np.abs(lam - max_oii)).argmin()

        lam4 = lam[imin_oii : imax_oii]
        cube4 = cube[imin_oii : imax_oii]
        sig4 = sig[imin_oii : imax_oii]

    #junta as janelas
    lam_fin1 = np.append(lam1,lam2)
    cube_fin1 = np.append(cube1,cube2)
    sig_fin1 = np.append(sig1,sig2)
    lam_fin2 = np.append(lam_fin1, lam3)
    cube_fin2 = np.append(cube_fin1, cube3)
    sig_fin2 = np.append(sig_fin1, sig3)
    lam_fin = np.append(lam_fin2, lam4)
    cube_fin = np.append(cube_fin2, cube4)
    sig = np.append(sig_fin2, sig4)

    lam_final = np.append(lam_fin, lam5)
    cube_final = np.append(cube_fin, cube5)
    sig_final = np.append(sig, sig5)

    #chute inicial dos parâmetros
    if flag_correction == 'off': guess = np.array([10, 10, 10, 10, 10, cen_oiii1, 60*cen_oiii1/(2.99792*1e5), 100, 100, 100, 100, 100, 100, 100, 100])
    if flag_correction == 'on':  guess = np.array([10, 10, 10, 10, 10, cen_oiii1, 60*cen_oiii1/(2.99792*1e5), 100, 100, 100, 100, 100, 100, 100, 100])

    #ajuste das funções aos dados
    try:
        if (data.lcgID[i] == 3866) & (data.extension[i] == 13):
            p, pv = curve_fit(gauss, lam_final, cube_final, guess, sigma=None, method='lm')
        else:
            p, pv = curve_fit(gauss, lam_final, cube_final, guess, sigma=sig_final, method='lm')
    except:
        print('Erro no curve_fit: LCG'+str(data.lcgID[i])+'_'+str(data.extension[i]))
        continue

    #parâmetros do ajuste
    a1 = p[0] #continuo 1
    a2 = p[1] #continuo 2
    a3 = p[2] #continuo 3
    a4 = p[3] #continuo 4
    a5 = p[4] #continuo 5
    center_oiii1 = p[5]
    sig_oiii1 = copy.deepcopy(p[6])
    amp_oiii1 = p[7]
    amp_hbeta = p[8]
    amp_halpha = p[9]
    amp_nii1 = p[10]
    amp_oii1 = p[11]
    amp_oii2 = p[12]
    amp_sii1 = p[13]
    amp_sii2 = p[14]

    #outros parâmetros
    center_oiii2 = lam_oiii2 + (lam_oiii2/lam_oiii1)*(center_oiii1 - lam_oiii1)
    center_hbeta = lam_hbeta + (lam_hbeta/lam_oiii1)*(center_oiii1 - lam_oiii1)
    center_halpha = lam_halpha + (lam_halpha/lam_oiii1)*(center_oiii1 - lam_oiii1)
    center_nii1 = lam_nii1 + (lam_nii1/lam_oiii1)*(center_oiii1 - lam_oiii1)
    center_nii2 = lam_nii2 + (lam_nii2/lam_oiii1)*(center_oiii1 - lam_oiii1)
    center_oii1 = lam_oii1 + (lam_oii1/lam_oiii1)*(center_oiii1 - lam_oiii1)
    center_oii2 = lam_oii2 + (lam_oii2/lam_oiii1)*(center_oiii1 - lam_oiii1)

    center_sii1 = lam_sii1 + (lam_sii1/lam_oiii1)*(center_oiii1 - lam_oiii1)
    center_sii2 = lam_sii2 + (lam_sii2/lam_oiii1)*(center_oiii1 - lam_oiii1)

    sigma_oiii2 = (lam_oiii2/lam_oiii1)*sig_oiii1
    sigma_hbeta = (lam_hbeta/lam_oiii1)*sig_oiii1
    sigma_halpha = (lam_halpha/lam_oiii1)*sig_oiii1
    sigma_nii1 = (lam_nii1/lam_oiii1)*sig_oiii1
    sigma_nii2 = (lam_nii2/lam_oiii1)*sig_oiii1
    sigma_oii1 = (lam_oii1/lam_oiii1)*sig_oiii1
    sigma_oii2 = (lam_oii2/lam_oiii1)*sig_oiii1

    sigma_sii1 = (lam_sii1/lam_oiii1)*sig_oiii1
    sigma_sii2 = (lam_sii2/lam_oiii1)*sig_oiii1

    amp_oiii2 = amp_oiii1/3.
    amp_nii2 = amp_nii1/3.

    vel = p[5]*(299792.458/p[4])

    #plot dos espectros e ajustes
    fig = plt.figure(figsize=(6,20))
    axis1 = fig.add_subplot(511)
    axis1.plot(lam1,cube1)
    x = lam1
    y = (a1 + amp_oiii1*(1./math.sqrt(2.*math.pi))*np.exp((-((x - center_oiii1)/sig_oiii1)**2.)/2.)/sig_oiii1
         + amp_oiii2*(1./math.sqrt(2.*math.pi))*np.exp((-((x - center_oiii2)/sigma_oiii2)**2.)/2.)/sigma_oiii2)
    axis1.plot(x,y, color = 'red')
    plt.title('LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i])+': OIII Window')

    axis2 = fig.add_subplot(512)
    axis2.plot(lam2, cube2)
    x1 = lam2
    y1 = (a4 + amp_hbeta*(1./math.sqrt(2.*math.pi))*np.exp((-((x1 - center_hbeta)/sigma_hbeta)**2.)/2.)/sigma_hbeta )
    axis2.plot(x1,y1, color = 'red')
    plt.title('LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i])+': H Beta Window')

    axis3 = fig.add_subplot(513)
    axis3.plot(lam3, cube3)
    x2 = lam3
    y2 = (a2 + amp_halpha*(1./math.sqrt(2.*math.pi))*np.exp((-((x2 - center_halpha)/sigma_halpha)**2.)/2.)/sigma_halpha
         + amp_nii1*(1./math.sqrt(2.*math.pi))*np.exp((-((x2 - center_nii1)/sigma_nii1)**2.)/2.)/sigma_nii1
         + amp_nii2*(1./math.sqrt(2.*math.pi))*np.exp((-((x2 - center_nii2)/sigma_nii2)**2.)/2.)/sigma_nii2)
    axis3.plot(x2,y2, color = 'red')
    plt.title('LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i])+': NII Window')

    axis4 = fig.add_subplot(514)
    axis4.plot(lam4,cube4)
    x3 = lam4
    y3 = (a3 + amp_oii1*(1./math.sqrt(2.*math.pi))*np.exp((-((x3 - center_oii1)/sigma_oii1)**2.)/2.)/sigma_oii1
         + amp_oii2*(1./math.sqrt(2.*math.pi))*np.exp((-((x3 - center_oii2)/sigma_oii2)**2.)/2.)/sigma_oii2)
    axis4.plot(x3,y3, color = 'red')
    plt.title('LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i])+': OII Window')

    axis5 = fig.add_subplot(515)
    axis5.plot(lam5,cube5)
    x4 = lam5
    y4 = (a5 + amp_sii1*(1./math.sqrt(2.*math.pi))*np.exp((-((x4 - center_sii1)/sigma_sii1)**2.)/2.)/sigma_sii1
         + amp_sii2*(1./math.sqrt(2.*math.pi))*np.exp((-((x4 - center_sii2)/sigma_sii2)**2.)/2.)/sigma_sii2)
    axis5.plot(x4,y4, color = 'red')

    plt.title('LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i])+': SII Window')

    if flag_correction == 'off':
        if data.flag_sdss[i] == 1:
            fig.savefig('/home/vitorbootz/research/flux_measurements/log_images/' + 'sdss_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.png')
        if data.flag_sdss[i] == 0:
            fig.savefig('/home/vitorbootz/research/flux_measurements/log_images/' + 'gemini_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.png')

    if flag_correction == 'on':
        if data.flag_sdss[i] == 1:
            fig.savefig('/home/vitorbootz/research/flux_measurements/to_correct/' + 'sdss_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.png')
        if data.flag_sdss[i] == 0:
            fig.savefig('/home/vitorbootz/research/flux_measurements/to_correct/' + 'gemini_LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]) + '.png')

    ###########################################################################
    ###########################################################################
    ###########################################################################

    ### OUTPUT DATA ###

    amp_oiii1 = round(amp_oiii1,5)
    amp_oiii2 = round(amp_oiii2,5)
    amp_hbeta = round(amp_hbeta,5)
    amp_halpha = round(amp_halpha,5)
    amp_nii1 = round(amp_nii1,5)
    amp_oii1 = round(amp_oii1,5)
    amp_oii2 = round(amp_oii2,5)
    amp_sii1 = round(amp_sii1,5)
    amp_sii2 = round(amp_sii2,5)

    fluxes = pd.DataFrame([data.lcgID[i], data.extension[i], data.ra[i], data.dec[i], amp_oiii1, amp_oiii2, amp_hbeta, amp_halpha, amp_nii1, amp_nii2, amp_oii1, amp_oii2, amp_sii1, amp_sii2]).T
    fluxes[0] = int(fluxes[0])
    fluxes[1] = int(fluxes[1])
    print('LCG' + str(data.lcgID[i]) + '_' + str(data.extension[i]))
    if flag_correction == 'on': fluxes.to_csv(output_dir+output_file, index=False, header=None, mode='a')
    if flag_correction == 'off': fluxes.to_csv(output_dir+output_file, index=False, header=None, mode='a')


#script scale.py
#!/usr/bin/python3
"""
Auteur: Vianney Adou
date: 9 /Nov/ 2014
Version: 0.0.1
Licence: GPL2
"""

from tkinter import *
""" BKM couleur est une apllication pour les webmasters
   BKM Vianny couleur produit les codes de couleurs en hexadecimal """

rougeb ="00"
vertb ="00"
bleub ="00"

#decodeur hexadecimale sur deux digits
def decode(valeur):
    # nouvelle valeur en argument
    if((valeur%16)==10):
        result = "A"
    elif((valeur%16)==11):
        result = "B"
    elif((valeur%16)==12):
        result = "C"
    elif((valeur%16)==13):
        result = "D"
    elif((valeur%16)==14):
        result = "E"
    elif((valeur%16)==15):
        result = "F"
    else:
        result =str(valeur%16)
    return result

def maj(valeur):
    conver = []
    #permet le decodage sur deux digits
    conver.append(decode(valeur))
    valeur //= 16
    conver.append(decode(valeur))
    var = conver[1]+conver[0]
    return var

#fonction pour la gestion des modificateurs (Scale)
def coderouge(rouge):
    global vertb
    global bleub
    global rougeb
    rougeb = maj(int(rouge))
    cod.set("#"+rougeb+vertb+bleub)
    couleur["bg"]=str(cod.get())

def codevert(vert):
    global rougeb
    global bleub
    global vertb
    vertb  = maj(int(vert))
    cod.set("#"+rougeb+vertb+bleub)
    couleur["bg"]=str(cod.get())

def codebleu(bleu):
    global rougeb
    global vertb
    global bleub
    global codCouleur
    bleub  = maj(int(bleu))
    cod.set("#"+rougeb+vertb+bleub)
    couleur["bg"]=str(cod.get())

#cration de la fenetre principale (main window)

Mafenetre = Tk()
Mafenetre.title("BKM couleur")

rouge, vert, bleu = StringVar(), StringVar(), StringVar()
rouge.set(0)
vert.set(0)
bleu.set(0)

#creatrion de 3 widgets scale

Srouge = Scale(Mafenetre, from_ = 0, to=255, resolution=1, orient=HORIZONTAL, length=300, width=10, label='Rouge', variable=rouge, command=coderouge)
Srouge.grid(row=0, column=0)

Svert = Scale(Mafenetre, from_ = 0, to=255, resolution=1, orient=HORIZONTAL, length=300, width=10, label='Vert', variable=vert, command=codevert)
Svert.grid(row=1, column=0)

Sbleu = Scale(Mafenetre, from_ = 0, to=255, resolution=1, orient=HORIZONTAL, length=300, width=10, label='Bleu',  variable=bleu, command=codebleu)
Sbleu.grid(row=2, column=0)

#Creation du canvas pour la couleur
couleur = Canvas(Mafenetre, width=100, height=100)
couleur.grid(row=0, column=1, rowspan=2)

#ceation du label pour le code
cod = StringVar()
code = Entry(Mafenetre, textvariable=cod ,width=10, bg="white")

code.grid(row=2, column=1)
Mafenetre.mainloop()


class Solution:
    def hasGroupsSizeX(self, deck):
        """
        :type deck: List[int]
        :rtype: bool
        """
        def gcd(a,b):
            while b:
                a,b = b, a%b #how to get the gcd using euclidean algo
            return a
        a = [v for k,v in collections.Counter(deck).items()] #get list of values for counts
        ans = a[0]
        for i in range(1,len(a)): #run through every count value and get the gcd for each pair of counts from beginning to end
            ans = gcd(ans,a[i])
        return ans >= 2 # if final value is not 1, true else false


import math
import numpy as np
import scipy.stats
import os
import matplotlib.pyplot as plt

def entropy(string):
        "Calculates the Shannon entropy of a string"

        # get probability of chars in string
        prob = [ float(string.count(c)) / len(string) for c in dict.fromkeys(list(string)) ]

        # calculate the entropy
        entropy = - sum([ p * math.log(p) / math.log(2.0) for p in prob ])

        return entropy

def mean_confidence_interval(data, confidence):
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)
    return h

def bin_entropy(file):
    archivo = open(file,'rb').read()
    string = str(archivo)
    prob = [ float(string.count(c)) / len(string) for c in dict.fromkeys(list(string)) ]
    entropy = - sum([ p * math.log(p) / math.log(2.0) for p in prob ])
    return entropy

programas = os.listdir("/home/r/Semestre2020-2/malware_connections/test_small_sample")
print(programas)
entropias = []

cont = 0
for i in programas:
    path = '/home/r/Semestre2020-2/malware_connections/test_small_sample/'
    file = path + i
    entropias.append(bin_entropy(file))
    print("vamos en {}/{}".format(cont, len(programas)))
    cont+=1

#calculos de todas las propiedades de las entropias   |mar oct 20 18:15:59 -05 2020|
media_entropias = sum(entropias) / len(entropias)
print("media" , media_entropias)

var_entropias = sum([(entropia-media_entropias) **2 for entropia in entropias])/len(entropias)
print("var" , var_entropias)

desv_estandar_entropias = math.sqrt(var_entropias)
print("desv estandar:" , desv_estandar_entropias)

h_intervalo_confianza_entropias = mean_confidence_interval(entropias , 0.95)
intervalo_top = media_entropias + h_intervalo_confianza_entropias
intervalo_down = media_entropias - h_intervalo_confianza_entropias
intervalo_confianza_entropias = [intervalo_down , intervalo_top]
print("intervalos de confianza:" , intervalo_confianza_entropias)
print("\n \n \n")

name = "not_encoded.png"
x = range(0,len(programas))
y = entropias
plt.stem(x,y)
plt.savefig(name)


#
#                                                                  gemini_python
#
#                                                      primitives_gsaoi_image.py
# ------------------------------------------------------------------------------
from gempy.gemini import gemini_tools as gt

from geminidr.core import Image, Photometry
from recipe_system.utils.decorators import parameter_override

from .primitives_gsaoi import GSAOI
from . import parameters_gsaoi_image

@parameter_override
class GSAOIImage(GSAOI, Image, Photometry):
    """
    This is the class containing all of the preprocessing primitives
    for the F2Image level of the type hierarchy tree. It inherits all
    the primitives from the level above
    """
    tagset = {"GEMINI", "GSAOI", "IMAGE"}

    def __init__(self, adinputs, **kwargs):
        super().__init__(adinputs, **kwargs)
        self._param_update(parameters_gsaoi_image)

    def makeLampFlat(self, adinputs=None, **params):
        """
        This produces an appropriate stacked GSAOI imaging flat, based on
        the inputs, since one of two procedures must be followed.

        In the standard recipe, the inputs will have come from getList and
        so will all have the same filter and will all need the same recipe.
        """
        log = self.log
        log.debug(gt.log_message("primitive", self.myself(), "starting"))

        # Leave now with empty list to avoid error when looking at adinputs[0]
        if not adinputs:
            return adinputs

        if adinputs[0].effective_wavelength(output_units='micrometers') < 1.4:
            log.stdinfo('Using stackFrames to make flatfield')
            params.update({'scale': False, 'zero': False})
            adinputs = self.stackFrames(adinputs, **params)
        else:
            log.stdinfo('Using standard makeLampFlat primitive to make flatfield')
            adinputs = super().makeLampFlat(adinputs, **params)

        return adinputs


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse
from django.views.generic import ListView,DetailView
from products.models import Product
from django.urls import reverse
import json

# IMPLEMENT THE COMMENT VIEWS HERE. THANKS

def get_product(request):
      productlist= Product.objects.all()
      user=request.user
      context={
          'products' : productlist,
          'user':user,
      }
      return render(request,'blog/homeview.html', context)

def details(request, id):
    product=Product.objects.get(id=id)
    user=request.user
    context={
        'product':product,
        'user':user,
    }
    return render(request,'blog/product_details.html', context)

def like_view(request):
    #user =request.user
    if (request.method=="POST"):
       if request.POST.get("operation") == "like_submit" and request.is_ajax():
         content_id=request.POST.get("content_id",None)
         content=get_object_or_404(Product,pk=content_id)
         if content.likes.filter(id=request.user.id): #already liked the content
            content.likes.remove(request.user) #remove user from likes
            liked=False
         else:
             content.likes.add(request.user)
             liked=True
         ctx={"likes_count":content.total_likes(),"liked":liked,"content_id":content_id}
         return HttpResponse(json.dumps(ctx), content_type='application/json')

    contents=Product.objects.all()
    already_liked=[]
    id=request.user.id
    for content in contents:
        if(content.likes.filter(id=id).exists()):
            already_liked.append(content.id)
    ctx={"contents":contents,"already_liked":already_liked }
    return render(request,"blog/product_details.html",ctx)


import rdflib
from mFuhsionPerfect import MFuhsionPerfect
import configparser
import requests
import json
import os.path
import sys, getopt
import  fileinput

allJoins = []
mergedMolecules = []

# When the Class use an URI as a Fusion Policy the Elements to Join will be
# joined by the predicate URI.
def uriPolicy(toJoin, g, g2, fusion_policy):
	gNew = rdflib.Graph()
	for elem in toJoin:
		subject1 = rdflib.URIRef(elem['uri1'])
		for node in g.predicate_objects(subject=subject1):
			gNew.add((subject1, node[0], node[1]))

		subject2 = rdflib.URIRef(elem['uri2'])
		for node in g2.predicate_objects(subject=subject2):
			gNew.add((subject2, node[0], node[1]))

		gNew.add((subject1, rdflib.URIRef(fusion_policy) ,subject2))

	return (gNew.serialize(format='nt'))[:-1]

def unionPolicy(fusion_policy, jsonToJoin):
	headers = {'content-type': "application/json"}
	url = "http://localhost:9001/fusion/"+fusion_policy
	data = jsonToJoin
	data = json.dumps(data)
	response = requests.post(url, data=data, headers=headers)
	return response.text

def getPredicateObject(subjects, graph, index, name_class, class_identifier):

	mergeDict = []

	for s in subjects:

		success = False
		for o in graph.objects(subject=s, predicate=class_identifier):
			if str(o)==name_class:
				success = True

		if not success:
			continue

		dict_new = {}
		dict_new['head'] = {}
		dict_new['head']['index'] = index
		index += 1
		dict_new['head']['uri'] = s.n3()
		dict_new['head']['row'] = True
		dict_new['tail'] = []
		for i in graph.predicate_objects(subject=s):
			dict_predicateObject = {}
			# PREDICATE = PROP
			dict_predicateObject['prop'] = i[0].n3()
			# OBJECT = VALUE
			dict_predicateObject['value'] = i[1].n3()
			dict_new['tail'].append(dict_predicateObject)

		mergeDict.append(dict_new)

	return mergeDict

def getSubject(subjects, graph, toJoin, uri):

	belong = False
	newGraph = rdflib.Graph()
	for subject in subjects:
		for tj in toJoin:
			belong = False

			compare = tj[uri].replace('<','').replace('>','')
			if compare == str(subject):
				belong = True

			if belong:
				break

		if not belong:
			for node in graph:
				if node[0] == subject:
					newGraph.add(node)

	return newGraph

def getUnusedNodes(subjects, subjects2, g, g2, toJoin, F):
	toSearch = getSubject(subjects, g, toJoin, 'uri1')
	toSearch2 = getSubject(subjects2, g2, toJoin, 'uri2')

	len_toSearch = toSearch.serialize(format='nt')
	len_toSearch2 = toSearch2.serialize(format='nt')

	if len(len_toSearch) > 1:
		for s,p,o in toSearch:
			elem = s.n3() + ' '
			elem += p.n3() + ' '
			obj = o.n3().replace('"', '\"')
			elem += obj + '.\n'
			F.write(elem)

	if len(len_toSearch2) > 1:
		for s,p,o in toSearch2:
			elem = s.n3() + ' '
			elem += p.n3() + ' '
			obj = o.n3().replace('"', '\"')
			elem += obj + '.\n'
			F.write(elem)

	return 0

def integratePerClass(g, g2, subjects, subjects2, n, F, config, class_identifier):
	mergeDict = []
	mergeDict2 = []
	index = 0

	n += 1

	threshold = float(config.get('Class'+str(n),'threshold'))
	simfunction = config.get('Class'+str(n),'similarity_metric')
	fusion_policy = config.get('Class'+str(n),'fusion_policy')
	name_class = config.get('Class'+str(n),'class')

	mergeDict = getPredicateObject(subjects, g, index, name_class, class_identifier)
	mergeDict2 = getPredicateObject(subjects2, g2, index, name_class, class_identifier)

	perfectOp = MFuhsionPerfect(threshold, simfunction)

	perfectOp.execute_new(mergeDict, mergeDict2)
	toJoin = []
	for tpl in perfectOp.toBeJoined:
		tupl = {}
		tupl['uri1'] = str(tpl[0])
		tupl['uri2'] = str(tpl[1])
		toJoin.append(tupl)

	global allJoins

	allJoins += toJoin

	if(name_class == 'http://project-iasis.eu/vocab/Publication'):
		print(toJoin)

	for elem in toJoin:
		elem['uri1'] = elem['uri1'].replace('<','').replace('>','')
		elem['uri2'] = elem['uri2'].replace('<','').replace('>','')

	for elemToJoin in toJoin:
		jsonToJoin = {}
		jsonToJoin["tasks"] = [elemToJoin]

		if fusion_policy == 'union':
			mergedUris = {}
			resp_object = unionPolicy(fusion_policy, jsonToJoin)
			resp_object = resp_object.replace('http://vocab.lidakra.de/fuhsen/search/merged_entity/','http://project-iasis.eu/')
			subject = (resp_object.split())[0]
			mergedUris['uri1'] = '<' + elemToJoin['uri1'] + '>'
			mergedUris['uri2'] = '<' + elemToJoin['uri2'] + '>'
			mergedUris['newUri'] = subject

			global mergedMolecules
			mergedMolecules.append(mergedUris)

		elif fusion_policy != '':
			resp_object = uriPolicy(toJoin, g, g2, fusion_policy)
		else:
			resp_object = []

		F.write(resp_object)

def integrator(config_file):
	config = configparser.ConfigParser()
	config.read(config_file)

	save_path = config.get('RDFData','pathToSave')
	completeName = os.path.join(save_path, "new_rdfGraph.nt")

	F = open(completeName, "w")

	file_name1 = config.get('RDFData','rdf1')
	file_name2 = config.get('RDFData','rdf2')
	file_ontologies = config.get('RDFData','ontology')

	g=rdflib.Graph()
	result1 = g.parse(location=file_name1, format="nt")
	g2 = rdflib.Graph()
	result2 = g2.parse(location=file_name2, format="nt")
	subjects = []
	predicates = []
	objects = []
	subjects2 = []
	predicates2 = []
	objects2 = []

	for subject,predicate,obj in g:
	    subjects.append(subject)
	    predicates.append(predicate)
	    if 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' == str(predicate):
	    	class_identifier = predicate
	    objects.append(obj)

	for subject,predicate,obj in g2:
	    subjects2.append(subject)
	    predicates2.append(predicate)
	    objects2.append(obj)

	subjects = list(set(subjects))
	subjects2 = list(set(subjects2))

	number_classes = int(config.get('RDFData','number_classes'))

	FileAux = open('auxFile.nt', 'w+')

	f = open(file_ontologies, 'r')
	for line in f.readlines():
		FileAux.write(line)
	f.close()

	f = open(file_name1, 'r')
	for line in f.readlines():
		FileAux.write(line)
	f.close()

	f = open(file_name2, 'r')
	for line in f.readlines():
		FileAux.write(line)
	f.close()
	FileAux.close()

	ontology_path = os.path.realpath(FileAux.name)

	url = "http://localhost:9000/similarity/initialize?model_1="+ontology_path
	headers = {'content-type': "application/json"}
	response = requests.get(url, headers=headers)

	url = "http://localhost:9001/setlocation"
	data = {"location1":file_name1,
		"location2":file_name2}
	data = json.dumps(data)
	response = requests.post(url, data=data, headers=headers)

	for n in range(number_classes):
		integratePerClass(g, g2, subjects, subjects2, n, F, config, class_identifier)

	getUnusedNodes(subjects, subjects2, g, g2, allJoins, F)

	F.close()

	for line in fileinput.input(completeName, inplace=True):
		global mergedMolecules
		replace = False
		for elem in mergedMolecules:
			if (line.find(elem['uri1']) > 0):
				print(line.replace(elem['uri1'],elem['newUri'])[:-1])
				replace = True
			elif (line.find(elem['uri2']) > 0):
				print(line.replace(elem['uri2'],elem['newUri'])[:-1])
				replace = True
		if not (replace):
			print(line[:-1])

	g = rdflib.Graph()
	result = g.parse(location=completeName, format="nt")

	file = open(completeName, "w")
	for s,p,o in g:
			elem = s.n3() + ' '
			elem += p.n3() + ' '
			obj = o.n3().replace('"', '\"')
			elem += obj + '.\n'
			file.write(elem)

	file.close()

def readConfig():
	argv = sys.argv[1:]
	try:
		opts, args = getopt.getopt(argv, 'hc:', 'configFile=')
	except getopt.GetoptError:
		print('python integrator.py -c <configFile>')
		sys.exit(1)

	if len(opts) == 0:
		print('Error: python integrator.py -c <configFile>')
		sys.exit(1)

	for opt, arg in opts:
		if opt == '-h':
			print('python integrator.py -c <configFile>')
			sys.exit()
		elif opt == '-c' or opt == '--configFile':
			config_file = arg

	return config_file

def main():

	config_file = readConfig()
	integrator(config_file)

if __name__ == "__main__":
    main()


from os import walk, sep
from os.path import basename, normpath, join

from peewee import DoesNotExist

from logic.identifier import add_identifiers
from logic.reserved import add_reserveds, get_reserved_by_name, \
    reserved_prefixes

def file_gen(from_dir, do_recursive=True, file_name=None):
    """Process a file or directory.

    Parameters
    ----------
    do_recursive : bool
    file_name : str
    from_dir : str

    Yields
    ------
    dir_name : str
    do_obfuscate : bool
    file_name : str
    is_reserved_dir : bool
    is_reserved_file : bool
    """
    # Process for a single file
    if file_name:
        # Skip excluded files
        is_reserved_dir = None
        try:
            get_reserved_by_name(''.join([
                reserved_prefixes.reserved_file, file_name]))
        except DoesNotExist:
            is_reserved_file = False
        else:
            is_reserved_file = True

        yield file_name, from_dir, is_reserved_dir, is_reserved_file
        return

    # Process for a directory
    for dir_name, subdir_list, file_list in walk(from_dir):
        if do_recursive and not dir_name:
            break
        # Remove unwanted files from file_list
        file_list = [file_name for file_name in file_list
                     if file_name[-4:] not in ['.pyc', '.pyo']
                     ]
        #####################
        # Process directories
        #####################
        is_reserved_dir = True
        do_obfuscate_dir = False

        # Identify reserved directories
        try:
            reserved_row = get_reserved_by_name(
                ''.join([reserved_prefixes.reserved_dir, basename(dir_name)]))
        except DoesNotExist:
            is_reserved_dir = False
        else:
            # Add identifiers to reserved
            if subdir_list:
                add_reserveds(reserved_row.primary_package, subdir_list,
                              reserved_prefixes.reserved_dir)
            if file_list:
                add_reserveds(basename(dir_name), file_list,
                              reserved_prefixes.reserved_file)

        # Identify non-obfuscated directories
        if not is_reserved_dir:
            try:
                get_reserved_by_name(''.join([
                    reserved_prefixes.non_obfuscated_dir, basename(dir_name)]))
            except DoesNotExist:
                do_obfuscate_dir = True

                # Add to obfuscated identifiers
                add_identifiers(basename(dir_name))
                if subdir_list:
                    add_identifiers(subdir_list)
                if file_list:
                    ob_files = [
                        file_name[:-3] for file_name in file_list
                        if file_name[-3:] in ['.py', '.ky']
                        ]
                    if ob_files:
                        add_identifiers(ob_files)
            else:
                # Add non-obfuscated identifiers
                add_identifiers([basename(dir_name)], do_obfuscate=False)
                if subdir_list:
                    add_reserveds(basename(dir_name), subdir_list,
                                  reserved_prefixes.non_obfuscated_dir)
                if file_list:
                    add_reserveds(basename(dir_name), file_list,
                                  reserved_prefixes.non_obfuscated_file)

        for file_name in file_list:
            do_obfuscate = False
            is_reserved_file = False
            # Files in reserved directory are reserved
            if is_reserved_dir:
                is_reserved_file = True
            else:
                # Identify reserved files
                try:
                    get_reserved_by_name(''.join([
                        reserved_prefixes.reserved_file, file_name]))
                except DoesNotExist:
                    # Identify files to obfuscate
                    if do_obfuscate_dir:
                        try:
                            get_reserved_by_name(''.join([
                                reserved_prefixes.non_obfuscated_file,
                                file_name]))
                        except DoesNotExist:
                            do_obfuscate = True
                else:
                    is_reserved_file = True

            if file_name == '__init__.py' and \
                    not is_reserved_dir and \
                    not do_obfuscate_dir:
                do_obfuscate = True

            yield file_name, dir_name, is_reserved_dir, is_reserved_file, \
                do_obfuscate

def obfuscate_path(file_path):
    """Create an obfuscated path name.

    Parameters
    ----------
    file_path : str

    Returns
    -------
    obfuscated path : str
    """
    from logic.identifier import get_obfuscated_name

    obf_path = [get_obfuscated_name(p) for
                p in normpath(file_path).split(sep)]

    if obf_path[-1][-3:] in ['.py', '.kv']:
        obf_path[-1] = ''.join([get_obfuscated_name(obf_path[-1][:-3]),
                                obf_path[-1][-3:]])

    return join(*obf_path)

def split_path(file_path):
    """Split a path into its parts.

    Parameters
    ----------
    file_path : str

    Returns
    -------
    path : list
    """
    return normpath(file_path).split(sep)

def to_str(unicode_or_str):
    """Convert a string to unicode.

    Parameters
    ----------
    unicode_or_str : unicode  # or str

    Returns
    -------
    value : str
    """
    if isinstance(unicode_or_str, unicode):
        value = unicode_or_str.encode('utf-8')
    else:
        value = unicode_or_str
    return value

def to_unicode(unicode_or_string):
    """Convert a string to unicode.

    Parameters
    ----------
    unicode_or_string : str  # or unicode

    Returns
    -------
    value : unicode
    """
    if isinstance(unicode_or_string, str):
        value = unicode_or_string.decode('utf-8')
    else:
        value = unicode_or_string
    return value


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2020 Confluent Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# This is a simple example of the SerializingProducer using protobuf.
#
# To regenerate Protobuf classes you must first install the protobuf
# compiler. Once installed you may call protoc directly or use make.
#
# See the protocol buffer docs for instructions on installing and using protoc.
# https://developers.google.com/protocol-buffers/docs/pythontutorial
#
# After installing protoc execute the following command from the examples
# directory to regenerate the user_pb2 module.
# `make`
#
import argparse

# Protobuf generated class; resides at ./user_pb2.py
import user_pb2
from confluent_kafka import DeserializingConsumer
from confluent_kafka.schema_registry.protobuf import ProtobufDeserializer
from confluent_kafka.serialization import StringDeserializer

def main(args):
    topic = args.topic

    protobuf_deserializer = ProtobufDeserializer(user_pb2.User)
    string_deserializer = StringDeserializer('utf_8')

    consumer_conf = {'bootstrap.servers': args.bootstrap_servers,
                     'key.deserializer': string_deserializer,
                     'value.deserializer': protobuf_deserializer,
                     'group.id': args.group,
                     'auto.offset.reset': "earliest"}

    consumer = DeserializingConsumer(consumer_conf)
    consumer.subscribe([topic])

    while True:
        try:
            # SIGINT can't be handled when polling, limit timeout to 1 second.
            msg = consumer.poll(1.0)
            if msg is None:
                continue

            user = msg.value()
            if user is not None:
                print("User record {}: name: {}\n"
                      "\tfavorite_number: {}\n"
                      "\tfavorite_color: {}\n"
                      .format(msg.key(), user.name,
                              user.favorite_color,
                              user.favorite_number))
        except KeyboardInterrupt:
            break

    consumer.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="DeserializingConsumer Example")
    parser.add_argument('-b', dest="bootstrap_servers", required=True,
                        help="Bootstrap broker(s) (host[:port])")
    parser.add_argument('-s', dest="schema_registry", required=True,
                        help="Schema Registry (http(s)://host[:port]")
    parser.add_argument('-t', dest="topic", default="example_serde_protobuf",
                        help="Topic name")
    parser.add_argument('-g', dest="group", default="example_serde_protobuf",
                        help="Consumer group")

    main(parser.parse_args())


from django.conf.urls import url

from house_requests import views

app_name = 'house_requests'

urlpatterns = [
  url(r'^maintenance_list/$', views.MaintenanceRequestList.as_view(), name='maintenance-list'),
  url(r'^maintenance_json/$', views.MaintenanceRequestJSON.as_view(), name='maintenance-json'),
  url(r'^maintenance_request/$', views.MaintenanceRequestCreate.as_view(), name='maintenance-request'),
  url(r'^maintenance_update/(?P<pk>\d+)$', views.MaintenanceRequestUpdate.as_view(), name='maintenance-update'),
  url(r'^maintenance_detail/(?P<pk>\d+)$', views.MaintenanceRequestDetail.as_view(), name='maintenance-detail'),
  url(r'^maintenance_delete/(?P<pk>\d+)$', views.MaintenanceRequestDelete.as_view(), name='maintenance-delete'),
  url(r'^maintenance_modify/(?P<status>[CPF])/(?P<id>\d+)$', views.modify_maintenance_status, name='maintenance-modify-status'),
  url(r'^maintenance_tacomment/(?P<pk>\d+)$', views.MaintenanceRequestTAComment.as_view(), name='maintenance-tacomment'),
  url(r'^maintenance_report/$', views.MaintenanceReport, name='maintenance-report'),

  url(r'^linens_list/$', views.LinensRequestList.as_view(), name='linens-list'),
  url(r'^linens_json/$', views.LinensRequestJSON.as_view(), name='linens-json'),
  url(r'^linens_request/$', views.LinensRequestCreate.as_view(), name='linens-request'),
  url(r'^linens_update/(?P<pk>\d+)$', views.LinensRequestUpdate.as_view(), name='linens-update'),
  url(r'^linens_detail/(?P<pk>\d+)$', views.LinensRequestDetail.as_view(), name='linens-detail'),
  url(r'^linens_delete/(?P<pk>\d+)$', views.LinensRequestDelete.as_view(), name='linens-delete'),
  url(r'^linens_modify/(?P<status>[CPF])/(?P<id>\d+)$', views.modify_linens_status, name='linens-modify-status'),
  url(r'^linens_tacomment/(?P<pk>\d+)$', views.LinensRequestTAComment.as_view(), name='linens-tacomment'),

  url(r'^framing_list/$', views.FramingRequestList.as_view(), name='framing-list'),
  url(r'^framing_json/$', views.FramingRequestJSON.as_view(), name='framing-json'),
  url(r'^framing_request/$', views.FramingRequestCreate.as_view(), name='framing-request'),
  url(r'^framing_update/(?P<pk>\d+)$', views.FramingRequestUpdate.as_view(), name='framing-update'),
  url(r'^framing_detail/(?P<pk>\d+)$', views.FramingRequestDetail.as_view(), name='framing-detail'),
  url(r'^framing_delete/(?P<pk>\d+)$', views.FramingRequestDelete.as_view(), name='framing-delete'),
  url(r'^framing_modify/(?P<status>[CPF])/(?P<id>\d+)$', views.modify_framing_status, name='framing-modify-status'),
  url(r'^framing_tacomment/(?P<pk>\d+)$', views.FramingRequestTAComment.as_view(), name='framing-tacomment'),

  url(r'^house_requests/$', views.NewRequestPage, name='house-requests'),
]


import re

from django import template

register = template.Library()

REPO_REGEXP = re.compile(r'https://github\.com/(?P<name>\w+)/(?P<repo>\S+)/?')
REPO_SCHEMA = 'https://github.com/{name}/{repo}/'
WEBSITE_REGEXP = re.compile(
    r'http://(?P<name>\w+)\.github\.(io|com)/(?P<repo>\S+)/?'
)
WEBSITE_SCHEMA = 'http://{name}.github.io/{repo}/'

@register.assignment_tag
def format_link_to_repository(link):
    """
    Use: {% format_link_to_repository event.url as repo_link %}
    Then: {{ repo_link|urlize|default_if_none:"—" }}
    """
    if not link:
        return link
    elif REPO_REGEXP.match(link):
        return link
    else:
        try:
            repository = WEBSITE_REGEXP.findall(link)[0]
            return REPO_SCHEMA.format(name=repository[0], repo=repository[-1])
        except IndexError:
            return link

@register.assignment_tag
def format_link_to_website(link):
    """
    Use: {% format_link_to_website event.url as website_link %}
    Then: {{ website_link|urlize|default_if_none:"—" }}
    """
    if not link:
        return link
    elif WEBSITE_REGEXP.match(link):
        return link
    else:
        try:
            site = REPO_REGEXP.findall(link)[0]
            return WEBSITE_SCHEMA.format(name=site[0], repo=site[-1])
        except IndexError:
            return link


from __future__ import print_function
import numpy as np
import torch as t
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import random

#This AI model follows encoder - decoder architecture using Attention
#Encoder
#Encoder essentially encodes the input language embeddings into a GRU architecture and
#outputs the GRU output and hidden state data for each words in the input sentence.

class Trans_encoder(nn.Module):
    def __init__(self,input_size, hidden_size):
        super(Trans_encoder,self).__init__()
        self.hidden_size = hidden_size
        self.embedded = nn.Embedding(input_size,hidden_size)
        self.gru = nn.GRU(hidden_size,hidden_size)

    def forward(self, input, hidden):
        embedded = self.embedded(input).view(1,1,-1) #dim 1,1,hidden_size

        output, hidden = self.gru(embedded, hidden) #dim 1,1,hidden_size
        return output, hidden

#Decoder
#Decoder here follows a seq to seq architecture where output of each word that passes through an decoder is given as the input
#to the next step of decoder.
#Attention
#Attention scores are calculated to inform the model which words of the input sentence GRU outputs needs to be attended,
#while predicting its french word counterpart.

class Trans_attnDecoder(nn.Module):
    def __init__(self,hidden_size, output_size, drop_out, max_length ):
        super(Trans_attnDecoder,self).__init__()
        self.hidden_size =hidden_size
        self.output_size = output_size
        self.drop_out = drop_out
        self.max_length = max_length
        self.dropout = nn.Dropout(self.drop_out)
        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
        self.attn_connec = nn.Linear(self.hidden_size*2, self.max_length)
        self.attn_fc = nn.Linear(self.hidden_size*2, self.hidden_size)
        self.gru  = nn.GRU(self.hidden_size,self.hidden_size)
        self.decoder_fc = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, hidden, encoder_outs):
        embedded = self.embedding(input).view(1,1,-1) #dim 1,1,hidden_size
        embedded = self.dropout(embedded)
        attn_weights = F.softmax(self.attn_connec(t.cat((embedded[0],hidden[0]),1)),dim =1) #dim (1, max_length)
        attn_combined = t.bmm(attn_weights.unsqueeze(0),encoder_outs.unsqueeze(0))#dim 1, 1, hidden_size
        attn_apply = t.cat((embedded[0],attn_combined[0]),1)# dim  1, 512
        output = self.attn_fc(attn_apply).unsqueeze(0)#dim 1, 1, 256
        output = F.relu(output)
        gated_out, hidden = self.gru(output,hidden)#dim 1, 1, 256
        output = F.log_softmax(self.decoder_fc(gated_out[0]),dim=1) #dim 1, vocab_size_of_output_language
        return output, hidden, attn_weights

class Brain():
    def __init__(self,input_size, hidden_size, output_size, drop_out, max_length):
            self.max_length = max_length
            self.hidden_size = hidden_size
            self.teacher_forcing_ratio = 0.5
            self.EOS_token = 1
            self.SOS_token =0
            self.encoder = Trans_encoder(input_size,hidden_size)
            self.decoder = Trans_attnDecoder(hidden_size,output_size,drop_out,max_length)
            self.cost_fn = nn.NLLLoss()
            self.encoder_optim = optim.SGD(self.encoder.parameters(),lr=0.01)
            self.decoder_optim = optim.SGD(self.decoder.parameters(),lr=0.01)

#Preparing sequence from the dictionary.
    def prepare_sequence(self, seq, to_ix):
        idxs = [to_ix[w] for w in seq]
        return idxs

    # Running the training dataset in iteration as defined in the epochs
    def learniter(self, input_train,target_train, inlang_word2ix, outlang_word2ix,iterations, cal_loss_every):
        iter_count = 0
        iter_loss = 0
        for iter in range(iterations):
            sentence_loss = 0

            for i in range(len(input_train)):
                input_word2ix = self.prepare_sequence(input_train[i].split(), inlang_word2ix)
                input_word2ix = t.tensor(input_word2ix, dtype=t.long,)
                target_word2ix = self.prepare_sequence(target_train[i].split(), outlang_word2ix)
                target_word2ix = t.tensor(target_word2ix, dtype=t.long)
                hidden = t.zeros(1, 1, self.hidden_size, )

                loss = self.learn(input_word2ix, target_word2ix,hidden)
                sentence_loss += loss
            iter_loss += (sentence_loss/(len(input_train)))
            if iter - iter_count == cal_loss_every:
                print(sentence_loss)
                print(iter_loss/cal_loss_every)
                iter_count = iter
                iter_loss = 0

#Running the train function for single sentence
    def learn(self, input_ix_tensor, target_ix_tensor, hidden):
        self.encoder.zero_grad()
        self.decoder.zero_grad()
        len_input = input_ix_tensor.size()[0]
        len_target = target_ix_tensor.size()[0]
        encoder_outs = t.zeros(self.max_length, self.hidden_size)
        encoder_hidden = hidden
        loss =0
        for each_ix in range(len_input):
            encoder_out, encoder_hidden = self.encoder(input_ix_tensor[each_ix],encoder_hidden)
            encoder_outs[each_ix] = encoder_out[0,0]
        decoder_hidden = encoder_hidden
        decoder_input = t.tensor([[self.SOS_token]], dtype = t.long)
        #Teacher forcing is giving the actual output words in the training set as the input into the decoder instead of the words predicted by the model,
        #this helps the model to converge quickly than not using teaching forcing.
        #But this needs to be given in only for some random sentences or else the model will not learn effectively.

        use_teacher_forcing = True if random.random() < self.teacher_forcing_ratio else False
        if use_teacher_forcing:
            for n in range(len_target):
                decoder_output, decoder_hidden, decoder_attn = self.decoder(decoder_input,decoder_hidden,encoder_outs)
                target_out = target_ix_tensor[n]
                loss += self.cost_fn(decoder_output, target_out.unsqueeze(0))
                decoder_input = target_out

        else:
            for n in range(len_target):
                decoder_output, decoder_hidden, decoder_attn = self.decoder(decoder_input,decoder_hidden,encoder_outs)
                decoder_input = decoder_output.max(1)[1].detach()
                target_out = target_ix_tensor[n]
                loss += self.cost_fn(decoder_output,target_out.unsqueeze(0))
                if decoder_input.item() == self.EOS_token:
                    break
        loss.backward()
        self.encoder_optim.step()
        self.decoder_optim.step()

        return loss.item()/len_target
#This function is very similar to the training iteration function, except that this is used only on the testing data with no grad enabled.
    def test_samples(self,input_train,target_train, inlang_word2ix, outlang_word2ix, outix2word):
        decorder_words=[]

        decoder_attentions=[]
        decoder_inputs = []
        for i in range(len(input_train)):
           input_word2ix = self.prepare_sequence(input_train[i].split(), inlang_word2ix)
           input_word2ix = t.tensor(input_word2ix, dtype=t.long)
           target_word2ix = self.prepare_sequence(target_train[i].split(), outlang_word2ix)
           target_word2ix = t.tensor(target_word2ix, dtype=t.long)
           hidden = t.zeros(1, 1, self.hidden_size)
           decoder_input,decorder_word, decoder_attention = self.test_sample(input_word2ix,target_word2ix, hidden,outix2word)
           decorder_words.append(decorder_word)
           decoder_attentions.append(decoder_attention)
           decoder_inputs.append(decoder_input)
        return decoder_inputs,decorder_words, decoder_attentions

#Test sample function is used similar to the train function with no teaching forcing feature and with no_grad enabled.

    def test_sample(self,input_ix_tensor, target_ix_tensor, hidden, ix2word):
        with t.no_grad():
            len_input = input_ix_tensor.size(0)
            len_target = target_ix_tensor.size(0)
            encoder_outs = t.zeros(self.max_length, self.hidden_size)
            encoder_hidden = hidden
            for each_ix in range(len_input):
                encoder_out, encoder_hidden = self.encoder(input_ix_tensor[each_ix], encoder_hidden)
                encoder_outs[each_ix] = encoder_out
            decoder_hidden = encoder_hidden
            decoder_input = t.tensor([self.SOS_token])
            decoder_attentions = t.zeros(self.max_length, self.max_length)
            decoder_words=[]
            decoder_inputs = []

            for n in range(len_target):
                decoder_output, decoder_hidden, decoder_attn = self.decoder(decoder_input,decoder_hidden,encoder_outs)
                decoder_input = decoder_output.max(1)[1].detach()
                decoder_inputs.append(decoder_input)
                decoder_attentions[n] = decoder_attn
                if decoder_input.item() == self.EOS_token:
                    decoder_words.append('<EOS>')
                    break
                else:
                    decoder_words.append(ix2word[decoder_input.item()])

        return decoder_inputs,decoder_words, decoder_attentions
#this function  helps to save the model.
    def save(self,path):
        t.save(self.encoder.state_dict(), path+'\encoder.pth')
        t.save(self.decoder.state_dict(), path+'\decoder.pth')
        print('model saved')


###############################################################################
# idea adapted from:
# Deepak Pathak, Pulkit Agrawal, Alexei A. Efros, Trevor Darrell
# University of California, Berkeley
# Curiosity-driven Exploration by Self-supervised Prediction

# added by Paula
###############################################################################

import tensorflow as tf
import model_prediction_curiosity as mpc
import os
import numpy as np
from utils import Settings

class Curious(object):
    def __init__(self):
        tf.reset_default_graph()
        self.learning_rate = 0.001
        self.forward_loss_wt = 0.2
        self.feat_size = 200
        self.num_actions = 16
        self.num_belief_states = 268
        self.layer2 = 200

        if Settings.config.has_option("eval", "feat_size"):
            self.feat_size = Settings.config.getint("eval", "feat_size")

        with tf.variable_scope('curiosity', reuse=tf.AUTO_REUSE):
            self.predictor = mpc.StateActionPredictor(self.num_belief_states, self.num_actions,
                                                      feature_size=self.feat_size, layer2=self.layer2)

            self.predloss = self.predictor.invloss * (1 - self.forward_loss_wt) + \
                            self.predictor.forwardloss * self.forward_loss_wt

        self.optimizer = tf.train.AdamOptimizer(self.learning_rate)
        self.optimize = self.optimizer.minimize(self.predloss)
        # self.optimize = self.optimizer.minimize(self.predictor.forwardloss)  # when no feature encoding is used!
        self.cnt = 1

        self.sess2 = tf.Session()
        self.sess2.run(tf.global_variables_initializer())
        all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES)
        self.saver = tf.train.Saver(var_list=[v for v in all_variables if "Variab" not in v.name and "beta" not in v.name])

    def training(self, state_vec, prev_state_vec, action_1hot):
        _, predictionloss = self.sess2.run([self.optimize, self.predloss],
                                           feed_dict={self.predictor.s1: prev_state_vec,
                                           self.predictor.s2: state_vec,
                                           self.predictor.asample: action_1hot})
        return predictionloss

    def reward(self, s1, s2, asample):
        error = self.sess2.run(self.predictor.forwardloss,
                         {self.predictor.s1: [s1], self.predictor.s2: [s2], self.predictor.asample: [asample]})
        return error

    def inv_loss(self, s1, s2, asample):
        predloss, invloss = self.sess2.run([self.predloss, self.predictor.invloss],
                               {self.predictor.s1: [s1], self.predictor.s2: [s2], self.predictor.asample: [asample]})
        return predloss, invloss

    def predictedstate(self, s1, s2, asample):
        pred, orig = self.sess2.run([self.predictor.predstate, self.predictor.origstate],
                                    {self.predictor.s1: [s1], self.predictor.s2: [s2],
                                     self.predictor.asample: [asample]})
        return pred, orig

    def load_curiosity(self, load_filename):
        self.saver.restore(self.sess2, load_filename)
        print('Curiosity model has successfully loaded.')

    def save_ICM(self, save_filename):
        self.saver.save(self.sess2, save_filename)
        print('Curiosity model saved.')


'''
Written by Ankeet Tendulkar
'''

import nltk
import pandas as pd

trainFile = pd.read_csv('train.csv')

listOfFeatures = []
listOfLabels = []

def convertToBigrams(l):
    onlyTokens = []
    bigramList = []
    for i in range(0, len(l)):
        onlyTokens.append(l[i][0])
    for i in range(0, len(onlyTokens)):
        if(i == 0):
            bigramList.append("#_" + onlyTokens[i])
        else:
            bigramList.append(onlyTokens[i-1] + "_" + onlyTokens[i])
    return bigramList

for i in range(0,len(trainFile)):
    #print(trainFile["Tweet"][i])
    tweet = nltk.word_tokenize(trainFile["Tweet"][i])
    posTaggedTweet = nltk.pos_tag(tweet)
    #print(posTaggedTweet)
    featureDictionary = {}
    listOfBigrams = convertToBigrams(posTaggedTweet)
    for j in range(0,len(posTaggedTweet)):
        key = posTaggedTweet[j][0]
        #val = key
        val = posTaggedTweet[j][1]
        #val = listOfBigrams[j]
        featureDictionary[key] = val

    #print(featureDictionary)
    listOfFeatures.append(featureDictionary)
    listOfLabels.append(trainFile["Type"][i])
    #print(listOfLabels[i])

trainInput = []
for i in range(0,len(listOfFeatures)):
    trainInput.append((dict(listOfFeatures[i]),listOfLabels[i]))

print(len(listOfFeatures))
print(listOfFeatures[0])
print("Training Input Ready")

maxent = nltk.MaxentClassifier.train(trainInput)
print("Done Training")

testFile = pd.read_csv('test.csv')
outputFile = open("OutputMaxEntLen.csv",'w')
outputFile.write("Tweet,Type\n")

for i in range(0,len(testFile)):
    tweet = nltk.word_tokenize(testFile["Tweet"][i])
    posTaggedTweet = nltk.pos_tag(tweet)
    featureDictionary = {}
    listOfBigrams = convertToBigrams(posTaggedTweet)
    for j in range(0,len(posTaggedTweet)):
        key = posTaggedTweet[j][0]
        #val = key
        val = posTaggedTweet[j][1]
        #val = listOfBigrams[j]
        featureDictionary[key] = val
    label = maxent.classify(featureDictionary)
    outputFile.write(testFile["Tweet"][i].replace(",","") + "," + label + "\n")

outputFile.close()

print("Done Classifying Program")

checkFile = pd.read_csv('test.csv')
output = pd.read_csv('OutputMaxEntLen.csv')

#numberOfMismatch = 0

trueLabels = []
predictedLabels = []
labels = ["AbusiveHC","NonAbusiveHC","AbusiveDT","NonAbusiveDT"]

for i in range(0,len(checkFile)):
    trueLabels.append(checkFile["Type"][i])
    predictedLabels.append(output["Type"][i])

#print("Mismatches = {}".format(numberOfMismatch))
print("Total = {}".format(len(checkFile)))

from sklearn.metrics import classification_report
# getting a full report
print(classification_report(trueLabels, predictedLabels, target_names=labels))
print("Done Classifying....")


#!/usr/bin/env/python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import asyncio
import logging
import os
import pathlib
import shutil
from typing import Dict, List

from fbpcs.pcf import call_process
from fbpcs.pcf.errors import MPCRuntimeError, MPCStartupError
from fbpcs.pcf.games import (
    ConversionLift,
    ConverterLift,
    SecretShareConversionLift,
    SecretShareConverterLift,
)
from fbpcs.pcf.mpc.base import ServerClientMPCFramework
from fbpcs.pcf.structs import Game, Metric, Status

EMP_GAME_DIR = pathlib.Path(os.environ.get("EMP_GAME_DIR", os.getcwd()))
MAX_ROWS_PER_PARTITION = 1000000  # 1 million

class EmpMPCFramework(ServerClientMPCFramework):
    """
    Implementation of EMP SH2PC MPC Framework
    https://github.com/emp-toolkit/emp-sh2pc
    """

    SUPPORTED_GAMES: List[Game] = [
        ConversionLift,
        ConverterLift,
        SecretShareConversionLift,
        SecretShareConverterLift,
    ]

    async def prepare_input(self) -> Status:
        # We purposefully do not want to use the base class's prepare_input
        # method since it will sort the input which breaks the secret_share
        # game logic (since IDs won't appear to match).
        return Status.OK

    async def run_mpc(self) -> Dict[str, Dict[Metric, int]]:
        """
        Run the MPC game as the given player.
        """
        logger = logging.getLogger(
            f"EmpMPCFramework <Game:{self.game.name}> <{self.player.role!s}>"
        )

        game_path = EMP_GAME_DIR / self.game.base_game
        game_path_absolute = game_path.absolute()

        self._check_executable(game_path_absolute)

        if len(self.other_players) != 0:
            # pre_setup should have validated this, but we put another check
            # here just to reinforce the invariant.
            if len(self.other_players) != 1:
                raise ValueError(
                    f"Must be run with exactly one other player, not {len(self.other_players)}"
                )
            other_player = self.other_players[0]
            ip_address = other_player.ip_address
            port = other_player.port
        else:
            ip_address = self.player.ip_address
            port = self.player.port

        cmd = (
            f"{game_path_absolute} --role={self.player.id}"
            f" --data_directory={self.input_file.parent.absolute()}"
            f" --input_filename={self.input_file.name}"
            f" --server_ip={ip_address}"
            f" --port={port}"
            f" --output_filename={self.output_file}"
        )
        if self.output_s3_path:
            cmd = cmd + f" --output_s3_path={self.output_s3_path}"
        cmd = cmd.split(" ") + self.game.extra_args
        self.base_logger.debug(f"running command: {cmd}")

        try:
            operating_dir = pathlib.Path(os.getcwd())
            result = await asyncio.wait_for(
                call_process.run_command(cmd, operating_dir, logger=logger),
                timeout=self.run_timeout,
            )
        except Exception as e:
            # TODO: Should log e and raise an MPCRuntimeError instead
            raise e

        if result.returncode != 0:
            raise MPCRuntimeError(result.returncode)

        # At this point, assuming everything went correctly, we should have a
        # File with one result per line
        result_filepath = self.input_file.parent / self.output_file
        all_results: Dict[str, Dict[Metric, int]] = {}
        with open(result_filepath) as f:
            for line in f.readlines():
                if len(line) == 0:
                    # For some reason, we sometimes read an empty line from the
                    # output of the EMP MPC program in the result file.
                    continue
                parts = line.strip().split(",")
                feature_group = parts[0]
                contents = [int(field) for field in parts[1:]]
                all_results[feature_group] = {
                    metric: value
                    for metric, value in zip(self.game.output_metrics, contents)
                }
        return all_results

    def _check_executable(self, absolute_path: pathlib.Path) -> None:
        self.base_logger.debug(f"Checking {absolute_path} is executable.")

        if shutil.which(absolute_path) is None:
            raise MPCStartupError(f"Executable {absolute_path} not found.")

    def _check_file_exists(self, absolute_path: pathlib.Path) -> None:
        self.base_logger.debug(f"Checking {absolute_path} exists.")

        if not os.path.isfile(absolute_path):
            raise MPCStartupError(f"File {absolute_path} not found.")

    @staticmethod
    def get_max_rows_per_partition() -> int:
        return MAX_ROWS_PER_PARTITION


import sys

def hextodec(filename):
    file = open(filename)
    for line in file:
        line.strip('\n')
        for word in line.split():
            print(int(str(word), 16))
    file.close()

hextodec(sys.argv[1])


#! /usr/bin/python3
import sys
import math
import numpy as np

import explicit_runge_kutta3 as rk
import matplotlib.pyplot as plt

global k
k = 1.0

def func_h(t, xt):
    return np.array([xt[1], -k * xt[0]])

if __name__ == "__main__":

    if len(sys.argv) >= 2:
        Tau = float(sys.argv[1])
    else:
        Tau = 0.1

    if len(sys.argv) >= 3:
        k = float(sys.argv[2])

    if len(sys.argv) >= 4:
        t_final = math.pi * 2 * float(sys.argv[3])
    else:
        t_final = math.pi * 2

    times, vals = rk.approximation(func_h, \
                                   #T=[0.0, 2 * math.pi], \
                                   T=[0.0, t_final], \
                                   X0=np.array([1.0, 0]), \
                                   tau=Tau)

    appx = list(list(zip(*vals))[0])
    appy = list(list(zip(*vals))[1])
    plt.plot(appx, appy, label='Explicit RK 3 Approximation')

    n = round(t_final/0.01)
    sol_times = [t * 0.01 for t in range(n)]
    if not sol_times[-1] == t_final:
        sol_times.append(t_final)
    solx = [math.cos(math.sqrt(k) * t) for t in sol_times]
    soly = [-1 * math.sin(math.sqrt(k) * t) for t in sol_times]
    plt.plot(solx, soly, label='solution')
    plt.legend()

    plt.xlabel("X1 values")
    plt.ylabel("X2 values")
    plt.title("Explicit RK 3 Approximation for tau=" + str(Tau) + \
              ", k=" + str(k))
    plt.axis("equal")
    plt.show()


#coding:utf-8
#Created on 2018-12-13
#Author：Ethan Wang

import os
import requests
import re

def get_sort_list():
    response  =  requests.get('http://www.quanshuwang.com/list/1_1.html')
    response.encoding = 'gbk'
    result = response.text
    reg = r''
    novel_url_list = re.findall(reg, result)
    return novel_url_list

def get_novel_url(url):
    response = requests.get(url)
    response.encoding = 'gbk'
    result = response.text
    reg = ''
    novel_url = re.findall(reg, result)[0]

    response = requests.get(novel_url)
    response.encoding = 'gbk'
    result = response.text
    reg = r'
(.*?)
'
    chapter_url_list = re.findall(reg, result)
    return chapter_url_list

def get_chapter_conten(url):
    response = requests.get(url)
    response.encoding = 'gbk'
    result = response.text
    reg = r'style5\(\);</script>(.*?)


'''
Created on 4 abr. 2017

@author: Martin Rosas
'''
import logging

OK = 200
CREATED = 201
ERROR = 400
NOT_FOUND = 404
WRONG_PERMISSIONS = 412
INTERNAL_ERROR = 500
DELETED = 200

logger = logging.getLogger(__name__)

def simplest(action_name, status, payload=None, rejected_fields=None,
             callback=None, normal_mode=True):

    if action_name == "get_all" or action_name == "get":
        return __get_all_response(status, payload)
    elif action_name == "update":
        return __update_response(status, payload, rejected_fields)
    elif action_name == "create":
        return __create_response(status, payload, rejected_fields, normal_mode)
    elif action_name == "delete":
        return __delete_response(status)
    elif action_name == "custom" and callback is not None:
        return callback(action_name, status, payload, rejected_fields)

def __delete_response(status):
    if status == OK:
        return {"status": status}
    elif status == NOT_FOUND:
        return __error_NOT_FOUND_response("Entity not found")
    elif status == ERROR:
        return __error_ERROR_response("Unable to delete the "
                                      "entity")
    elif status == 500:
        return __error_500_response("We had an internal server error")

def __create_response(status, payload=None, rejected_fields=None, normal_mode=True):
    if status == CREATED:
        response = {
                    "rejected_fields": rejected_fields,
                    "status": status
                    }
        if normal_mode:
            response["id"] = payload
        else:
            response["list_id"] = payload
        return response
    elif status == ERROR:
        return __error_ERROR_response(payload, rejected_fields)
    elif status == 500:
        return __error_500_response("We had an internal server error",
                                    payload)

def __get_all_response(status, payload):
    if status == OK or status == NOT_FOUND:
        return {"payload": payload}
    elif status == ERROR:
        return __error_ERROR_response("Unable to obtain the "
                                      "requested entity data")

def __update_response(status, payload, rejected_fields):
    if status == OK:
        return {
                "payload": payload,
                "rejected_fields": rejected_fields,
                "description": "Entity updated properly",
                "status": status
                }
    elif status == ERROR:
        descriptive_message = "Entity could not be updated properly"
        return __error_ERROR_response(payload, descriptive_message)
    elif status == NOT_FOUND:
        descriptive_message = "Entity could not be found"
        return __error_NOT_FOUND_response(payload, descriptive_message)

def __error_ERROR_response(descriptive_message, payload=None):
    return {
            "payload": payload,
            "description": descriptive_message,
            "status": ERROR
            }

def __error_NOT_FOUND_response(descriptive_message, payload=None):
    return {
            "payload": payload,
            "description": descriptive_message,
            "status": NOT_FOUND
            }

def __error_500_response(descriptive_message, payload=None):
    return {
            "payload": payload,
            "description": descriptive_message,
            "status": 500
            }


"""An example of how to use your own dataset to train a classifier that
 recognizes people.
"""
import argparse
import os
import sys
import math
import pickle

import tensorflow as tf
import numpy as np
from sklearn.externals import joblib
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier

import facenet

def main(args):

    with tf.Graph().as_default():
        gpu_options = tf.GPUOptions(
            per_process_gpu_memory_fraction=args.gpu_memory_fraction
        )
        sess = tf.Session(config=tf.ConfigProto(
            gpu_options=gpu_options, log_device_placement=False)
        )
        with tf.Session() as sess:

            np.random.seed(seed=args.seed)
            print('Loading feature extraction model')
            facenet.load_model(args.model)

            # Get input and output tensors
            images_placeholder = \
                tf.get_default_graph().get_tensor_by_name("input:0")
            embeddings = \
                tf.get_default_graph().get_tensor_by_name("embeddings:0")
            phase_train_placeholder = \
                tf.get_default_graph().get_tensor_by_name("phase_train:0")
            # (?, 128)
            print(">>> Embedding size: ", embeddings.get_shape())
            # embedding_size = embeddings.get_shape()[1]

            test_image = facenet.load_test_data(
                args.path_image, False, False, args.image_size
            )
            feed_dict = {
                images_placeholder: test_image,  # ndarray
                phase_train_placeholder: False
            }
            emb_array = sess.run(
                embeddings, feed_dict=feed_dict
            )
            # emb_array = sess.run(
            #     [embeddings], feed_dict=feed_dict
            # )[0]

            print(emb_array)
            labels, class_names, emb_arrays = joblib.load(
                args.model_filename
            )
            # Classify images
            model = joblib.load(args.classifier_filename)
            # predictions = model.predict_proba(emb_array)
            emb_array = np.array(emb_array).reshape(1, -1)
            print(emb_array)
            if args.mode == 'SVM':
                predictions = model.predict_proba(emb_array)
                joblib.dump(predictions, 'models/predictions_svm.pkl')
                best_class_indices = np.argmax(predictions, axis=1)
                best_class_probabilities = predictions[np.arange(
                    len(best_class_indices)), best_class_indices]

                for i in range(len(best_class_indices)):
                    print('%4d  %s: %.3f' % (
                        i, class_names[best_class_indices[i]],
                        best_class_probabilities[i])
                    )
                max_ind = np.argmax(best_class_indices)
                print('%d  %s: %f' % (
                    max_ind, class_names[max_ind],
                    best_class_probabilities[max_ind])
                )
            elif args.mode == 'KNN':
                print("emb_array".upper(), emb_array.shape)
                predictions = model.predict(emb_array)
                joblib.dump(predictions, 'models/predictions_knn.pkl')
                print(predictions)
                max_ind = predictions[0]
                print('%d  %s' % (max_ind, class_names[max_ind]))

            # accuracy = np.mean(np.equal(best_class_indices, labels))
            # print('Accuracy: %.3f' % accuracy)

def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument(
        'mode', type=str, choices=['SVM', 'KNN'],
        help='Indicates if a new classifier should be trained or a classification model should be used for classification', default='SVM'  # noqa
    )
    parser.add_argument(
        'path_image', type=str,
        help='Path to test image'
    )
    parser.add_argument(
        'model', type=str,
        help='Could be either a directory containing the meta_file and ckpt_file or a model protobuf (.pb) file'  # noqa  # noqa
    )
    parser.add_argument(
        '--classifier_filename',
        help='Classifier model file name as a pickle (.pkl) file. For training this is the output and for classification this is an input.'  # noqa
    )
    parser.add_argument(
        '--model_filename',
        help='Classifier model file name as a pickle (.pkl) file. For training this is the output and for classification this is an input.'  # noqa
    )
    parser.add_argument(
        '--image_size', type=int,
        help='Image size (height, width) in pixels.', default=160
    )
    parser.add_argument(
        '--seed', type=int,
        help='Random seed.', default=42
    )
    parser.add_argument(
        '--gpu_memory_fraction', type=float,
        help='Upper bound on the amount of GPU memory that will be used by the process.', default=0.5  # noqa
    )

    return parser.parse_args(argv)

if __name__ == '__main__':
    main(parse_arguments(sys.argv[1:]))


# -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    This module Copyright (C) 2014 Therp BV (<http://therp.nl>).
#    All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
'''Extend model account.invoice'''
from openerp.osv import orm, fields

class AccountInvoice(orm.Model):
    '''Modify account invoice to add delivery address'''
    _inherit = 'account.invoice'

    def _modify_vals(self, cr, uid, vals, browse_obj=None, context=None):
        '''Utility function called on create and write to deliver consistent
        values. In this case fill delivery address when possible.'''
        if 'partner_shipping_id' in vals:
            return  # do not overwrite when explicitly filled
        partner_id = False
        if browse_obj:
            # Do nothing when invoice no longer in draft or sent state
            # or when shipping id already set:
            if (not browse_obj.state in ['draft', 'sent']
            or browse_obj.partner_shipping_id):
                return
            partner_id = (
                browse_obj.partner_id and browse_obj.partner_id.id or False)
        else:
            partner_id = vals.get('partner_id') or False
        if not partner_id:
            return
        # We have a partner, find delivery address
        partner_model = self.pool.get('res.partner')
        addr = partner_model.address_get(
            cr, uid, [partner_id], ['delivery'])
        vals['partner_shipping_id'] = addr['delivery']

    def create(self, cr, uid, vals, context=None):
        '''get delivery address, when not already in vals'''
        self._modify_vals(cr, uid, vals, browse_obj=None, context=context)
        return super(AccountInvoice, self).create(
            cr, uid, vals, context)

    def write(self, cr, uid, ids, vals, context=None):
        '''get delivery address, when not already in vals, or filled'''
        for item_id in ids:
            browse_records = self.browse(cr, uid, [item_id], context=context)
            browse_obj = browse_records[0]
            self._modify_vals(
                cr, uid, vals, browse_obj=browse_obj, context=context)
            super(AccountInvoice, self).write(
                cr, uid, [item_id], vals, context=context)
        return True

    _columns = {
        'partner_shipping_id': fields.many2one(
            'res.partner', 'Delivery Address',
            readonly=True,
            states={
                'draft': [('readonly', False)],
                'sent':  [('readonly', False)]
            },
            help="Delivery address for current invoice."),
    }


import os
import argparse
from encoder import DataEncoder
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
# import torch.backends.cudnn as cudnn
import torchvision
import torchvision.transforms as transforms

from loss import FocalLoss
from retinanet import RetinaNet
from datagen import CoCoDataset
import time
from torch.autograd import Variable
from utils import format_time
import lbtoolbox as lb
from signal import SIGINT, SIGTERM

parser = argparse.ArgumentParser(description='PyTorch RetinaNet Training')
parser.add_argument('--lr', default=0.0005, type=float, help='learning rate')
parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
parser.add_argument('--batch_size', default=2, type=int, help='the training batch size')
args = parser.parse_args()

# assert torch.cuda.is_available(), 'Error: CUDA not found!'
best_loss = float('inf')  # best test loss
start_epoch = 0  # start from epoch 0 or last epoch

# Data
print('==> Preparing data..')
transform = transforms.Compose([
    transforms.ToTensor(),
    # transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225))
])

trainset = CoCoDataset(root='./data/hologram',
                       json_file='./data/annotations/annotations_train.json',
                       train=True, transform=transform, input_size=1024)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=1, collate_fn=trainset.collate_fn)

testset = CoCoDataset(root='./data/hologram',
                      json_file='./data/annotations/annotations_val.json',
                      train=False, transform=transform, input_size=1024)
testloader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size,
                                         shuffle=False, num_workers=1, collate_fn=testset.collate_fn)

# Model
net = RetinaNet()
# net.load_state_dict(torch.load('./model/net.pth'))

# args.resume = 1
if args.resume:
    print('==> Resuming from checkpoint..')
    checkpoint = torch.load('./checkpoint/ckpt_interrupted.pth')
    net.load_state_dict(checkpoint['net'])
    best_loss = checkpoint['loss']
    start_epoch = checkpoint['epoch']
if torch.cuda.is_available():
    net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
    net.cuda()

criterion = FocalLoss()
# optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=1e-4)
optimizer = optim.Adam(net.parameters(),lr=args.lr)

# Training
def train(epoch):
    print('\nEpoch: %d' % epoch)
    print('==================================================================')
    net.train()
    iter_p_epoch = int(len(trainloader)/ args.batch_size)
    # net.module.freeze_bn()
    train_loss = 0

    with lb.Uninterrupt(sigs=[SIGINT, SIGTERM], verbose=True) as u:

        for batch_idx, (inputs, loc_targets, cls_targets) in enumerate(trainloader):
            print('Iter: %4d / %d ' % (batch_idx+1, iter_p_epoch))
            t1 = time.time()
            if torch.cuda.is_available():
                inputs = Variable(inputs.cuda())
                loc_targets = Variable(loc_targets.cuda())
                cls_targets = Variable(cls_targets.cuda())
            else:
                inputs = Variable(inputs)
                loc_targets = Variable(loc_targets)
                cls_targets = Variable(cls_targets)
            optimizer.zero_grad()
            loc_preds, cls_preds = net(inputs)
            loss = criterion(loc_preds, loc_targets, cls_preds, cls_targets)
            loss.backward()
            optimizer.step()

            train_loss += loss.data.item()
            t2 = time.time()
            time_elp = t2-t1
            total_time = time_elp*iter_p_epoch
            time_pass = (batch_idx+1)*time_elp
            print('train_loss: %.3f | avg_loss: %.3f' % (loss.data.item(), train_loss/(batch_idx+1)))
            print('Time : %s / %s' % (format_time(time_pass), format_time(total_time)))
            print('\n')

            if u.interrupted:
                print("Interrupted on request!")
                break

    print('Saving..')
    state = {
        'net': net.state_dict(),
        'loss': train_loss,
        'epoch': epoch,
    }
    if not os.path.isdir('checkpoint'):
        os.mkdir('checkpoint')
    torch.save(state, './checkpoint/ckpt_interrupted.pth')
    print('\nSavd')

# Test
def test(epoch):
    print('\nTest==================================================================')
    net.eval()
    test_loss = 0
    for batch_idx, (inputs, loc_targets, cls_targets) in enumerate(testloader):
        print('Batch: %3d/%3d' % (batch_idx, int(len(testloader)/args.batch_size)))
        if torch.cuda.is_available():
            inputs = Variable(inputs.cuda(), volatile=True)
            loc_targets = Variable(loc_targets.cuda())
            cls_targets = Variable(cls_targets.cuda())
        else:
            inputs = Variable(inputs, volatile=True)
            loc_targets = Variable(loc_targets)
            cls_targets = Variable(cls_targets)

        loc_preds, cls_preds = net(inputs)
        loss = criterion(loc_preds, loc_targets, cls_preds, cls_targets)
        test_loss += loss.item()
        print('test_loss: %.3f | avg_loss: %.3f' % (loss.item(), test_loss/(batch_idx+1)))

    # Save checkpoint
    global best_loss
    test_loss /= len(testloader)
    if test_loss < best_loss:
        print('Saving..')
        state = {
            'net': net.state_dict(),
            'loss': test_loss,
            'epoch': epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt.pth')
        best_loss = test_loss
        print('\nSavd')

def visualization():
    import numpy as np
    from PIL import Image,ImageDraw
    print('visulization\n')
    net.eval()
    test_loss = 0
    for batch_idx, (inputs, loc_targets, cls_targets) in enumerate(trainloader):
        # inputs = Variable(inputs.cuda(), volatile=True)
        # loc_targets = Variable(loc_targets.cuda())
        # cls_targets = Variable(cls_targets.cuda())
        print('Batch: %3d/%3d' % (batch_idx+1, int(len(testloader)/args.batch_size)))
        inputs = Variable(inputs,volatile=True)
        loc_targets = Variable(loc_targets)
        cls_targets = Variable(cls_targets)
        # loc_preds, cls_preds = net(inputs)
        cls_preds = torch.rand([loc_targets.shape[1],256], dtype=torch.float)
        encoder = DataEncoder()
        boxes, labels = encoder.decode(loc_targets.squeeze(), cls_preds, (1024, 1024))
        images = inputs.numpy()[0].transpose(1, 2, 0)
        images = images*255
        image = Image.fromarray(images.astype(np.uint8))
        draw = ImageDraw.Draw(image)
        for bbox in boxes.numpy():
            x,y,x_,y_ = bbox
            draw.rectangle([int(x), int(y), int(x_), int(y_)], width=2, outline='yellow')
        image.show()
        break

for epoch in range(start_epoch, start_epoch+15):
    # visualization()
    train(epoch)
    # test(epoch)