import matplotlib.pyplot as plt
import os
from scipy import optimize
from utils import stats as ss
from utils import fio
import numpy as np

plot = True
debug = False
ttfname = fio.tmpfile('distr-{}.pdf')
p2 = (.025,.975)
p3 = (.025,.5,.975)
p5 = (.025,.25,.5,.75,.975)

# FUNCTIONS ----------------------------------------------------------------------------------------

def pdfmerge():
  os.system(' && '.join([
    'pdftk {} cat output {}'.format(ttfname.format('*'),ttfname.format('all')),
    'mv {} pyplots.pdf'.format(ttfname.format('all')),
    'rm {}'.format(ttfname.format('*')),
  ]))

def section(name):
  print('-'*50+'\n'+name+'\n'+'-'*50)

def wfun(x,w):
  x,w = np.asarray(x),np.asarray(w)
  return np.sum(x*w)/np.sum(w)

def p2r(p,t=1):
  return -np.log(1-p)/t

def q_err(q,distrx,p=p2,pwr=2):
  dq = distrx.ppf(p)
  return np.sum(np.abs(q - dq)**pwr)

def print_distr(distr,x,name,m=None,q=None,p=p2,ok=True):
  xstr = {
    'betabin':     'p = {:.4f}, n = {:5.0f}',
    'gamma':       'm = {:.5f}, sd = {:.5f}',
    'lnorm':       'm = {:.5f}, sd = {:.5f}',
    'skewnorm':    'm = {:.5f}, sd = {:.5f}, a = {:.4f}',
    'invgauss':    'm = {:.5f}, sd = {:.5f}, z = {:.5f}',
    'uniform':     'l = {:.3f}, h = {:.3f}',
    'ratio_binom': 'p1 = {:.4f}, n1 = {:5.0f}, p2 = {:.4f}, n2 = {:5.0f}',
  }[distr.__name__].format(*x)
  qstr = '('+',~'.join(['{:.4f}']*len(p)).format(*distr(*x).ppf(p))+')'
  s = ' --- '.join((distr.__name__,xstr,qstr,name))
  print(s if ok else '(!) FAILED: '+s,flush=True)
  if plot: plot_distr(distr(*x),m,q,p,name)

def fit_distr(distr,m,q,w=1,p=p2,name='',n0=100,sd0=1,a0=0,z0=0,pwr=2):
  m  = wfun(m,w)
  q = [wfun(qi,w) for qi in q]
  dname = distr.__name__
  x0 = {
    'betabin': [m,n0],
    'gamma': [m,sd0],
    'lnorm': [m,sd0],
    'invgauss': [m,sd0,z0],
    'skewnorm': [m,sd0,a0],
  }[distr.__name__]
  xb = {
    'betabin': [(.001,.999),(5,1e5)],
    'gamma': None,
    'lnorm': None,
    'invgauss': [(-1e5,+1e5),(1e-9,1e5),(1e-9,100)],
    'skewnorm': [(-1e5,+1e5),(1e-9,1e5),(-100,100)],
  }[distr.__name__]

  jfun = lambda x: q_err(q,distr(*x),p=p,pwr=pwr)
  opt = optimize.minimize(jfun,x0,method='L-BFGS-B',bounds=xb,
    options=dict(ftol=1e-15,gtol=1e-12,eps=1e-9))
  ok = opt.success and np.sqrt(opt.fun) < (1e-2*m)
  print_distr(distr,opt.x,name+('' if ok else ' (!)'),m,q,p,ok)
  if debug and not ok: print(opt)
  return opt.x

def plot_distr(distrx,m=None,q=None,p=p2,name=''):
  dq = distrx.ppf(p); q = dq if q is None else q
  dm = distrx.mean(); m = dm if m is None else m
  xq = (min(*q,*dq),max(*q,*dq))
  x = np.linspace(1.5*xq[0]-.5*xq[1],1.5*xq[1]-.5*xq[0],256)
  d = distrx.pdf(x)
  y = np.quantile(d,.99)*1.01
  plt.figure(figsize=(5,3))
  plt.plot(x,d,color='#CC0033')
  for dk,k,pk in zip((dm,*dq),(m,*q),(False,*p)):
    plt.plot([ k, k],[0,y],'-',color='#0099CC')
    plt.plot([dk,dk],[0,y],':',color='#CC0033')
    if pk: plt.text(k,y*pk,' {:.4f} '.format(distrx.cdf(k)),ha='left',va='center')
  plt.ylim((0,y))
  plt.title(name)
  plt.savefig(ttfname.format(name))
  plt.close()

# PARAMETERS ---------------------------------------------------------------------------------------

def beta():
  section('beta')
  fit_distr(ss.gamma,m=.00072,q=(.0005,.00097),sd0=.0001,name='beta_0') # Boily2009
  fit_distr(ss.gamma,m=.0013,q=(.0005,.0025),sd0=.0005,name='beta_0_adj') # Boily2009
  fit_distr(ss.gamma,m=5.3,q=(1,5.3,15),p=p3,pwr=1,name='Rbeta_acute') # Bellan2015 (CI adj: 15 <- 56)
  fit_distr(ss.gamma,m=.1417,q=(.046,.1417,.5),p=p3,pwr=1,sd0=.1,name='dur_acute') # Bellan2015 - years (CI adj: )
  fit_distr(ss.gamma,m=1.6,q=(1.3,1.9),name='Rbeta_350') # Wawer2005, Boily2009, Donnell2010
  fit_distr(ss.gamma,m=8.3,q=(4.5,13),name='Rbeta_200') # Wawer2005, Boily2009, Donnell2010
  fit_distr(ss.gamma,m=1.45,q=(1,2),name='Rbeta_mtf') # assume
  fit_distr(ss.gamma,m=5.29,q=(1.43,5.29,15),p=p3,name='Rbeta_gud_sus') # Boily2009
  fit_distr(ss.gamma,m=2.9, q=(1.03,2.9,5.69),p=p3,name='Rbeta_gud_inf') # Gray2001
  fit_distr(ss.gamma,m=2.15,q=(0.2,6.0),name='aRbeta_gud_sus_adj') # Boily2009
  fit_distr(ss.gamma,m=0.95,q=(0.2,2.4),name='aRbeta_gud_inf_adj') # Gray2001
  fit_distr(ss.betabin,m=.3,q=(.2,.4),name='P_gud_fsw_l') # (JK) & 18/124 EswIBBS2022
  fit_distr(ss.gamma,m=1.3,q=(1,1.6),name='RP_gud_fsw_h:l') # (JK)
  # @FYI SDHS2006 Table 13.14: ~.07 GUD among wider pop P12M
  # @FYI SDHS2006 Table 13.14: ~.61 GUD among STI/symp/GUD (7% of 11.4%)
  # fit_distr(ss.betabin,m=.25,q=(.015,.65),name='Rbeta_uvls') # Donnell2010 (CI adj)
  return

def circumcision():
  section('circumcision')
  print_distr(ss.betabin,x=[.082,4156],name='circum_2006') # SDHS2006
  print_distr(ss.betabin,x=[.171,8329],name='circum_2011') # SHIMS1 Bicego2013
  print_distr(ss.betabin,x=[.300,3988],name='circum_2016') # SHIMS2
  fit_distr(ss.betabin,m=.7,q=(.5,.9),name='circum_2050') # assume
  return

def condoms():
  section('condoms')
  fit_distr(ss.betabin,m=1-.26,q=(1-.43,1-.13),name='Rbeta_condom') # Giannou2016
  # ai vs vi
  fit_distr(ss.betabin,m=.540,q=(.387,.692),name='PF_condom_ai)') # Owen2020a
  fit_distr(ss.betabin,m=.684,q=(.555,.813),name='PF_condom_vi)') # Owen2020a
  ru = .789; rlse = np.sqrt(1/22 - 1/40 + 1/33 - 1/48); ci = np.exp(np.log(ru)+[-rlse,+rlse])
  fit_distr(ss.betabin,m=.789,q=ci,name='RPF_condom_av') # Owen2020a
  fit_distr(ss.betabin,m=.80,q=(.50,.95),name='RPF_condom_av') # Owen2020a (adj) - TODO
  fit_distr(ss.betabin,m=.80,q=(.7,.9),name='RPF_condom_av (FSW,2011)') # FSW 2011
  fit_distr(ss.betabin,m=.55,q=(.4,.7),name='RPF_condom_av (FSW,2014)') # FSW 2014
  # sex work
  fit_distr(ss.betabin,m=.744*.5,q=(.744*.2,.744),name='PF_condom_swo_2002 (vi)') # SBSS2002
  fit_distr(ss.betabin,m=.600*.5,q=(.600*.2,.600),name='PF_condom_swr_2002 (vi)') # SBSS2002
  fit_distr(ss.betabin,m=.848,q=(.772*.75,.924),name='PF_condom_swo_2011 (vi)') # Baral2014
  fit_distr(ss.betabin,m=.829,q=(.758*.75,.900),name='PF_condom_swr_2011 (vi)') # Baral2014
  # fit_distr(ss.betabin,m=.511,q=(.418,.604), name='PF_condom_npp_2011 (vi)') # Baral2014 [omit]
  print_distr(ss.betabin,x=[.856,100],name='PF_condom_swo_2014') # EswKP2014 (n adj, was 620)
  print_distr(ss.betabin,x=[.885,100],name='PF_condom_swr_2014') # EswKP2014 (n adj, was 595)
  # print_distr(ss.betabin,x=[.806,100],name='PF_condom_npp_2014') # EswKP2014 (n adj, was 395) [omit]
  fit_distr(ss.betabin,m=.884,q=(.549,.95),name='PF_condom_swo_2014 (vi)') # EswKP2014
  fit_distr(ss.betabin,m=.853,q=(.479,.95),name='PF_condom_swr_2014 (vi)') # EswKP2014
  # fit_distr(ss.betabin,m=.801,q=(.647,.90),name='PF_condom_npp_2014 (vi)') # EswKP2014 [omit]
  print_distr(ss.betabin,x=[.500,100],name='PF_condom_swx_2021 (vi)') # EswIBBS2022 always
  print_distr(ss.betabin,x=[.455,11],name='PF_condom_swx_2021 (ai)') # EswIBBS2022 always
  # wider pop
  # fit_distr(ss.betabin,m=.006,q=(.004,.013),name='PF_condom_nsw_wom_1998') # SFHS1988 [omit]
  # fit_distr(ss.betabin,m=.073,q=(.059,.121),name='PF_condom_nsw_men_1998') # SFHS1988 [omit]
  fit_distr(ss.betabin,m=.040,q=(.004,.121),name='PF_condom_cas_1998') # SFHS1988 (JK)
  fit_distr(ss.betabin,m=.600,q=(.535,.660),name='PF_condom_cas_2006') # SDHS2006 Tables 14.7.1, 14.7.2
  fit_distr(ss.betabin,m=.693,q=(.649,.737),name='PF_condom_cas_2016') # SHIMS2 Tables Table 15.4.A, Table 15.4.B
  fit_distr(ss.betabin,m=.222,q=(.194,.269),name='PF_condom_msp_2006') # SDHS2006 Tables 14.7.1, 14.7.2
  fit_distr(ss.betabin,m=.414,q=(.308,.529),name='PF_condom_msp_2016') # SHIMS2 Tables Table 15.4.A, Table 15.4.B
  return

def PX():
  section('pop sizes')
  # FSW
  fit_distr(ss.betabin,m=.029, q=(.007,.065),name='PX_w_fsw_2011') # EswKP2014 Table 4
  fit_distr(ss.betabin,m=.0243,q=(.0117,.0502),name='PX_w_fsw_2021') # EswKP2014 Table 4
  fit_distr(ss.betabin,m=.164,q=(.096,.278),name='PX_w_h') # (JK)
  # fit_distr(ss.betabin,m=.252,q=(.150,.440),name='PX_m_h') # (JK) [omit]
  fit_distr(ss.betabin,m=.130,q=(.100,.170),name='PX_m_m') # (JK)
  # fit_distr(ss.betabin,m=.130,q=(.021,.385),name='PX_m_cli') # (JK) [omit]
  return

def CF():
  section('partnership numbers')
  # fit_distr(ss.betabin,m=.38,q=(.21,.57),name='C_msp_wl') # (JK) [omit]
  # fit_distr(ss.betabin,m=.32,q=(.09,.55),name='C_msp_wh') # (JK) [omit]
  # fit_distr(ss.betabin,m=.35,q=(.23,.50),name='C_msp_ml') # (JK) [omit]
  fit_distr(ss.betabin,m=.37,q=(.25,.50),name='C_msp_xl') # (JK) synthesis
  fit_distr(ss.betabin,m=.35,q=(.20,.55),name='C_cas_xl') # (JK) synthesis
  fit_distr(ss.gamma,m=1.5,q=(1.2,2),name='C_cas_wm') # assume
  print_distr(ss.uniform,x=[.25,1],name='RC_cas_cli:wm') # assume
  fit_distr(ss.gamma,m=8.4,q=(6,11),name='CF_swr_l') # (JK)
  print_distr(ss.gamma,x=[3.5,.70],name='C_swo_fsw_l') # (JK) Baral2014,EswKP2014
  print_distr(ss.gamma,x=[6.0,1.2],name='C_swr_fsw_l') # (JK) Baral2014,EswKP2014
  print_distr(ss.gamma,x=[14,2.8], name='C_swo_fsw_h') # (JK) Baral2014,EswKP2014
  print_distr(ss.gamma,x=[21,4.2], name='C_swr_fsw_h') # (JK) Baral2014,EswKP2014
  # fit_distr(ss.gamma,m=4.1,q=(2.5,6.0),name='C_swo_fsw_l') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=2.0,q=(1.6,2.5),name='RC_swo_fsw_h:l') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=8.4,q=(6.0,11.0),name='C_swr_fsw_l') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=8.4*5/6,q=(6.0*5/6,11.0*5/6),name='C_swr_fsw_l_adj') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=1.5,q=(1.3,1.7),name='RC_swr_fsw_h:l') # (JK) Baral2014,EswKP2014 [omit]
  fit_distr(ss.gamma,m=36,q=(18,72),name='KF_swx_cli') # assume
  fit_distr(ss.gamma,m=2.0,q=(1.6,2.5),name='RKF_swx_cli_h:l') # assume
  section('sex frequency')
  fit_distr(ss.gamma,m=52*1.5,q=(.5*52,3*52),name='F_msp') # Shisana2005,Delva2013
  print_distr(ss.uniform,x=[12,48],name='F_swr') # assume
  fit_distr(ss.gamma,m=.05,q=(.006,.165),sd0=.01,name='PF_ai_mcx') # Owen2017
  print_distr(ss.uniform,x=[1,2],name='RPF_ai_swx:mcx') # Owen2017
  # fit_distr(ss.gamma,m=.08,q=(.006,.292),sd0=.01,name='PF_ai_swx_old') # Owen2017 [omit]
  # fit_distr(ss.gamma,m=.08,q=(.024,.159),sd0=.01,name='PF_ai_swx') # Owen2020 [omit]
  return

def mixing():
  section('mixing')
  # fit_distr(ss.gamma,m=9,q=(2,19),name='pref_mcx_swx') # assume [omit]
  # fit_distr(ss.gamma,m=2,q=(1.5,3),name='pref_msp_xl') # (JK) [omit]
  print_distr(ss.uniform,x=[-2,+2],name='lpref_mxc_swx') # assume
  print_distr(ss.uniform,x=[-2,+2],name='lpref_msp_xl') # assume
  return

def dur():
  section('durations: risk groups')
  # fit_distr(ss.betabin,m=.383,q=(.275,.383,.491),p=p3,name='dur_fsw_0-2')  # Baral2014 [prelim]
  # fit_distr(ss.betabin,m=.321,q=(.236,.321,.407),p=p3,name='dur_fsw_3-5')  # Baral2014 [prelim]
  # fit_distr(ss.betabin,m=.202,q=(.132,.202,.271),p=p3,name='dur_fsw_6-10') # Baral2014 [prelim]
  # fit_distr(ss.betabin,m=.094,q=(.044,.094,.144),p=p3,name='dur_fsw_11+')  # Baral2014 [prelim]
  fit_distr(ss.gamma,m=4.06,q=(2.29,6.34),name='dur_fsw') # (JK) - Knight2024* bias
  fit_distr(ss.gamma,m=0.5,q=(1/6,1.00),name='dur_sw_h') # assume
  # fit_distr(ss.invgauss,m=2,q=(1.54,2,3.25),p=p3,name='Rdur_fsw_hl') # (JK)
  # fit_distr(ss.gamma,m=4.07,q=(2.96, 5.48),name='dur_fsw_lr') # (JK)
  # fit_distr(ss.gamma,m=9.33,q=(6.30,13.13),name='dur_fsw_hr') # (JK)
  # fit_distr(ss.betabin,m=.5,q=(.4,.6),name='Pturn_e_fsw') # assume
  # fit_distr(ss.betabin,.075,q=(.056,.075,.100),p=p3,name='ybs_ever_15-24') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.088,q=(.065,.088,.117),p=p3,name='ybs_ever_25-34') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.077,q=(.057,.077,.103),p=p3,name='ybs_ever_35-54') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.051,q=(.036,.051,.071),p=p3,name='ybs_p12m_15-24') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.039,q=(.027,.039,.056),p=p3,name='ybs_p12m_25-34') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.022,q=(.015,.022,.032),p=p3,name='ybs_p12m_35-54') # Hodgins2022 [prelim]
  fit_distr(ss.gamma,m=8,q=(4,15),name='dur_cli') # (JK)
  fit_distr(ss.gamma,m=0.5,q=(1/6,1.00),name='dur_cli_h') # assume
  fit_distr(ss.gamma,m=.2,q=(.05,.50),sd0=.1,name='turn_xm_xl') # assume
  fit_distr(ss.betabin,m=.75,q=(.50,.90),name='Pturn_fsw_m:l') # assume
  fit_distr(ss.betabin,m=.60,q=(.25,.90),name='Pturn_cli_m:l') # assume
  section('durations: partnership types')
  print_distr(ss.uniform,x=[14.5,18.5],name='dur_msp') # (JK)
  # print_distr(ss.uniform,x=[.01,.02],name='pcr_msp') # (JK) [omit]
  fit_distr(ss.gamma,m=.75,q=(.25,1.5),name='dur_cas') # (JK)
  fit_distr(ss.gamma,m=.5,q=(1/6,1),name='dur_swr') # (JK)
  return

def dx():
  section('HIV test p12m')
  # FSW
  fit_distr(ss.gamma,m=p2r(.617),q=(p2r(.556),p2r(.675)),name='dx_fsw_2011') # Baral2014
  print_distr(ss.gamma,[p2r(.617)/p2r(.468)-1,.2],name='aRdx_fsw:w_2011_adj') # (JK)
  print_distr(ss.betabin,[.746,100],name='test_fsw_2014') # EswKP2014 (adj n was 781)
  fit_distr(ss.gamma,m=p2r(.75),q=(p2r(.657),p2r(.826)),name='dx_fsw_2014_adj') # EswKP2014 (adj)
  print_distr(ss.gamma,[p2r(.746)/p2r(.571)-1,.2],name='aRdx_fsw:w_2016_adj') # (JK)
  # 2000 / 2002
  print_distr(ss.betabin,[.133,5642],name='test_ever_women_2000') # EswMICS2000
  fit_distr(ss.betabin,m=.1,q=(.05,.15),name='dx_w_2002') # assume
  # SDHS2006
  fit_distr(ss.betabin,m=p2r(.219),q=(p2r(.206),p2r(.233)),name='dx_w_2006') # SDHS2006
  fit_distr(ss.betabin,m=p2r(.089),q=(p2r(.078),p2r(.100)),name='dx_m_2006') # SDHS2006
  print_distr(ss.betabin,[p2r(.219),100],name='dx_w_2006_adj') # SDHS2006 (adj n)
  print_distr(ss.gamma,[p2r(.089)/p2r(.219),.1],name='Rdx_m:w_2006_adj') # SDHS2006 (adj n)
  # SHIMS
  print_distr(ss.betabin,[.5051,100],name='test_w_2011') # SHIMS1T (adj n was 9838)
  print_distr(ss.betabin,[.3167,100],name='test_m_2011') # SHIMS1T (adj n was 8318)
  print_distr(ss.betabin,[.468,100],name='test_w_2011_adj') # SHIMS1T (adj 15-17 & n was 9838)
  print_distr(ss.betabin,[.284,100],name='test_m_2011_adj') # SHIMS1T (adj 15-17 & n was 8318)
  fit_distr(ss.gamma,m=p2r(.468),q=(p2r(.372),p2r(.566)),sd0=.01,name='dx_w_2011_adj') # (JK)
  fit_distr(ss.gamma,m=p2r(.284),q=(p2r(.200),p2r(.376)),sd0=.01,name='dx_m_2011_adj') # (JK)
  print_distr(ss.gamma,[p2r(.284)/p2r(.468),.1],name='Rdx_m:w_2011_adj') # (JK)
  # 2016
  print_distr(ss.betabin,[.529,100],name='test_all_2016') # SHIMS2 (adj n was 9075) [omit]
  print_distr(ss.betabin,[p2r(.571),100],name='dx_w_2016') # SHIMS2 (adj n was 5127)
  print_distr(ss.betabin,[p2r(.478),100],name='dx_m_2016') # SHIMS2 (adj n was 3948)
  print_distr(ss.gamma,[p2r(.571)/p2r(.505)-1,.05],name='Rdx_w_16:11') # (JK)
  print_distr(ss.gamma,[p2r(.478)/p2r(.317)-1,.05],name='Rdx_m_16:11') # (JK) [omit]
  print_distr(ss.gamma,[p2r(.479)/p2r(.571),.1],name='Rdx_m:w_2016_adj') # (JK)
  return

def tx():
  section('ART initiation rate amond diagnosed')
  fit_distr(ss.gamma,m=1.5,q=(.5,3),name='tx_2010') # assume
  fit_distr(ss.gamma,m=9,q=(6,12),name='tx_2012') # assume
  fit_distr(ss.gamma,m=.75,q=(.5,1),name='revx_2010') # assume
  fit_distr(ss.gamma,m=.6,q=(.33,1),sd0=.1,name='ivx') # assume
  return

# TARGETS ------------------------------------------------------------------------------------------

def prevalence():
  section('prevalence')
  # prevalence ratios
  print_distr(ss.ratio_binom,x=[.605,127,.280,7747],p=p3,name='prev_fsw_w_2011') # Baral2014, Bicego2013
  fit_distr(ss.gamma,m=1.46,q=(1.30,1.63),name='prev_fsw_hl_2011') # Baral2014 (JK)
  fit_distr(ss.gamma,m=2.3, q=(1.92,2.75),name='prev_fsw_hl_2014') # EswKP2014 (JK) [omit] SR
  args = dict(p=p5,pwr=2,sd0=.1)
  fit_distr(ss.invgauss,m=1.979,q=(1.843,1.918,1.977,2.063,2.335),**args,name='prev_women_hl_2006_adj') # SDHS2006 (adj wp)
  fit_distr(ss.invgauss,m=2.571,q=(2.155,2.361,2.572,2.927,5.280),**args,name='prev_men_hl_2006_adj')   # SDHS2006 (adj wp)
  fit_distr(ss.invgauss,m=1.526,q=(1.467,1.501,1.526,1.558,1.660),**args,name='prev_women_hl_2011_adj') # Bicego2013 (adj wp & 15-17)
  fit_distr(ss.invgauss,m=1.236,q=(1.200,1.220,1.236,1.261,1.344),**args,name='prev_men_hl_2011_adj')   # Bicego2013 (adj wp & 15-17)
  fit_distr(ss.invgauss,m=1.410,q=(1.368,1.391,1.409,1.434,1.506),**args,name='prev_women_hl_2016_adj') # SHIMS2 (adj wp)
  fit_distr(ss.invgauss,m=1.305,q=(1.258,1.283,1.305,1.337,1.454),**args,name='prev_men_hl_2016_adj')   # SHIMS2 (adj wp)
  # FSW
  fit_distr(ss.betabin,m=.605,q=(.521,.690),name='prev_fsw_2011') # Baral2014 (RDS-adj)
  fit_distr(ss.betabin,m=.588,q=(.539,.636),name='prev_fsw_2021') # EswIBBS2022 (RDS-adj)
  # lr: 0-1 pp6m; nlr: 2+ p6m
  # SDHS2006 (Tables B.2, 14.3, 14.6, 14.7)
  fit_distr(ss.betabin,m=.259,q=(.244,.273),name='prev_all_2006')
  fit_distr(ss.betabin,m=.311,q=(.294,.329),name='prev_women_2006')
  fit_distr(ss.betabin,m=.197,q=(.179,.214),name='prev_men_2006')
  print_distr(ss.betabin,x=[wfun([.033,.298,.330],[1971,922,4696]),1971+922+4696],name='prev_all_lr_2006') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.052,.388,.358],[ 780,577,2989]), 780+577+2989],name='prev_women_lr_2006') # (unadj wp
  print_distr(ss.betabin,x=[wfun([.021,.149,.280],[1191,345,1707]),1191+345+1707],name='prev_men_lr_2006') # (unadj wp)
  fit_distr(ss.betabin,m=0.268,q=(.227,.287),name='prev_women_lr_2006_adj') # (adj wp) [omit->pr]
  fit_distr(ss.betabin,m=0.141,q=(.065,.167),name='prev_men_lr_2006_adj')   # (adj wp) [omit->pr]
  print_distr(ss.betabin,x=[wfun([.380,.414],[524,55]),524+55],name='prev_all_nlr_2006')   # [omit->pr]
  print_distr(ss.betabin,x=[wfun([.523,.645],[ 68, 4]), 68+ 4],name='prev_women_nlr_2006') # [omit->pr]
  print_distr(ss.betabin,x=[wfun([.359,.380],[456,50]),456+50],name='prev_men_nlr_2006')   # [omit->pr]
  # SHIMS1 Bicego2013 (Table 3) - adj n by fratio '06 & '16
  print_distr(ss.betabin,x=[.321,18172],name='prev_all_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.388, 9843],name='prev_women_2011')     # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.241, 8329],name='prev_men_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.280, 7747],name='prev_all_2011_adj')   # (adj n & 15-17)
  print_distr(ss.betabin,x=[.342, 6015],name='prev_women_2011_adj') # (adj n & 15-17)
  print_distr(ss.betabin,x=[.207, 4977],name='prev_men_2011_adj')   # (adj n & 15-17)
  print_distr(ss.betabin,x=[wfun([.198,.360],[4062,12083]),4062+12083],name='prev_all_lr_2011') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.338,.392],[1794,7618]),1794+7618],name='prev_women_lr_2011') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.087,.306],[2267,4466]),2267+4466],name='prev_men_lr_2011')   # (unadj wp)
  fit_distr(ss.betabin,m=.315,q=(.289,.328),name='prev_women_lr_2011_adj') # (adj wp & 15-17) [omit->pr]
  fit_distr(ss.betabin,m=.195,q=(.180,.201),name='prev_men_lr_2011_adj')   # (adj wp & 15-17) [omit->pr]
  print_distr(ss.betabin,x=[.333,1887],name='prev_all_nlr_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.545, 373],name='prev_women_nlr_2011')     # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.281,1515],name='prev_men_nlr_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.290, 792],name='prev_all_nlr_2011_adj')   # (adj n & 15-17) [omit->pr]
  print_distr(ss.betabin,x=[.481, 216],name='prev_women_nlr_2011_adj') # (adj n & 15-17) [omit->pr]
  print_distr(ss.betabin,x=[.241, 955],name='prev_men_nlr_2011_adj')   # (adj n & 15-17) [omit->pr]
  # SHIMS 2 (Table C.2, [6.3.B], 15.3.A)
  fit_distr(ss.betabin,m=.272,q=(.258,.287),name='prev_all_2016')
  fit_distr(ss.betabin,m=.343,q=(.326,.360),name='prev_women_2016')
  fit_distr(ss.betabin,m=.189,q=(.173,.204),name='prev_men_2016')
  print_distr(ss.betabin,x=[wfun([.324,.321],[1723,6164]),1723+6164],name='prev_all_lr_2016') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.361,.367],[1355,3848]),1355+3848],name='prev_women_lr_2016') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.226,.255],[ 368,2316]), 368+2316],name='prev_men_lr_2016') # (unadj wp)
  fit_distr(ss.betabin,m=.321,q=(.300,.331),name='prev_women_lr_2016_adj') # (adj wp) [omit->pr]
  fit_distr(ss.betabin,m=.175,q=(.157,.181),name='prev_men_lr_2016_adj')   # (adj wp) [omit->pr]
  print_distr(ss.betabin,x=[.287,914],name='prev_all_nlr_2016')   # [omit->pr]
  print_distr(ss.betabin,x=[.453,263],name='prev_women_nlr_2016') # [omit->pr]
  print_distr(ss.betabin,x=[.228,651],name='prev_men_nlr_2016')   # [omit->pr]
  # SHIMS 3 (Summary Sheet)
  fit_distr(ss.betabin,m=.237,q=(.226,.249),name='prev_all_2021')
  fit_distr(ss.betabin,m=.316,q=(.298,.334),name='prev_women_2021')
  fit_distr(ss.betabin,m=.156,q=(.143,.169),name='prev_men_2021')
  return

def incidence():
  section('incidence')
  # EswIBBS2022 (Table 13)
  q = ss.betabin(p=-np.log(1-30/676)/141*365,n=676).ppf(p3) * [141/160,1,141/119]
  fit_distr(ss.invgauss,m=q[1],q=q,p=p3,sd0=.1,name='inc_fsw_2021')
  # incidence ratios
  args = dict(p=(.05,.25,.50,.75,.95)) # p != p5
  fit_distr(ss.invgauss,m=5.891,q=(4.398,5.088,5.891,7.310,13.394),**args,name='inc_women_hl_2011_adj') # Bicego2013 (adj wp & 15-17)
  fit_distr(ss.invgauss,m=3.939,q=(2.908,3.395,3.939,5.055,10.864),**args,name='inc_men_hl_2011_adj')   # Bicego2013 (adj wp & 15-17)
  # SHIMS1 Justman2016 (Tables 2,1)
  fit_distr(ss.skewnorm,m=.031,q=(.026,.037),a0=2,name='inc_women_2011')        # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.017,q=(.013,.021),name='inc_men_2011')               # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.0294,q=(.0252,.0347),a0=2,name='inc_women_2011_adj') # (adj 15-17)
  fit_distr(ss.skewnorm,m=.0150,q=(.0116,.0184),name='inc_men_2011_adj')        # (adj 15-17)
  fit_distr(ss.skewnorm,m=[.011,.036],q=([.006,.030],[.022,.044]),w=[789,4135],a0= 3,name='inc_women_lr_2011') # (unadj wp & 15-17)
  fit_distr(ss.skewnorm,m=[.004,.020],q=([.001,.015],[.018,.027]),w=[909,2946],a0=10,name='inc_men_lr_2011')  # (unadj wp & 15-17)
  fit_distr(ss.skewnorm,m=.0163,q=(.0040,.0224),a0=-10,name='inc_women_lr_2011_adj')  # (adj wp & 15-17) [omit->pr]
  fit_distr(ss.skewnorm,m=.0084,q=(.0001,.0117),a0=-10,name='inc_men_lr_2011_adj')    # (adj wp & 15-17) [omit->pr]
  fit_distr(ss.skewnorm,m=.100,q=(.050,.192),a0=8,name='inc_women_nlr_2011')          # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.038,q=(.025,.056),a0=2,name='inc_men_nlr_2011')            # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.0948,q=(.0476,.1829),a0=8,name='inc_women_nlr_2011_adj')   # (adj 15-17) [omit->ir]
  fit_distr(ss.skewnorm,m=.0339,q=(.0221,.0494),a0=2,name='inc_men_nlr_2011_adj')     # (adj 15-17) [omit->ir]
  # SHIMS2 (Table C.1)
  fit_distr(ss.skewnorm,m=.0148,q=(.0096,.0199),name='inc_all_2016')
  fit_distr(ss.skewnorm,m=.0199,q=(.0116,.0280),name='inc_women_2016')
  fit_distr(ss.skewnorm,m=.0099,q=(.0039,.0159),name='inc_men_2016')
  # SHIMS 3 (Summary Sheet)
  fit_distr(ss.skewnorm,m=.0077,q=(.0039,.0115),name='inc_all_2021')
  fit_distr(ss.skewnorm,m=.0145,q=(.0069,.0220),name='inc_women_2021')
  fit_distr(ss.skewnorm,m=.0020,q=(.0005,.0048),a0=10,name='inc_men_2021')
  return

def diagnosed():
  section('diagnosed')
  # EswR2P2013 (Table 10)
  fit_distr(ss.betabin,m=.450,q=(.395,.506),name='diag_u_fsw_2011') # n.b. among FSW not FSW-LHIV + prev (simulate) -> below
  fit_distr(ss.skewnorm,m=.741,q=(.617,.741,.898),p=p3,pwr=1,a0=2,name='diag_fsw_2011')
  # EswIBBS2022 (Table 13)
  print_distr(ss.betabin,x=[363/411,411],name='diag_fsw_2021')
  # SHIMS1T (Table 6)
  print_distr(ss.betabin,x=[.626,5807],name='diag_all_2011')
  print_distr(ss.betabin,x=[.691,3810],name='diag_women_2011')
  print_distr(ss.betabin,x=[.501,1997],name='diag_men_2011')
  # SHIMS2 (Table C.7)
  fit_distr(ss.betabin,m=.861,q=(.847,.876),name='diag_all_2016')
  fit_distr(ss.betabin,m=.902,q=(.886,.918),name='diag_women_2016')
  fit_distr(ss.betabin,m=.773,q=(.740,.806),name='diag_men_2016')
  return

def treated():
  section('treated')
  # EswKP2014 (Table 10)
  fit_distr(ss.betabin,m=.369,q=(.301,.442),name='treat_c_fsw_2011')
  fit_distr(ss.skewnorm,m=.274,q=(.209,.274,.357),p=p3,pwr=1,a0=2,name='treat_u_fsw_2011')
  print_distr(ss.betabin,x=[354/363,363],name='treat_c_fsw_2021')
  print_distr(ss.betabin,x=[354/411,411],name='treat_u_fsw_2021')
  # SHIMS1T (Table 6)
  print_distr(ss.betabin,x=[.521,3635],name='treat_c_all_2011')
  print_distr(ss.betabin,x=[.480,2633],name='treat_c_women_2011')
  print_distr(ss.betabin,x=[.627,1002],name='treat_c_men_2011')
  print_distr(ss.betabin,x=[.319,5807],name='treat_u_all_2011')
  print_distr(ss.betabin,x=[.332,3810],name='treat_u_women_2011')
  print_distr(ss.betabin,x=[.314,1997],name='treat_u_men_2011')
  # SHIMS2 (Tables C.6 C.7)
  fit_distr(ss.betabin,m=.878,q=(.860,.896),name='treat_c_all_2016')
  fit_distr(ss.betabin,m=.875,q=(.854,.896),name='treat_c_women_2016')
  fit_distr(ss.betabin,m=.884,q=(.852,.916),name='treat_c_men_2016')
  fit_distr(ss.betabin,m=.756,q=(.736,.775),name='treat_u_all_2016')
  fit_distr(ss.betabin,m=.789,q=(.768,.811),name='treat_u_women_2016')
  fit_distr(ss.betabin,m=.683,q=(.647,.720),name='treat_u_men_2016')
  return

def vls():
  section('vls')
  # SHIMS2 (Tables C.6 C.7)
  fit_distr(ss.betabin,m=.903,q=(.890,.916),name='vls_c_all_2016')
  fit_distr(ss.betabin,m=.914,q=(.899,.928),name='vls_c_women_2016')
  fit_distr(ss.betabin,m=.876,q=(.844,.909),name='vls_c_men_2016')
  fit_distr(ss.betabin,m=.682,q=(.661,.704),name='vls_u_all_2016')
  fit_distr(ss.betabin,m=.721,q=(.697,.745),name='vls_u_women_2016')
  fit_distr(ss.betabin,m=.599,q=(.561,.637),name='vls_u_men_2016')
  # SHIMS 3 (Summary Sheet)
  fit_distr(ss.betabin,m=.866,q=(.850,.881),name='vls_u_all_2021')
  fit_distr(ss.betabin,m=.886,q=(.870,.902),name='vls_u_women_2021')
  fit_distr(ss.betabin,m=.824,q=(.793,.855),name='vls_u_men_2021')
  return

def cascade_ideal():
  section('cascade: 90-90-90 / 95-95-95')
  # assume
  fit_distr(ss.betabin,m=.90**1,q=(.99*.90**1,1.01*.90**1),name='90^1')
  fit_distr(ss.betabin,m=.90**2,q=(.99*.90**2,1.01*.90**2),name='90^2')
  fit_distr(ss.betabin,m=.90**3,q=(.99*.90**3,1.01*.90**3),name='90^3')
  fit_distr(ss.betabin,m=.95**1,q=(.99*.95**1,1.01*.95**1),name='95^1')
  fit_distr(ss.betabin,m=.95**2,q=(.99*.95**2,1.01*.95**2),name='95^2')
  fit_distr(ss.betabin,m=.95**3,q=(.99*.95**3,1.01*.95**3),name='95^3')
  return

# MAIN ---------------------------------------------------------------------------------------------

# beta()
# circumcision()
# condoms()
# PX()
# CF()
# mixing()
# dur()
# dx()
# tx()
# prevalence()
# incidence()
# diagnosed()
# treated()
# vls()
# cascade_ideal()

if plot: pdfmerge()


import sys
input = sys.stdin.readline

n = 10
graph = [list(map(int, input().split())) for _ in range(n)]
papers = [0,5,5,5,5,5] # 각 색종이 개수

# 색종이를 붙일 수 있는지 확인
def checkarr(x,y,size):
    for i in range(x,x+size):
        for j in range(y,y+size):
            if 0<=i<n and 0<=j<n and graph[i][j] == 1:
                pass
            else:
                return False
    return True

# 색종이를 붙이거나 떼기
def updatepaper(x,y,size,method):
    # 붙였을 경우 0으로 떼는 경우 1로
    for i in range(x, x+size):
        for j in range(y, y+size):
            graph[i][j] =  method

result = 25
# 완전탐색 dfs
def dfs(cnt):
    global result

    if cnt > result:
        return

    F = 0
    for i in range(n):
        for j in range(n):
            if graph[i][j] == 1:
                x,y = i,j
                F = 1
                break
        if F ==1 :
            break

    # 1이 더이상 없을 경우
    if F == 0:
        if cnt < result:
            result = cnt
        return

    for size in range(5,0,-1): # 5부터 1까지 색종이 size
        if papers[size]>0 and checkarr(x,y,size):
            updatepaper(x,y,size,0)
            papers[size] -= 1
            dfs(cnt+1)
            updatepaper(x,y,size,1)
            papers[size] += 1

dfs(0)
if result == 25:
    print(-1)
else:
    print(result)


from collections import deque

rows = open("input.txt", "r").read().split("\n")
players = deque([[int(row.split(" ")[-1]), 0] for row in rows]) # pos, score
rolls = 0
dice = deque(range(1,101))

while all([p[1] < 1000 for p in players]):
    to_move = sum([dice[i] for i in range(3)])
    dice.rotate(-3)
    rolls += 3

    players[0][0] = (players[0][0] + to_move) % 10 or 10
    players[0][1] += players[0][0]

    players.rotate(1)

res = min(players, key=lambda p: p[1])[1] * rolls
print(res)


class Button:
    def __init__(self,x,y,w,h,font,content,bg,func, arg):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.font = font
        self.content = content
        self.bg = bg
        self.func = func
        self.arg = arg

    def inRange(self,m):
        return (m[0] >= self.x and m[0] <= self.x + self.w and m[1] >= self.y and m[1] <= self.y + self.h)

    def blit(self,screen):
        screen.blit( self.bg, (self.x, self.y) )
        if not self.content == "":
            Text = self.font.render(self.content, True, (0,0,0))
            screen.blit( Text ,(self.x + (self.w - Text.get_width() ) / 2, self.y + (self.h - Text.get_height()) / 2) )


import datetime
import Core_Algorithm
import Distance
import Packages

# Name: Tanner Brake   Student ID: 581136
if __name__ == '__main__':
    # Initializing values and populating distance/package id comparison table
    Distance.compare_addresses()
    total_distance = 0
    start_time_truck_1_2 = datetime.datetime(2021, datetime.datetime.now().month, datetime.datetime.now().day, hour=8,
                                             minute=0)
    # Deliver truck 1 and 2. Also get end times / distance
    truck_1_end_time = Core_Algorithm.deliver_packages(Packages.first_truck, start_time_truck_1_2)[0]
    truck_2_end_time = Core_Algorithm.deliver_packages(Packages.second_truck, start_time_truck_1_2)[0]
    truck_1_distance = Core_Algorithm.deliver_packages(Packages.first_truck, start_time_truck_1_2)[1]
    truck_2_distance = Core_Algorithm.deliver_packages(Packages.second_truck, start_time_truck_1_2)[1]
    # Truck 3 start time
    start_time_truck_3 = datetime.datetime(2021, datetime.datetime.now().month, datetime.datetime.now().day, hour=10,
                                           minute=30)
    # Deliver truck 3 and get distance
    truck_3_distance = Core_Algorithm.deliver_packages(Packages.third_truck, start_time_truck_3)[1]
    # Sum total distance
    total_distance += truck_1_distance + truck_2_distance + truck_3_distance
    # Checking if all deadlines are met
    Packages.packages_delivered_on_time()

    # Command line interface
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def menu():
        print("Menu options - enter number below")
        print("Type 'close application' to stop application")
        print("1 - Search packages by any criteria")
        print("2 - Search packages status by time")
        print("3 - Show total distance traveled")
        user_input = input("Type Here: ")
        while user_input != "close application":
            if user_input == "1":
                while user_input != "menu" or user_input != "close application":
                    print("Type 'menu' to return to main menu")
                    print("Type 'close application' to stop application")
                    user_input = input("Search for packages: ")
                    print("\n")
                    Packages.search_Packages(user_input)
                    print("\n")
                    if user_input == "menu":
                        menu()
                    if user_input == "close application":
                        break
            if user_input == "2":
                while user_input != "menu" or user_input != "close application":
                    print("Enter time to check status of all packages")
                    print("Input based on 24 hr clock")
                    user_input_hour = input("hour = ")
                    user_input_minute = input("minutes = ")
                    user_input = input("Type 'menu' here to return to the main menu or just press enter to get results: ")
                    if user_input_hour != "" and user_input_minute != "":
                        Packages.get_all_package_status(int(user_input_hour), int(user_input_minute))
                    if user_input == "menu":
                        menu()
                    if user_input == "close application":
                        break
            if user_input == "3":
                while user_input != "menu" or user_input != "close application":
                    print("Select an option.")
                    print("Type 'menu' to return to main menu")
                    print("Type 'close application' to stop application")
                    print("1 - Truck 1 distance traveled")
                    print("2 - Truck 2 distance traveled")
                    print("3 - Truck 3 distance traveled")
                    print("4 - Combined distance of all trucks")
                    user_input = input("Type option here: ")
                    if user_input == "1":
                        print("\n", "Truck 1 distance: ", truck_1_distance, " miles", "\n")
                    if user_input == "2":
                        print("\n", "Truck 2 distance: ", truck_2_distance, " miles", "\n")
                    if user_input == "3":
                        print("\n", "Truck 3 distance: ", truck_3_distance, " miles", "\n")
                    if user_input == "4":
                        print("\n", "Total distance by all trucks: ", total_distance, " miles", "\n")
                    if user_input == "menu":
                        menu()
                    if user_input == "close application":
                        break
            else:
                menu()

    # Calling start menu
    menu()


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 26 14:18:21 2020

@author: aghangha
"""

import random
random.seed(0)

import pandas as pd
import numpy as np
import re
from pandas.api.types import is_string_dtype, is_numeric_dtype
import warnings
from pdb import set_trace
import torch
from torch import nn, optim, as_tensor
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F
from torch.nn.init import *
import sklearn
from sklearn_pandas import DataFrameMapper
from sklearn.preprocessing import LabelEncoder, StandardScaler

import matplotlib.pyplot as plt

#file1=open("output.txt","w")

lparams= ['Elevation','Drain_sqkm','Lattitude','Longitude','Two Year Flow','Water Surface Slope','Percent Impervious','Bathymetric Area']
list_param=['RASTERVALU','Drain_sqkm','LAT','LON','2yrFlow', 'Slope_nhd','per_imp','Area_corre']

#list_param=['RASTERVALU','Drain_sqkm','LAT','LON','2yrFlow', 'Slope_nhd','per_imp']
df=pd.read_csv('D:/ratingcurve/states/merge/nrmse_param_table_2yrflow.csv', converters={1:str,2:str} )
df['rmse_sign']=df['RMSE_sur_e']*df['nbias_sign']

label=[]
# for i in df.rmse_sign:
#     if 1.0 > abs(i) >= 0.0:
#         label.append(0.0)
#     elif 3.0 > abs(i) >= 1.0:
#         label.append(1.0)
#     elif 5.0 > abs(i) >= 3.0:
#         label.append(2.0)
#     elif 7.0 > abs(i) >= 5.0:
#         label.append(3.0)
#     else:
#         label.append(4.0)

for i in df.rmse_sign:
    if 2.0 > abs(i) >= 0.0:
        label.append(0.0)
    elif 4.0 > abs(i) >= 2.0:
        label.append(1.0)
    else:
        label.append(2.0)

df['rmse_label']=label

n_label=[]
for i in df.nrmse_sign:
    if -0.75 > i :
        n_label.append(0.0)
    elif -0.50> i >= -0.75:
        n_label.append(1.0)
    elif -0.25> i >= -0.50:
        n_label.append(2.0)
    elif -0.125 > i >= -0.50:
        n_label.append(3.0)
    elif 0.125 > i >= -0.125:
        n_label.append(4.0)
    elif 0.25 > i  >= 0.125:
        n_label.append(3.0)
    elif 0.375 > i >= 0.25:
        n_label.append(2.0)
    elif 0.50  >  i  >=0.375:
        n_label.append(2.0)
    elif 0.75 > i >=0.5:
        n_label.append(1.0)
    else :
        n_label.append(0.0)

df['nrmse_label']=n_label

#df1=df[df.Npoints>=10].copy()
##working on only npoints greater than 10.

df2=df[['rmse_label','RASTERVALU','Drain_sqkm','LAT','LON','2yrFlow', 'Slope_nhd','per_imp','Area_corre']]

#df2=df[['rmse_label','RASTERVALU','Drain_sqkm','LAT','LON','2yrFlow', 'Slope_nhd','per_imp']]

#split into train validate and test  with 70% train, 15% validate and 15% test
train, validate, test = np.split(df2.sample(frac=1),[int(0.7*len(df2)),int(0.85*len(df2))])

dep_var='rmse_label'

def scale_vars(df,mapper):
    warnings.filterwarnings('ignore',category=sklearn.exceptions.DataConversionWarning)
    if mapper is None:
        map_f = [([n],StandardScaler()) for n in df.columns if is_numeric_dtype(df[n])]
        mapper =DataFrameMapper(map_f).fit(df)

    df[mapper.transformed_names_] = mapper.transform(df)
    return mapper

def proc_df(df,cat_vars, cont_vars,y_fld=None,do_scale=False,mapper=None,na_dict=None):

    ignore_flds=[]
    skip_flds=[]

    dep_var=y_fld
    df=df[cat_vars + cont_vars + [dep_var]].copy()
    df[dep_var]=df[dep_var].astype(float)
    df=df.copy()
    ignored_flds=df.loc[:,ignore_flds]
    y=df[y_fld].values

    #deal with skip fields
    skip_flds += [y_fld]
    df.drop(skip_flds,axis=1,inplace=True)
    # initialise the na dictionary
    if na_dict is None : na_dict ={}
    else : na_dict =na_dict.copy()
    na_dict_initial =na_dict.copy()
    #fill missing
    for name, col in df.items():
        if is_numeric_dtype(col):
            if pd.isnull(col).sum():
                df[name +'_na']=pd.isnull(col)
                filler =col.median()
                df[name] - col.fillna(filler)
                na_dict[name] =filler
    # keep track of which entries are missing and possibly use them in the model
    if len(na_dict_initial.keys()) > 0:
        df.drop([a+'_na' for a in list(set(na_dict.keys())-set(na_dict_initial.keys()))], axis=1, inplace=True)
    #normalize
    if do_scale : mapper=scale_vars(df,mapper)
    res=[df,y,na_dict]
    # keep track of how things were normalized
    if do_scale : res = res + [mapper]
    return res

def custom_preproc(df, y_fld ,cont_vars, loc_vars=None,loc_min_max=None, do_scale=False,na_dict=None,log_vars=None,mapper=None):
    ignore_flds=[]
    skip_flds=[]

    dep_var=y_fld
    df=df[cont_vars + [dep_var]].copy()
    df[dep_var]=df[dep_var].astype(float)
    df=df.copy()
    ignored_flds=df.loc[:,ignore_flds]
    y=df[y_fld].values

    #deal with skip fields
    skip_flds += [y_fld]
    df.drop(skip_flds,axis=1,inplace=True)

    if na_dict is None : na_dict ={}
    else : na_dict =na_dict.copy()
    na_dict_initial =na_dict.copy()
    #fill missing
    for name, col in df.items():
        if is_numeric_dtype(col):
            if pd.isnull(col).sum():
                df[name +'_na']=pd.isnull(col)
                filler =col.median()
                df[name] - col.fillna(filler)
                na_dict[name] =filler
    # keep track of which entries are missing and possibly use them in the model
    if len(na_dict_initial.keys()) > 0:
        df.drop([a+'_na' for a in list(set(na_dict.keys())-set(na_dict_initial.keys()))], axis=1, inplace=True)

    #transformation

    df_log=df[log_vars].copy()
    df_log['RASTERVALU'][df_log['RASTERVALU']<0.0]=0.001
    df_log[df_log ==0.0]=0.001 # done so that we dont have any issue with log transformation.
    df_log=np.log(df_log) # log transforming log variables.
    if do_scale : mapper=scale_vars(df_log,mapper)
    i=0
    for x in loc_vars:
        df[x]=(df[x]-loc_min_max[i][0])/(loc_min_max[i][1]-loc_min_max[i][0])
        i+=1
    df['per_imp']=df['per_imp']/100
    df[log_vars]=df_log

    res=[df,y,na_dict]
    # keep track of how things were normalized
    if do_scale : res = res + [mapper]
    return res

# def custom_preproc(df, y_fld ,cont_vars, loc_vars=None,loc_min_max=None, do_scale=False,na_dict=None,log_vars=None,mapper=None):
#     ignore_flds=[]
#     skip_flds=[]

#     dep_var=y_fld
#     df=df[cont_vars + [dep_var]].copy()
#     df[dep_var]=df[dep_var].astype(float)
#     df=df.copy()
#     ignored_flds=df.loc[:,ignore_flds]
#     y=df[y_fld].values

#     #deal with skip fields
#     skip_flds += [y_fld]
#     df.drop(skip_flds,axis=1,inplace=True)

#     if na_dict is None : na_dict ={}
#     else : na_dict =na_dict.copy()
#     na_dict_initial =na_dict.copy()
#     #fill missing
#     for name, col in df.items():
#         if is_numeric_dtype(col):
#             if pd.isnull(col).sum():
#                 df[name +'_na']=pd.isnull(col)
#                 filler =col.median()
#                 df[name] - col.fillna(filler)
#                 na_dict[name] =filler
#     # keep track of which entries are missing and possibly use them in the model
#     if len(na_dict_initial.keys()) > 0:
#         df.drop([a+'_na' for a in list(set(na_dict.keys())-set(na_dict_initial.keys()))], axis=1, inplace=True)

#     #transformation

#     df_log=df[log_vars].copy()
#     df_log['RASTERVALU'][df_log['RASTERVALU']<0.0]=0.001
#     df_log['Area_corre'][df_log['Area_corre']==0.0]=np.nan
#     df_log['Slope_nhd'][df_log['Slope_nhd']==0.00001]=np.nan
#     df_log[df_log ==0.0]=0.001 # done so that we dont have any issue with log transformation.
#     df_log=np.log(df_log)
#     [sl_std,sl_mean]=[df_log.Slope_nhd.std(),df_log.Slope_nhd.mean()]
#     [area_std,area_mean]=[df_log.Area_corre.std(),df_log.Area_corre.mean()]
#     # log transforming log variables.
#     if do_scale : mapper=scale_vars(df_log,mapper)
#     df_log.Area_corre[df_log.Area_corre.isna()]=(np.log(0.001)-area_mean)/area_std
#     df_log.Slope_nhd[df_log.Slope_nhd.isna()]=(np.log(0.00001)-sl_mean)/sl_std

#     df[log_vars]=df_log

#     i=0
#     for x in loc_vars:
#         df[x]=(df[x]-loc_min_max[i][0])/(loc_min_max[i][1]-loc_min_max[i][0])
#         i+=1
#     df['per_imp']=df['per_imp']/100

#     res=[df,y,na_dict]
#     # keep track of how things were normalized
#     if do_scale : res = res + [mapper]
#     return res

class ColumnarDataset(Dataset):
    """Dataset class for column dataset.
    Args:
       cats (list of str): List of the name of columns contain
                           categorical variables.
       conts (list of str): List of the name of columns which
                           contain continuous variables.
       y (Tensor, optional): Target variables.
       is_reg (bool): If the task is regression, set ``True``,
                      otherwise (classification) ``False``.
       is_multi (bool): If the task is multi-label classification,
                        set ``True``.
    """
    def __init__(self, df, cat_flds, y, is_reg, is_multi):
        df_cat = df[cat_flds]
        df_cont = df.drop(cat_flds, axis=1)

        cats = [c.values for n,c in df_cat.items()]
        conts = [c.values for n,c in df_cont.items()]

        n = len(cats[0]) if cats else len(conts[0])
        self.cats = np.stack(cats, 1).astype(np.int64) if cats else np.zeros((n,1))
        self.conts = np.stack(conts, 1).astype(np.float32) if conts else np.zeros((n,1))
        self.y = np.zeros((n,1)) if y is None else y
        if is_reg: self.y =  self.y[:,None]
        self.is_reg = is_reg
        self.is_multi = is_multi
    def __len__(self):
        return len(self.y)
    def __getitem__(self, idx):
        return [self.cats[idx], self.conts[idx], self.y[idx]]

class MixedInputModel(nn.Module):
    """Model able to handle inputs consisting of both categorical and continuous variables.
    Args:
       emb_size (list of int): List of embedding size
       n_cont (int): Number of continuous variables in inputs
       emb_drop (float): Dropout applied to the output of embedding
       out_sz (int): Size of model's output.
       szs (list of int): List of hidden variables sizes
       drops (list of float): List of dropout applied to hidden
                              variables
       y_range (list of float): Min and max of `y`.
                                y_range[0] = min, y_range[1] = max.
       use_bn (bool): If use BatchNorm, set ``True``
       is_reg (bool): If regression, set ``True``
       is_multi (bool): If multi-label classification, set ``True``
    """
    def __init__(self, emb_size, n_cont, emb_drop, out_sz, szs,
                 drops, y_range=None, use_bn=False, is_reg=True,
                 is_multi=False):
        super().__init__()
        for i,(c,s) in enumerate(emb_size): assert c > 1, f"cardinality must be >=2, got emb_size[{i}]: ({c},{s})"
        if is_reg==False and is_multi==False: assert out_sz >= 2, "For classification with out_sz=1, use is_multi=True"
        self.embs = nn.ModuleList([nn.Embedding(c, s)
                                      for c,s in emb_size])
        for emb in self.embs: emb_init(emb)
        n_emb = sum(e.embedding_dim for e in self.embs)
        self.n_emb, self.n_cont=n_emb, n_cont

        szs = [n_emb+n_cont] + szs
        self.lins = nn.ModuleList([
            nn.Linear(szs[i], szs[i+1]) for i in range(len(szs)-1)])
        self.bns = nn.ModuleList([
            nn.BatchNorm1d(sz) for sz in szs[1:]])
        for o in self.lins: kaiming_normal_(o.weight.data)
        self.outp = nn.Linear(szs[-1], out_sz)
        kaiming_normal_(self.outp.weight.data)
        self.emb_drop = nn.Dropout(emb_drop)
        self.drops = nn.ModuleList([nn.Dropout(drop)
                                        for drop in drops])
        self.bn = nn.BatchNorm1d(n_cont)
        self.use_bn,self.y_range = use_bn,y_range
        self.is_reg = is_reg
        self.is_multi = is_multi
    def forward(self, x_cat, x_cont):
        if self.n_emb != 0:
            x = [e(x_cat[:,i]) for i,e in enumerate(self.embs)]
            x = torch.cat(x, 1)
            x = self.emb_drop(x)
        if self.n_cont != 0:
            x2 = self.bn(x_cont)
            # x = torch.cat([x, x2], 1) if self.n_emb != 0 else x2
            x= x2
        for l,d,b in zip(self.lins, self.drops, self.bns):
            x = F.relu(l(x))
            if self.use_bn: x = b(x)
            x = d(x)
        x = self.outp(x)
        if not self.is_reg:
            if self.is_multi:
                x = torch.sigmoid(x)
            else:
                x = F.log_softmax(x, dim=1)
        elif self.y_range:
            x = torch.sigmoid(x)
            x = x*(self.y_range[1] - self.y_range[0])
            x = x+self.y_range[0]
        return x

def emb_init(x):
    x = x.weight.data
    sc = 2/(x.size(1)+1)
    x.uniform_(-sc,sc)

#################

cat_vars=[]
cont_vars=list_param
emb_szs=[]

loc_vars=['LAT','LON']
loc_min_max=[[20,50],[-63,-131,]]
log_vars=['RASTERVALU','Drain_sqkm','2yrFlow','Slope_nhd','Area_corre'] # conus lattitude longitude min max [[Lat_min,Lat_max],[Lon_min.Lon_max]] # longitude coordinates with negative sign

#log_vars=['RASTERVALU','Drain_sqkm','2yrFlow','Slope_nhd']
########################

#df, y, nas, mapper = proc_df(train, cat_vars, cont_vars, dep_var,do_scale=True)
# df_val, y_val, nas, mapper = proc_df(validate, cat_vars, cont_vars, dep_var, do_scale=True, mapper=mapper, na_dict=nas)
# df_test, y_test, nas, mapper = proc_df(test, cat_vars, cont_vars, dep_var, do_scale=True)

df, y, nas, mapper = custom_preproc(train, dep_var, cont_vars,loc_vars=loc_vars, log_vars=log_vars, loc_min_max=loc_min_max ,do_scale=True)
df_val, y_val, nas, mapper = custom_preproc(validate, dep_var, cont_vars, loc_vars=loc_vars, log_vars=log_vars, loc_min_max=loc_min_max ,do_scale=True,mapper=mapper, na_dict=nas)
df_test, y_test, nas, mapper = custom_preproc(test, dep_var, cont_vars,loc_vars=loc_vars, log_vars=log_vars, loc_min_max=loc_min_max ,do_scale=True)

trn_ds = ColumnarDataset(df, cat_vars, y,is_reg=True,is_multi=False)
val_ds = ColumnarDataset(df_val, cat_vars, y_val,is_reg=True,is_multi=False)
test_ds = ColumnarDataset(df_test, cat_vars, y_test,is_reg=True,is_multi=False)

bs = 24
train_dl = DataLoader(trn_ds, bs, shuffle=True)
val_dl = DataLoader(val_ds, bs, shuffle=False)
test_dl = DataLoader(test_ds, len(df_test), shuffle=False)

# def train_model(model, train_dl, val_dl, n_epochs=10, lr=5e-2):
#         "Run training loops."
#         train_loss=[]
#         val_loss=[]
#         numb_epochs=[]
#         correct=0
#         total=0
#         epochs = n_epochs
#         opt = optim.SGD(model.parameters(), lr=lr)
#         loss_func = nn.MSELoss()
#         try:
#             for epoch in range(epochs):
#                 model.train()
#                 for xb1, xb2, yb in train_dl:
#                     preds = model(xb1, xb2)
#                     labels=yb.float()
#                     preds=preds[:,0]
#                     loss = loss_func(preds, labels)

#                     loss.backward()
#                     opt.step()
#                     opt.zero_grad()
#                     predicted=torch.round(preds).float()
#                     # for label, prediction in zip(labels,predicted):
#                     #     print(label,prediction)
#                     #     print(label.long(),prediction.long())
#                     #     set_trace()
#                     #     confusion_matrix[label.long()[0]][prediction.long()[0]] +=1
#                     #set_trace()
#                     total += labels.size(0)
#                     correct += (predicted==labels).sum().item()

#                 model.eval()
#                 with torch.no_grad():
#                     loss_train = sum(loss_func(model(xb1, xb2),
#                                              yb.float())
#                                    for xb1, xb2, yb in train_dl)
#                     loss_val = sum(loss_func(model(xv1, xv2),
#                                              yv.float())
#                                    for xv1, xv2, yv in val_dl)
#                 print('Validation loss:' + str(epoch), loss_val / len(val_dl))
#                 val_loss.append(loss_val/len(val_dl))
#                 numb_epochs.append(epoch)
#                 print('Training loss:' + str(epoch), loss_train /len(train_dl))
#                 train_loss.append(loss_train/len(train_dl))
#                 print('Training Accuracy : ' + str(correct*100/total))
#             plt.figure()
#             plt.plot(numb_epochs,val_loss,'b-',numb_epochs,train_loss,'r-')
#             plt.xlabel('Number of Epochs')
#             plt.ylabel('Loss')
#             plt.title('Validaiton vs Training Loss')
#         except Exception as e:
#             exception = e
#             raise

model = MixedInputModel(emb_szs,n_cont=len(df.columns)-len(cat_vars),
                        emb_drop = 0.04, out_sz = 3,
                        szs = [4000,2000], drops = [0.001,0.01],
                        y_range=(0,np.max(y)), use_bn=False,
                        is_reg=False, is_multi=True)

# train_model(model, train_dl, val_dl, n_epochs=50, lr=5e-4)

# def test_classifier(net):
#     net.eval()
#     loss_func = nn.MSELoss()
#     correct=0
#     total=0
#     confusion_matrix = torch.zeros(10,10)
#     with torch.no_grad():
#         for xt1,xt2,yt in test_dl:
#             predicted=net(xt1,xt2)
#             labels=yt.float()
#             predicted=torch.round(predicted).float()
#             for label, prediction in zip(labels,predicted):
#                 print(label,prediction)
#                 print(label.long(),prediction.long())
#                 #set_trace()
#                 confusion_matrix[label.long()[0]][prediction.long()[0]] +=1
#             total += labels.size(0)
#             correct += (predicted==labels).sum().item()
#         print(confusion_matrix)
#         print('accuracy : ' + str((correct/total)*100) )
#         #loss_test = sum(loss_func(model(xt1, xt2),yt.float()) for xt1, xt2, yt in test_dl)

def train_model_class(model, train_dl, val_dl, n_epochs=10, lr=5e-2, is_multi=False, n_classes=0, l_momentum=0.9):
        "Run training loops."
        train_loss=[]
        val_loss=[]
        numb_epochs=[]
        epochs = n_epochs
        opt = optim.SGD(model.parameters(), lr=lr, momentum=l_momentum)
        criterion = nn.CrossEntropyLoss()
        n_class=n_classes
        try:
            for epoch in range(epochs):
                model.train()

                running_loss=0.0
                correct=0
                total=0
                for xb1, xb2, yb in train_dl:
                    opt.zero_grad()
                    preds = model(xb1, xb2)
                    labels=yb.long()
                    labels=labels[:,0]    #look if your data needs this step. not necessary
                    loss = criterion(preds, labels)
                    loss.backward()
                    opt.step()
                    _, predicted = torch.max(preds.data,1)
                    # for label, prediction in zip(labels,predicted):
                    #     confusion_matrix[label][prediction] +=1
                    total += labels.size(0)
                    correct += (predicted==labels).sum().item()

                model.eval()
                with torch.no_grad():
                    loss_train = sum(criterion(model(xb1, xb2),
                                       yb.long()[:,0]) for xb1, xb2, yb in train_dl)
                    loss_val = sum(criterion(model(xv1, xv2),
                                             yv.long()[:,0])
                                   for xv1, xv2, yv in val_dl)
                print('Validation loss:' + str(epoch), loss_val / len(val_dl))
                val_loss.append(loss_val/len(val_dl))
                numb_epochs.append(epoch)
                print('Training loss:' + str(epoch), loss_train /len(train_dl))
                train_loss.append(loss_train/len(train_dl))
                print('Training Accuracy : ' + str(correct*100/total))

            plt.figure()
            plt.plot(numb_epochs,val_loss,'b-',numb_epochs,train_loss,'r-')
            plt.xlabel('Number of Epochs')
            plt.ylabel('Loss')
            plt.title('Validaiton vs Training Loss')
        except Exception as e:
            exception = e
            raise

def test_classifier_cross(net):
    net.eval()
    criterion = nn.CrossEntropyLoss()
    correct=0
    total=0
    confusion_matrix = torch.zeros(3,3)
    predicted_labels=[]
    with torch.no_grad():
        running_loss=0.0
        for xt1,xt2,yt in test_dl:
            predicted=net(xt1,xt2)
            labels=yt.long()
            labels=labels[:,0]
            loss = criterion(predicted, labels)
            running_loss += loss.item()
            _, predicted = torch.max(predicted.data,1)
            for label, prediction in zip(labels,predicted):
                confusion_matrix[label][prediction]+=1
            total += labels.size(0)
            correct += (predicted==labels).sum().item()
            predicted_labels.append(predicted)
        print(confusion_matrix)
        print('Testing Loss: ' + str(running_loss/len(test_dl)))
        print('accuracy : ' + str((correct/total)*100) )
    return predicted_labels

        #loss_test = sum(loss_func(model(xt1, xt2),yt.float()) for xt1, xt2, yt in test_dl)

train_model_class(model, train_dl, val_dl, n_epochs=300, lr=1e-4, n_classes=3)
print('Testing Loss : ' + str(loss_test/len(test_dl)))


import matplotlib.pyplot as plt
import pandas as pd

f1 = open("MendotaFinalTxt").readlines()
f2 = open("MononaFinalTxt").readlines()
year1 = []
days1 = []

for i in f1:
  row = i.strip().split()
  year1.append(int(row[0]))
  days1.append(int(row[1]))

year1_train = []
days1_train = []
year1_test = []
days1_test = []
for i in range(len(year1)):
  if year1[i] <= 1970:
    year1_train.append(year1[i])
    days1_train.append(days1[i])
  else:
    year1_test.append(year1[i])
    days1_test.append(days1[i])

year2 = []
days2 = []
for i in f2:
  row = i.strip().split()
  #year1.append(int(row[0]))
  #days1.append(int(row[1]))
  year2.append(int(row[0]))
  days2.append(int(row[1]))

year2_train = []
days2_train = []
year2_test = []
days2_test = []
for i in range(len(year2)):
  if year2[i] <= 1970:
    year2_train.append(year2[i])
    days2_train.append(days2[i])
  else:
    year2_test.append(year2[i])
    days2_test.append(days2[i])

plt.xlabel('year')
plt.ylabel('ice days')
plt.title('year vs ice days, Orange: Monona, Blue: Mendota')
plt.plot(year1,days1)
plt.plot(year2,days2)
#plt.scatter(x1,x2, 'o-', linestyle='dashed')
plt.show()

"""ymon_men = []
for i in range(len(days1)):
  ymon_men.append(days2[i]-days1[i])

#plt.figure(figsize=(5, 4))
plt.title('Year vs yMonona-yMendota')
plt.xlabel('Year')
plt.ylabel('ice days(yMonona-yMendota)')
#plt.plot(year1,norm_days1,'r-',label='Mendota')
plt.plot(year2,ymon_men)
plt.legend()
plt.show()"""


# 시험 점수 (5596번)
import sys
input = sys.stdin.readline

m1, m2, m3, m4 = map(int, input().rstrip().split())
n1, n2, n3, n4 = map(int, input().rstrip().split())

S = m1 + m2 + m3 + m4
T = n1 + n2 + n3 + n4
print(max(S, T))


import os
import sys
import time

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as functional
import torch.optim as optim

from datautils import PreTrainedResnet, Pixel

from dataloader import get_train_data

import torchvision
import torchvision.models as models

class CNN(nn.Module):

	def __init__(self):
		super(CNN, self).__init__()

		self.pretrained_model = models.resnet18(pretrained=True)
		if torch.cuda.is_available():
			self.pretrained_model = self.pretrained_model.cuda()
		for param in self.pretrained_model.parameters():
			param.requires_grad = True

		self.linear = nn.Linear(1000, 500)

	def forward(self, x):
		for name, module in self.pretrained_model._modules.items():
			if name=='fc':
				x = x.squeeze().contiguous()
			x = module(x)
			if name=='fc':
				lastlayer = x

		features = self.linear(lastlayer)
		return features

class SiemeseCNN(nn.Module):

	def __init__(self):
		super(SiemeseCNN, self).__init__()
		self.cnn = CNN()
		self.linear1 = nn.Linear(1000, 500)
		self.linear2 = nn.Linear(500, 2)

	def forward(self, x1, x2):
		feat1 = self.cnn(x1)
		feat2 = self.cnn(x2)
		return self.linear2(functional.ReLU(self.linear1(torch.cat((feat1, feat2), dim=-1))))

def train_cnn():

	modeltype = 'cnn'
	train_batch_size = 2
	sequence_length = 1
	directory_type = 'DPM'

	pixel = Pixel('pixel.pkl')

	train_dataloader, train_data_size, val_dataloader, val_data_size = \
		get_train_data(train_batch_size, sequence_length, directory_type, pixel)

	cnnmodel = CNN()
	if torch.cuda.is_available(): cnnmodel = cnnmodel.cuda()

	for i, batch in enumerate(train_dataloader):
		frame1 = Variable(torch.stack(batch[0])).squeeze(1)
		frame2 = Variable(torch.stack(batch[1]))
		labels = Variable(torch.LongTensor(batch[2]))

		print(frame1.size())
		print(frame2.size())
		print(labels)

		print(cnnmodel(frame1).size())
		break

if __name__=='__main__':
	train_cnn()


import unittest
import sqlite3
import os
import time
from database import Database

class DataaseCase(unittest.TestCase):
    filename = 'test_datebase.db'

    def setUp(self):
        try:
            os.remove(self.filename)
        except:
            pass
        db = Database(self.filename, new=True)

    def tearDown(self):
        os.remove(self.filename)

    def test_create(self):
        con = sqlite3.connect(self.filename)
        cur = con.execute("SELECT * from Word;")
        col_names = [description[0] for description in cur.description]
        self.assertEqual(col_names, ['word', 'rank', 'known_earlier', 'known_now', 'mod_time'])

        rows = cur.fetchall()
        self.assertEqual(rows, [])

    def test_modify_word(self):
        con = sqlite3.connect(self.filename)

        cur = con.execute("SELECT * from Word;")
        rows = cur.fetchall()
        self.assertEqual(rows, [])

        db = Database(self.filename)
        db.modify_word('x', 0, True, False)
        cur = con.execute("SELECT * from Word;")
        rows = [(a, b, c, d) for (a, b, c, d, _) in cur.fetchall()]
        self.assertEqual(rows, [('x', 0, True, False)])

        cur = con.execute("SELECT mod_time from Word WHERE word = 'x';")
        mod_time1 = cur.fetchall()[0][0]

        time.sleep(1)

        db.modify_word('x', 1, True, True)
        cur = con.execute("SELECT * from Word;")
        rows = [(a, b, c, d) for (a, b, c, d, _) in cur.fetchall()]
        self.assertEqual(rows, [('x', 1, True, True)])

        cur = con.execute("SELECT mod_time from Word WHERE word = 'x';")
        mod_time2 = cur.fetchall()[0][0]

        self.assertNotEqual(mod_time1, mod_time2)

    def test_known_earlier(self):
        db = Database(self.filename)
        db.modify_word('a', 0, known_earlier=True, known_now=False)
        db.modify_word('b', 0, known_earlier=False, known_now=False)
        self.assertEqual(db.known_earlier('a'), True)
        self.assertEqual(db.known_earlier('b'), False)
        self.assertEqual(db.known_earlier('c'), False)

    def test_known_now(self):
        db = Database(self.filename)
        db.modify_word('k', 0, known_earlier=True, known_now=False)
        db.modify_word('l', 0, known_earlier=False, known_now=True)
        self.assertEqual(db.known_now('k'), False)
        self.assertEqual(db.known_now('l'), True)
        self.assertEqual(db.known_now('m'), False)

    def test_known_words(self):
        db = Database(self.filename)
        db.modify_word('q', 1000, known_earlier=True, known_now=False)
        db.modify_word('w', 1001, known_earlier=False, known_now=True)
        db.modify_word('e', 1002, known_earlier=True, known_now=False)
        db.modify_word('r', 1003, known_earlier=False, known_now=True)
        db.modify_word('t', 1004, known_earlier=False, known_now=True)
        self.assertEqual(db.known_words(1000, 1000), None)
        self.assertEqual(db.known_words(1005, 1009), None)
        self.assertEqual(db.known_words(1000, 1001), 1/1)
        self.assertEqual(db.known_words(1000, 1002), 1/2)
        self.assertEqual(db.known_words(1000, 1005), 2/5)

if __name__ == '__main__':
    unittest.main()


import json

from django.http import HttpRequest, HttpResponse, JsonResponse
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt
from controller.Network.JSONProtocol import JSONTalker, JSONPacket
from django.views.decorators.http import require_POST
import config

def handlePacket(packet : JSONPacket):
    if packet.type == JSONPacket.PING:
        JSONTalker.sendToController(JSONPacket.response("pong", packet))

hostSession = None
talker = JSONTalker(config.WebServer.ADDR, handlePacket)

def index(request : HttpRequest):
    return render(request,"index.html")

@require_POST
@csrf_exempt
def trigger(request : HttpRequest, id : int):
    wasSent = JSONTalker.sendToController(
        JSONPacket.trigger(str(id),
        config.WebServer.NAME,
        config.Controller.NAME))
    print(request.body, wasSent)
    if not wasSent: return JsonResponse({"status":"failed", "reason": "NoConnection"})
    return JsonResponse({"status":"ok", "reason":""})

@csrf_exempt
def requestsf(request : HttpRequest): pass
    # try:
    #     data = json.loads(request.body)
    #     print(data)
    #     hostSession = request.session
    #     hostSession.set_expiry(0)
    #     if data['type'] == 'request':
    #         if 'question' in data['msg'].keys() and data['msg']['question'] == 'isAvailable':
    #             return JsonResponse(SmartMessage.respond('host', data['uid'], "web",{"isAvailable": "True"}).msg)
    #         if 'test' in data['msg'].keys() and data['msg']['test'] == 'test':
    #             return JsonResponse(SmartMessage.respond('host', data['uid'], "web",{"test": "hehehe"}).msg)
    #         if 'trigger' in data.keys():
    #             print(data)
    #     return
    # except:
    #     return JsonResponse({'response':'InvalidCommand'})


#!/usr/bin/env python
# coding: utf-8

# Data Analysis where i import the libraries and the datas

# In[98]:

import pandas as pd
import numpy as np
import plotly.express as px
import requests
import io
from datetime import date

# In[6]:

url = 'https://raw.githubusercontent.com/Marcellopp/Desarollo-de-applicaciones-mobiles/main/healthy_lifestyle_city_2021.csv?token=GHSAT0AAAAAABZYTLOLF5NFJYHJFPDWOMTUY3UWNFA'
download = requests.get(url).content

# Here i create a dataframe with the data captured

# In[78]:

df = pd.read_csv(io.StringIO(download.decode('utf-8')))
df.head()

# In[ ]:

# In[ ]:

# It's know that the more sunshine has a city the happier it is. So what is the Happiest city?

# In[70]:

df.iloc[df['Sunshine hours(City)'].argsort()[::-1][:1]]

# But is it true? which is the happiest city based on the data present in the database?

# In[82]:

df.iloc[df['Happiness levels(Country)'].argsort()[::-1][:1]]

# No, those are not the same cities, CAIRO has a Happiness value of 4.15 compared to HELSINKI that has a 7.8

# In[ ]:

# In[ ]:

# What are the 3 bigger cities based on number of takeouts

# In[68]:

df.iloc[df['Number of take out places(City)'].argsort()[::-1][:3]]

# I need to convert the type of the data to be able to use those in an eventual comparison so i also remove symbols

# In[79]:

df['Cost of a bottle of water(City)'] = df['Cost of a bottle of water(City)'].str.lstrip('£')
df['Obesity levels(Country)'] = df['Obesity levels(Country)'].str.rstrip('%')
df['Cost of a monthly gym membership(City)'] = df['Cost of a monthly gym membership(City)'].str.lstrip('£')

# In[80]:

df['Sunshine hours(City)'] = pd.to_numeric(df['Sunshine hours(City)'], errors = 'coerce')
df['Cost of a bottle of water(City)'] = pd.to_numeric(df['Cost of a bottle of water(City)'], errors = 'coerce')
df['Obesity levels(Country)'] = pd.to_numeric(df['Obesity levels(Country)'], errors = 'coerce')
df['Pollution(Index score) (City)'] = pd.to_numeric(df['Pollution(Index score) (City)'], errors = 'coerce')
df['Annual avg. hours worked'] = pd.to_numeric(df['Annual avg. hours worked'], errors = 'coerce')
df['Cost of a monthly gym membership(City)'] = pd.to_numeric(df['Cost of a monthly gym membership(City)'], errors = 'coerce')
print(df.dtypes)

# What city is more Healhy? Comparison between Pollution and Cost of gym membership

# In[83]:

df_new = df[['City','Pollution(Index score) (City)', 'Cost of a monthly gym membership(City)']]
df_new

# In[89]:

df_new.sort_values('Pollution(Index score) (City)')

# In[90]:

df_new.sort_values('Cost of a monthly gym membership(City)')

# We can see that the aren't strong relationships between the pollution and the cost of a gym membership also
# because this last one depends a lot on the economy of the city itself but we can notice that the most polluted
# countriees are also beneth the poorest and so the cost of gym membership is noticable lower...

# In[ ]:

# In[ ]:

# Relation between obesity and Life expectancy(years).

#

# In[95]:

df_nonan = df.dropna()

# In[101]:

df_nonan.sort_values('Obesity levels(Country)')
fig = px.pie(df_nonan, values='Obesity levels(Country)', names='City', title='Obesity percentage in cities')
fig.show()

# In[ ]:

# In[103]:

df_nonan.sort_values('Life expectancy(years) (Country)')
fig = px.pie(df_nonan, values='Life expectancy(years) (Country)', names='City', title='Life expectancy(years) (Country)')
fig.show()

# In[115]:

df_ob=df_nonan.iloc[df_nonan['Obesity levels(Country)'].argsort()[:10]]
df_ob

# In[ ]:

# In[113]:

df_life=df_nonan.iloc[df_nonan['Life expectancy(years) (Country)'].argsort()[::-1][:10]]
df_life

# We notice that there is a clear relation between the obesity level anda the life expectancy, for example we can see how 5 out of 10 of the cities with lover % of obesity are present in the top 10 cities with the higher life expectancy.

# In[ ]:

# In[ ]:

# In[116]:

df_sport = df[['City','Outdoor activities(City)', 'Cost of a monthly gym membership(City)']]
df_sport

# In[117]:

df_sport.sort_values('Cost of a monthly gym membership(City)')


class ListNode(object):
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution(object):
    def isPalindrome(self, head):
        """
        :type head: ListNode
        :rtype: bool
        """

        arr = []

        p = head
        while p:
            arr.append(p.val)
            p = p.next

        return True if arr == arr[::-1] else False

n = ListNode(1)
curr = n
curr.next = ListNode(2)
curr = curr.next
curr.next = ListNode(3)
curr = curr.next
curr.next = ListNode(1)
curr = curr.next

print(Solution().isPalindrome(n))


import requests
from pydojah.endpoints import EndPoints

class PyDojah:

    def __init__(self, app_id, secret_key, sandbox=False):
        self.app_id = app_id
        self.secret_key = secret_key
        self.sandbox = sandbox
        self.endpoint = EndPoints(sandbox=self.sandbox)

    '''Private Methods'''
    # Private method to create headers dictionary
    def __authentication_headers(self):
        headers = {'Authorization': self.secret_key, 'AppId': self.app_id}
        return headers

    # Private method to send a POST request
    def __post_data(self, url, data):
        headers = self.__authentication_headers()
        response = requests.post(url, headers=headers, data=data)
        return response.json()

    # Private method to send a GET request
    def __get_data(self, url, params=None):
        headers = self.__authentication_headers()

        response = requests.get(url, headers=headers, params=params)
        return response.json()

    '''Dojah Wallet Function'''
    # method to get Dojah wallet balance
    def get_balance(self):
        return self.__get_data(self.endpoint.wallet_balance_endpoint())

    '''Crypto Functions'''
    # method to get crypto wallet details
    def crypto_wallet_details(self, wallet_id):

        payload = {
            "wallet_id": wallet_id
        }
        return self.__get_data(self.endpoint.get_crypto_wallet_endpoint(), params=payload)

    # method to create crypto wallet
    def create_crypto_wallet(self, wallet_type):
        data = {"wallet_type": wallet_type}
        return self.__post_data(self.endpoint.crypto_wallet_endpoint(), data)

    # method for crypto transaction details
    def crypto_transaction_detail(self, transaction_id):
        payload = {
            "transaction_id": transaction_id
        }
        return self.__get_data(self.endpoint.transaction_details_endpoint(), params=payload)

    # method for sending Crypto to another Address of the same currency(Tested)
    def send_crypto(self, sender_wallet_id, amount, recipient_address):
        if amount < 0:
            raise ValueError('Amount cannot be a negative value')
        data = {
            "sender_wallet_id": sender_wallet_id,
            "amount": amount,
            "recipient_address": recipient_address
        }
        return self.__post_data(self.endpoint.send_crypto_endpoint(), data)

    def send_crypto_to_your_wallet(self, amount, recipient_wallet_id, sender_wallet_id):

        if amount < 0:
            raise ValueError('Amount cannot be a negative value')
        data = {
            "amount": amount,
            "recipient_wallet_id": recipient_wallet_id,
            "sender_wallet_id": sender_wallet_id
        }
        return self.__post_data(self.endpoint.send_crypto_internal_endpoint(), data)

    '''Data and Airtime functions'''
    # method to buy Airtime
    # To check for negative amount
    def airtime(self, amount, destination):

        if amount < 0:
            raise ValueError('Amount cannot be a negative value')
        data = {
            "amount": amount,
            "destination": destination
        }
        return self.__post_data(self.endpoint.airtime_endpoint(), data)

    # method to buy data
    def data(self, plan, destination):

        data = {
            "plan": plan,
            "destination": destination
        }

        return self.__post_data(self.endpoint.buy_data_endpoint(), data)

    # method to fetch all data plans available
    def data_plan(self):
        return self.__get_data(self.endpoint.data_plans_endpoint())

    '''OTP and Messaging'''

    def send_otp(self, sender_id, destination, channels, priority=True, expiry=None, length=None):
        data = {
            "expiry": expiry,
            "length": length,
            "sender_id": sender_id,
            "destination": destination,
            "channel": channels
        }
        return self.__post_data(self.endpoint.otp_endpoint(), data)

    # method for validating OTP
    def check_otp(self, code, reference_id):
        payload = {
            "code": code,
            "reference_id": reference_id
        }

        return self.__get_data(self.endpoint.validate_otp_endpoint(), payload)

    # method for sending whatsapp message or sms
    def send_sms_or_whatsapp(self, channel, message, destination, sender_id):
        data = {
            "channel": channel,
            "message": message,
            "destination": destination,
            "sender_id": sender_id
        }
        return self.__post_data(self.endpoint.messaging_endpoint(), data)

    # Method to confirm message status
    def get_message_status(self, message_id):
        payload = {
            "message_id": message_id
        }

        return self.__get_data(self.endpoint.message_status_endpoint(), payload)

    # method to request for sender id
    def request_for_sender_id(self, sender_id):
        data = {
            "sender_id": sender_id
        }
        return self.__post_data(self.endpoint.request_sender_id_endpoint(), data)

    # method for fetching the list of all your sender ID
    def get_sender_id(self):
        return self.__get_data(self.endpoint.fetch_sender_ids_endpoint())


import pandas as pd
import numpy as np
import simulation as sim
import data_preparation as dp
import random
import time

df = pd.read_csv('/Users/Egor/Downloads/diapason_1.csv')

df_p = dp.data_prep(df, '[10, 15, 20, 25]') #выбираем рассматриваемый диапазон

res_prob = dp.probabilities(df_p, df, [10, 15, 20, 25], 1000, 1500) #подсчитываем вероятности выбора набора, каждой из 4 кнопок набора и вероятности для моделирования сумм чека

# создание массива сумм чека с исходным распределением
check_sum = []
check_sum_type = np.random.choice([0,1,2,3,4,5,6,7,8,9], 5000, p = res_prob[2]) #5000 сумм чека создается для симуляции

for i in range(5000):
  if check_sum_type[i] == 0:
    check_sum.append(random.randrange(1000, 1050, 1))
  elif check_sum_type[i] == 1:
    check_sum.append(random.randrange(1050, 1100, 1))
  elif check_sum_type[i] == 2:
    check_sum.append(random.randrange(1100, 1150, 1))
  elif check_sum_type[i] == 3:
    check_sum.append(random.randrange(1150, 1200, 1))
  elif check_sum_type[i] == 4:
    check_sum.append(random.randrange(1200, 1250, 1))
  elif check_sum_type[i] == 5:
    check_sum.append(random.randrange(1250, 1300, 1))
  elif check_sum_type[i] == 6:
    check_sum.append(random.randrange(1300, 1350, 1))
  elif check_sum_type[i] == 7:
    check_sum.append(random.randrange(1350, 1400, 1))
  elif check_sum_type[i] == 8:
    check_sum.append(random.randrange(1400, 1450, 1))
  elif check_sum_type[i] == 9:
    check_sum.append(random.randrange(1450, 1500, 1))

df_0 = df_p[df_p['type_of_selected_button'] == 0]['amount'] #чаевые оставленные вручную

stats = sim.model_percent(check_sum, 10000, res_prob[0], res_prob[1], [10,15,20,25],df_0) #модели на вход подаются набор сумм чека, количество симуляций, вероятность выбора набора и ручного ввода, вероятность выбора каждой кнопки, рассматриваемый набор и набор чаевых, оставленных вручную

st.to_csv('/Users/Egor/Desktop/все/Результаты симуляций/diapason_1_10_15_20_25') #сохранение результатов симуляции


from lxml.doctestcompare import LXMLOutputChecker, PARSE_XML
from doctest import Example
import unittest
import os

def assertEqualXML(test, expected):
    output_checker = LXMLOutputChecker()
    if not output_checker.check_output(expected, test, PARSE_XML):
        diff = output_checker.output_difference(Example("", expected), test, PARSE_XML)
        msg = diff
        for line in diff.split("\n"):
            if msg == diff:
                msg = msg + "\nDiff summary:\n"
            if "got:" in line or line.strip().startswith(('+', '-')):
                msg = msg + line + "\n"
        if msg == "":
            msg = diff
        raise AssertionError(msg)

# decorators for selective tests
integration_test = unittest.skipIf(os.environ.get('SKIP_INTEGRATION_TESTS', default='False').lower() in ['1', 'true'],
                                   'Integration tests are disabled via env SKIP_INTEGRATION_TESTS')
unit_test = unittest.skipIf(os.environ.get('SKIP_UNIT_TESTS', default='False').lower() in ['1', 'true'],
                            'Unit tests are disabled via env SKIP_UNIT_TESTS')


# evaluation methods
def compare(target, current):
    '''

    :param target: true CPDAG
    :param current: learned CPDAG
    :return: a dictionary contains TP, FP, FN and f1
    '''

    compare_dict = {}
    tp = 0
    fp = 0
    fn = 0
    for key, value in target.items():
        for par in value['par']:
            if par in current[key]['par']:
                tp = tp + 1
            else:
                fn = fn + 1
        for nei in value['nei']:
            if nei in current[key]['nei']:
                tp = tp + 0.5
            else:
                fn = fn + 0.5
    for key, value in current.items():
        for par in value['par']:
            if par not in target[key]['par']:
                fp = fp + 1
        for nei in value['nei']:
            if nei not in target[key]['nei']:
                fp = fp + 0.5

    compare_dict['tp'] = tp
    compare_dict['fp'] = fp
    compare_dict['fn'] = fn
    compare_dict['f1'] = 2 * tp / (2 * tp + fp + fn)
    return compare_dict

def shd(target, current):
    '''

    :param target: true CPDAG
    :param current: learned CPDAG
    :return: the SHD score of learned graph
    '''
    shd = 0
    for key, value in target.items():
        for par in value['par']:
            if par not in current[key]['par']:
                shd = shd + 1
        for nei in value['nei']:
            if nei not in current[key]['nei']:
                shd = shd + 0.5
    for key, value in current.items():
        for par in value['par']:
            if (par not in target[key]['par']) and (par not in target[key]['nei']) and (key not in target[par]['par']):
                shd = shd + 1
        for nei in value['nei']:
            if (nei not in target[key]['nei']) and (nei not in target[key]['par']) and (key not in target[nei]['par']):
                shd = shd + 0.5
    return int(shd)


from django.core.management.base import BaseCommand
from stores.models import Store
from django.db.utils import IntegrityError
from olmonopolet.vmp_api import stores as vmp_api_stores
import httpx

class Command(BaseCommand):
    help = 'Update Vinmonopolet stores'

    def handle(self, *args, **options):
        # Used for logging
        new_stores = 0

        # Get list of all stores in database
        db_stores = list(Store.objects.values_list('store_id', flat=True))

        # Instantiate httpx Client
        with httpx.Client() as client:
            # Retrieve all Vinmonopolet stores
            all_stores = vmp_api_stores.get_all_stores(client)

        for store in all_stores:

            # Only add the store if it does not exist in the database
            if int(store["storeId"]) in db_stores:
                pass
            else:

                try:
                    new_obj = Store.objects.create(
                        store_id = store["storeId"],
                        name = store["storeName"],
                        category = store["category"],
                        city = store["address"]["city"],
                        street_address = store["address"]["street"],
                        postalcode = store["address"]["postalCode"],
                        latitude = store["address"]["gpsCoord"].split(';')[0],
                        longitude = store["address"]["gpsCoord"].split(';')[1],
                    )

                    new_stores += 1
                except Exception as err:
                    self.stdout.write(f"Could not insert store: {store['storeId']}")

        # Log how many new products were added to the database
        self.stdout.write(f"Completed and inserted {new_stores} new stores.")

        return


from flask import Flask
from config import Config
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_login import LoginManager
import logging
from logging.handlers import RotatingFileHandler, SMTPHandler
import os
from flask_mail import Mail
from flask_bootstrap import Bootstrap
from flask_moment import Moment

db = SQLAlchemy()
migrate = Migrate()
login = LoginManager()
login.localize_callback
login.login_view = 'auth.login'
mail = Mail()
bootstrap = Bootstrap()
moment = Moment()

def create_app(config_class=Config):
    app = Flask(__name__)
    app.config.from_object(config_class)

    db.init_app(app)
    migrate.init_app(app, db)
    login.init_app(app)
    mail.init_app(app)
    bootstrap.init_app(app)
    moment.init_app(app)

    from app.errors import bp as errors_bp
    from app.auth import bp as auth_bp
    from app.users import bp as users_bp
    from app.blogs import bp as blogs_bp
    from app.main import bp as main_bp
    # 将 blueprint 注册到 app 中
    # 此时所有的视图函数、HTML模板、错误处理器(error handlers)
    # 将与 app 关联
    app.register_blueprint(errors_bp)
    app.register_blueprint(main_bp)
    # 使用 url_prefix 参数可以添加前缀，此时应该用 /auth/login.html 访问登录页面
    app.register_blueprint(auth_bp, url_prefix='/auth')
    app.register_blueprint(users_bp, url_prefix='/users')
    app.register_blueprint(blogs_bp, url_prefix='/blogs')

    if app.config['ENABLE_SENDING_ERROR_MAILS']:
        mail_handler = SMTPHandler(
            mailhost=(app.config['MAIL_SERVER'], app.config['MAIL_PORT']),
            fromaddr=app.config['MAIL_USERNAME'],
            toaddrs=app.config['ADMINS'],
            subject='PAPERWAREHOUSE SERVER ERROR',
            credentials=(app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'])
        )
        # 设置日志的级别为ERROR
        mail_handler.setLevel(logging.ERROR)
        app.logger.addHandler(mail_handler)

    if app.config['ENABLE_LOGGING_TO_FILES']:

        if not os.path.exists('logs'):
            os.mkdir('logs')
        file_handler = RotatingFileHandler('logs/paperwarehouse.log', maxBytes=10240, backupCount=10)
        # setFormatter 可以设置自定义的log格式
        file_handler.setFormatter(logging.Formatter(
            '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'
        ))
        file_handler.setLevel(logging.INFO)
        app.logger.addHandler(file_handler)
        app.logger.setLevel(logging.INFO)
        app.logger.info('Paperwarehouse start up')

    return app

from app import models, localization


import logging
import time

import aioboto3

from .retriable_operations import RetriableKinesisProducer

log = logging.getLogger(__name__.split('.')[-2])

# Following constants are originating from here:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.put_records
MAX_RECORDS_IN_BATCH = 500
MAX_RECORD_SIZE = 1024 * 1024           # 1 Mb
MAX_BATCH_SIZE = 5 * MAX_RECORD_SIZE    # 5 Mb

def _get_default_partition_key():
    return '{0}{1}'.format(time.process_time(), time.time())

class AsyncKinesisProducer:

    def __init__(self, stream_name, ordered=True, custom_kinesis_client=None):

        self.stream_name = stream_name
        self.ordered = ordered

        self.seq = '0'

        self.record_buf = []
        self.buf_size = 0

        # Allow a custom kinesis client to be passed in. This allows for setting of any additional parameters in
        # the client without needing to track them in this library.
        if custom_kinesis_client is not None:
            client = custom_kinesis_client
        else:
            client = aioboto3.client('kinesis')

        self.kinesis_client = RetriableKinesisProducer(client=client)
        log.debug("Configured kinesis producer for stream '%s'; ordered=%s",
                  stream_name, ordered)

    async def put_record(self, record, partition_key=None, explicit_hash_key=None):
        """
        Put single record into Kinesis stream
        :param record:              record to put, bytes
        :param partition_key:       partition key to determine shard; if none, time-based key is used
        :param explicit_hash_key:   hash value used to determine the shard explicitly, overriding partition key
        :return:                    response from kinesis client, see boto3 doc
        """

        if partition_key is None:
            partition_key = _get_default_partition_key()

        kwargs = {
            'StreamName': self.stream_name,
            'Data': record,
            'PartitionKey': partition_key,
        }

        if self.ordered:
            kwargs['SequenceNumberForOrdering'] = self.seq

        kwargs['PartitionKey'] = partition_key or _get_default_partition_key()
        if explicit_hash_key:
            kwargs['ExplicitHashKey'] = explicit_hash_key

        resp = await self.kinesis_client.put_record(**kwargs)
        if self.ordered:
            self.seq = resp.get('SequenceNumber')
        return resp

    async def put_records(self, records):
        """
        Put list of records into Kinesis stream
        This call is buffered until it outgrow maximum allowed sizes (500 records or 5 Mb of data including partition
        keys) or until explicitly flushed (see flush() below)

        :param records:             iterable with records to put; has following structure:
                                    records=[
                                        {
                                            'Data': b'bytes',
                                            'ExplicitHashKey': 'string',
                                            'PartitionKey': 'string'
                                        },
                                    ],
                                    If no 'PartitionKey' given, default time-based key will be used
                                    See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.put_records
                                    for details
        :return:                    Empty list if no records were flushed, list of responses from kinesis client
                                    otherwise

                                    Raises ValueError if single record exceeds 1 Mb
                                    Currently application should check for ProvisionedThroughputExceededException
                                    in response structure itself.
        """
        resp = []
        n = 1
        for datum in records:

            if len(self.record_buf) == MAX_RECORDS_IN_BATCH:
                resp.append(await self.flush())

            data = datum.get('Data')
            if not (isinstance(data, bytes) or isinstance(data, bytearray)):
                raise TypeError('Record # {} is of type {}; accepted types are "bytes" and "bytearray"'.format(
                    n, type(data)
                ))
            record_size = len(data)

            # boto3 docs say that combined size of record and partition key should not exceed 1 MB,
            # while in reality, at least with boto3==1.9.49 and botocore==1.12.49, the key size
            # is not taken into account
            if record_size > MAX_RECORD_SIZE:
                raise ValueError('Record # {} exceeded max record size of {}; size={}; record={}'.format(
                    n, MAX_RECORD_SIZE, record_size, datum))

            if datum.get('PartitionKey') is None:
                datum['PartitionKey'] = _get_default_partition_key()

            # The same applies to batch size - only record size is counted
            if self.buf_size + record_size > MAX_BATCH_SIZE:
                resp.append(await self.flush())

            self.record_buf.append(datum)
            self.buf_size += record_size
            n += 1

        return resp

    async def flush(self):

        if len(self.record_buf) == 0:
            return

        resp = await self.kinesis_client.put_records(
            Records=self.record_buf,
            StreamName=self.stream_name
        )
        self.record_buf = []
        self.buf_size = 0
        return resp


# -*- coding: utf-8 -*-
"""Input text file of terms, output text file of term pair and frequency"""
def terms_to_term_pair_freq(if_name, of_name, min_freq=3, max_term_len=10):
    """
        description='generate term pairs from terms')
    Returns:
        (input_file_name, output_file_name, min_freq, max_term_len)

        min_freq: the minimum frequency of a term pair to be stored
    """
    # parser.add_argument('-m', '--min_freq',
    #                     help='minimum frequency (default: %(default)s)',
    #                     type=int, default=3)
    # parser.add_argument('-l', '--max_term_len',
    #                     help='maximum length of terms (default: %(default)s)',
    #                     type=int, default=10)

    import itertools
    import collections
    import logging

    def get_term_list(line, term_len):
        """Return a term list with the specific term length

        Args:
            line: input line of terms, should be '/term1/term2/...'
            term_len: desire length of the return terms

        Returns:
            list of terms of term_len
        """
        black_list = ('',)
        # don't know why terms str such as
        # /sylvester/speech/issue/letter/s/sweetie-pie//s//name
        # will be generated, so filter '' here

        return [term for term in line[1:-1].split('/')
                if term not in black_list and len(term) == term_len]
        # remove line[0]: '/' and line[last]: '\n'
        # python2 cht version used to use pat = u'[\u4e00-\u9fa5]'  一 to 龥
        # and remove term that (re.search(pat, y.decode('utf-8')) == None)
        # not sure if it's still needed in python3

    def get_term_lists(line, term1_len, term2_len):
        """Return a tuple of 2 term list with the specific term lengths

        Args:
            line: input line of terms, should be '/term1/term2/...'
            term1_len: desire length of the 1st return terms
            term2_len: desire length of the 2nd return terms

        Returns:
            tuple of 2 list of terms of term1_len, term2_len respectively
        """
        black_list = ('',)

        terms = [term for term in line[1:-1].split('/') if term not in black_list]

        term1s = [term for term in terms if len(term) == term1_len]
        term2s = [term for term in terms if len(term) == term2_len]

        return (term1s, term2s)

    def yield_term_pairs_from_term_lists(term1s, term2s):
        """Return generator of all combinations of (term1, term2)

        This function generates all the combinations of (term1 from term1s, term2..)

        Args:
            term1s: list of term1s
            term2s: list of term2s

        Returns:
            generator of term pairs (term1, term2)
        """
        for term1 in term1s:
            for term2 in term2s:
                yield (term1, term2)

    def yield_term_pairs_from_term_list(term_list):
        """Return generator of all combinations of tuple(term_pair)

        ['an', 'apple', 'a'] generates ['a', 'an'], ['a', 'apple'], ...
        Pairs will always be ['apple', 'doctor'], not ['doctor', 'apple'].

        Args:
            term_list: list of terms

        Returns:
            generator of term pairs
        """
        for term_pair in itertools.combinations(term_list, 2):
            if term_pair[0] == term_pair[1]:  # two terms are identical
                continue
            yield sorted(list(term_pair))

    def yield_term_pair_freq_str(if_name, min_freq, term1_len, term2_len):
        """Generate term pair frequency str

        Args:
            if_name: name of input file of '/term1/term2\n/t1/t2/t3\n....'
            min_freq: minimum frequency of a term pair to be stored
            term1_len: length of term1
            term2_len: length of term2
        """
        tp_f_d = collections.defaultdict(int)
        with open(if_name, 'r') as input_file:
            for line in input_file:
                if term1_len == term2_len:
                    term_len = term1_len
                    terms = get_term_list(line, term_len)
                    for t1, t2 in yield_term_pairs_from_term_list(terms):
                        tp_f_d['{t1} {t2}'.format(**locals())] += 1

                else:  # term1 len != term2 len
                    (term1s, term2s) = get_term_lists(line, term1_len, term2_len)
                    for t1, t2 in yield_term_pairs_from_term_lists(term1s, term2s):
                        tp_f_d['{t1} {t2}'.format(**locals())] += 1

        for tp_str, f in tp_f_d.items():
            if not f < min_freq:
                yield '{tp_str} {f}\n'.format(**locals())
    """Main function"""
    logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)

    logging.info('Begin of script')
    with open(of_name, 'w') as output_file:
        for term1_len in range(1, max_term_len + 1):
            for term2_len in range(term1_len, max_term_len + 1):
                # generate term_pair_freq_(term1_len)-(term2_len)?
                logging.info('begin term1 length: {term1_len}, '
                             'term2 length: {term2_len}'
                             '\n\n'.format(**locals()))

                for tp_f_str in yield_term_pair_freq_str(if_name, min_freq,
                                                         term1_len, term2_len):
                    output_file.write(tp_f_str)

                logging.info('term1 length: {term1_len}, '
                             'term2 length: {term2_len} finished\n\n'
                             .format(**locals()))

if __name__ == "__main__":
    terms_to_term_pair_freq(if_name, of_name)


#!/usr/bin/env python

import os
import setuptools

VERSION = '0.0.1.dev100'

INSTALL_REQUIRES = [
    'scikit-learn',
    ]

TEST_REQUIRES = [
    'coverage',
    ]

setuptools.setup(
    name = 'sklc',
    version = VERSION,
    description = 'sklearn for error correction',
    packages = ['sklc'],
    install_requires = INSTALL_REQUIRES,
    #extras_require = {
    #    'tests': TEST_REQUIRES,
    #    },
    #test_suite = 'tests',
    #test_loader = 'unittest:TestLoader'
    )


import os

from fastapi import FastAPI, Depends, HTTPException
from sqlalchemy.orm import Session
from starlette.responses import HTMLResponse, FileResponse
from starlette.staticfiles import StaticFiles

import models
import schemas
import crud
from database import SessionLocal, engine

if not os.path.exists('.\mysqldb'):
    os.makedirs('.\mysqldb')

models.Base.metadata.create_all(bind=engine)

app = FastAPI()

app.mount("/static", StaticFiles(directory="static"), name="static")

def get_db():
    db = SessionLocal()
    try:
        yield db
    finally:
        db.close()

@app.get("/", response_class=HTMLResponse)
def read_root():
    return FileResponse("static/index.html")

@app.post("/restaurant/", response_model=schemas.Restaurant)
def create_restaurant(restaurant: schemas.RestaurantCreate, db: Session = Depends(get_db)):
    return crud.create_restaurant(db=db, restaurant=restaurant)

@app.get("/restaurants/", response_model=list[schemas.Restaurant])
def read_restaurants(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
    restaurants = crud.get_restaurants(db, skip, limit)
    return restaurants

@app.get("/restaurants/{restaurant_id}", response_model=schemas.Restaurant)
def read_restaurants(restaurant_id: int, db: Session = Depends(get_db)):
    db_restaurant = crud.get_restaurant(db, restaurant_id=restaurant_id)

    if db_restaurant is None:
        raise HTTPException(status_code=404, detail="Restaurant not found")

    return db_restaurant

@app.post("/owner/", response_model=schemas.Owner)
def create_owner(owner: schemas.OwnerCreate, db: Session = Depends(get_db)):
    return crud.create_owner(db=db, owner=owner)

@app.get("/owners/", response_model=list[schemas.Owner])
def read_owners(skip: int = 0, limit: int = 100, db: Session = Depends(get_db)):
    owners = crud.get_owners(db, skip, limit)

    return owners

@app.post("/menuitem/", response_model=schemas.MenuItem)
def create_menu_item(menu_item: schemas.MenuItemCreate, db: Session = Depends(get_db)):
    return crud.create_menu_item(db=db, menu_item=menu_item)

@app.get("/menuitems/{restaurant_id}", response_model=list[schemas.MenuItem])
def read_menu_items(restaurant_id: int, db: Session = Depends(get_db)):
    menu_items = crud.get_menu_items(db, restaurant_id)

    if not menu_items:
        raise HTTPException(status_code=404, detail="Restaurant not found or no items on menu")

    return menu_items

@app.delete("/restaurants/{restaurant_id}/menuitems/{menu_item_id}", response_model=schemas.MenuItem)
def delete_menu_item(restaurant_id: int, menu_item_id: int, db: Session = Depends(get_db)):
    deleted_menu_item = crud.delete_menu_item(db, restaurant_id, menu_item_id)

    if deleted_menu_item is None:
        raise HTTPException(status_code=404, detail="Restaurant or menu not found")

    return deleted_menu_item


"""
Shows how to use urllib.request to do GET and POST requests
"""
import json
import sys

from http.client import HTTPResponse
from typing import Dict
from urllib import request

def create_request(url: str, headers: Dict[str, str]) -> request.Request:
    req = request.Request(url)
    for key, value in headers.items():
        req.add_header(key, value)

    return req

def http_get(url: str, headers: Dict[str, str]) -> HTTPResponse:
    req = create_request(url, headers)
    return request.urlopen(req)

def http_post(url: str, headers: Dict[str, str], payload: Dict[str, str]) -> HTTPResponse:
    req = create_request(url, headers)
    json_payload = json.dumps(payload).encode("utf-8")
    return request.urlopen(req, json_payload)

if __name__ == "__main__":
    if len(sys.argv) < 2:
        sys.exit(f"Usage: `urlopen.py GET` or `urlopen.py POST [<key1>=<value>...]`")
    method = sys.argv[1]
    headers = {"User-Agent": "urlopen.py"}
    if method == "GET":
        response = http_get("https://httpbin.org/get", headers)
    elif method == "POST":
        payload = dict(x.split("=", 1) for x in sys.argv[2:])
        headers["Content-Type"] = "application/json"
        response = http_post("https://httpbin.org/post", headers, payload)
    else:
        sys.exit(f"Invalid method {method}")

    print(f"{response.status} {response.reason}")
    print()
    print("Headers:")
    for key, value in response.getheaders():
        print(f"- {key}: {value}")
    print()
    print("Content:")
    content = response.read().decode("utf-8", errors="backslashreplace")
    print(content)