#!/usr/bin/env python

import sys,os,glob, inspect
#,re,numpy,math,pyfits,glob,shutil,glob
import optparse
import scipy as sp
import numpy as np
import pylab as pl
from scipy.interpolate import interp1d
from scipy import optimize
#from mpmath import polyroots
import time
import pprint, pickle

#from snclasses import myass#setupvars

bandscols = {'U':1,'u':1,'B':3,'V':5,'R':7,'I':9,'g':13, 'r':15, 'i':17,'z':11, 'w1':-1,'w2':-2,'m2':-3}
TEMPLDIR=os.path.realpath(os.path.abspath(os.path.split(inspect.getfile\
( inspect.currentframe() ))[0]))
MINX,MAXX=-10,40
NEW_X=np.linspace(MINX,MAXX,(MAXX-MINX)*10)

def smoothListGaussian(data,strippedXs=False,degree=15):
     window=degree*2-1
     weight=np.array([1.0]*window)
     weightGauss=[]
     for i in range(window):
         i=i-degree+1
         frac=i/float(window)
         gauss=1/(np.exp((4*(frac))**2))
         weightGauss.append(gauss)
     weight=np.array(weightGauss)*weight
     smoothed=np.zeros(len(data),float)
     for i in range(len(smoothed)-window):
          smoothed[i+window/2+1]=np.sum(np.array(data[i:i+window])*weight)/np.sum(weight)
     smoothed[0:window/2]=data[0:window/2]
     smoothed[-window/2:]=data[-window/2:]
     return np.array(smoothed)

class setupvars:
    def __init__(self):
        self.sntypes = ['Ib','Ic','IIb','Ic-bl','IIb-n']
        self.bands = ['U','u','B','V','g','R','I','r','i', 'J','H','K','w1','w2','m2']
        self.bandoffsets = {'U': 2, 'u': 2, 'B': 1, 'b' : 1,
                            'g' : 0, 'V': 0, 'R': -1, 'I': -2,
                            'r': -1, 'i': -2,
                            'J': -3, 'H': -4, 'K': -5, 'w1': 3, 'm2': 4, 'w2': 5}
        #cfa only bands
        self.bandsnonir =['U','u','B','V','R','I','r','i']
        self.bandsnir =['J','H','K']
        self.bandsindex ={'U':0,'u':0,'B':1,'V':2,'R':3,'I':4,'r':5,'i':6,
                          'J':7,'H':8,'K':9, 'w1':10,'w2':11,'m2':12}
        self.cbands = ['U-B','B-i',
                       'B-I','B-V',
                       'V-R','R-I','r-i','V-r', 'V-i','V-I',
                       'V-H', 'r-K','V-K','H-K',
                       'B-r', 'B-J', 'B-H', 'B-K', 'H-i', 'J-H', 'K-J']
        self.cs = {'U-B':0,'B-V':1,'V-R':2,'R-I':3,'r-i':4,'V-r':5, 'V-i':6,
                   'V-I':7, 'V-H':8,'I-H':9,'H-J':9,'J-K':10,
                   'B-I':11,'B-i':11,'r-K':12,'V-K':13,'H-K':14,
                   'B-r':15,'B-R':19, 'B-J':16, 'B-H':17, 'B-K':18}
        self.photcodes = {'U':('01','06','Ul'), 'B':('02','07','Bl'),
                          'V':('03','08','Vl'), 'R':('04','09','Rl'),
                          'I':('05','0a','Il'), 'r':('13','0b','rl'),
                          'i':('14','0c','il'), 'u':('15','15','ul'),
                          'H':('H','H','Hl'), 'J':('J','J','Jl'),
                          'K':('K','K','Kl'),
                          'm2':('m2','m2','m2'), 'w2':('w2','w2','w2'),
                          'w1':('w1','w1','w1'), 'g':('g','g','g')}

        self.mycolors = {'U':'k','u':'k','B':'#0066cc',
                         'g': '#47b56c', 'V':'#47b56c','R':'#b20000','I':'m',
                         'r':'#b20000','i':'m',
                         'J':'#4F088A','H':'#FFB700','K':'#A4A4A4',
                         'm2':'#708090', 'w2':'#a9b2bc', 'w1':'#434d56'}

        self.mycolorcolors = {'U-B':'k','B-V':'#0066cc','V-R':'#47b56c','R-I':'#b20000',
                              'V-I':'m','V-i':'m','V-r':'#47b56c','r-i':'#b20000',
                              'V-H':'#9999EE','I-H':'#9999EE','J-K':'#70B8FF','H-J':'#FFCC80',
                              'r-K':'purple', 'V-K':'SlateBlue', 'B-I':'#0B0B3B',
                              'H-K':'#FFCC80',
                              'B-i':'#0B0B3B','B-r':'#0B0B3B' ,'B-R':'#0B0B3B',
                              'w1':'k','w2':'k','m2':'k'}

        self.myshapes = {'U':'^','u':'^','B':'s','V':'o','g':'<',
                         'R':'v','I':'>','r':'d','i':'h',
                         'J':'^','H':'s','K':'o',
                         'w1':'v','w2':'v','m2':'v'}
        self.mytypecolors = {'Ib':'k','Ic':'b','IIb':'g','Ic-bl':'r','IIb-n':'y', 'other':'k'}
        self.mysymbols = {'Ib':'o','Ic':'s','IIb':'^','Ic-bl':'v','IIb-n':'>', 'other':'x'}
        self.mylines = {'Ib':'dashed','Ic':'solid','IIb':'solid','Ic-bl':'dotted',
                        'IIb-n':'solid', 'other':'solid'}
        self.instcodes = {'kepler':0,'shooter':0,'mini':1}
        self.insts =['shooter', 'kepler', 'mini']

        self.ebmvs={'83V':0.0178, '93J':0.0690,'94I':0.0302,'95F':0.0311,
                    '95bb':0.0948,'96cb':0.0262,'97X':0.0237,'97dq':0.0382,
                    '97ef':0.0366,'98dt':0.0219,'98fa':0.0382,'00H':0.1964,
                    '99dn':0.0451,'01ai':0.0081,'01ej':0.0460,'01gd':0.0098,
                    '02ap':0.0620,'02ji':0.0217,'03dh':0.0214,'03jd':0.0784,
                    '04aw':0.0180,'04ao':0.0893,'04dk':0.1357,'04dn':0.0415,
                    '04eu':0.0466,'04fe':0.0210,'04ff':0.0281,'04ge':0.0754,
                    '04gk':0.0247,'04gq':0.0627,'04gt':0.0398,'04gv':0.0286,
                    '05U':0.0143,'05ar':0.0394,'05az':0.0097,'05bf':0.0385,
                    '05da':0.2483,'05ek':0.1811,'05eo':0.0585,'05hg':0.0901,
                    '05kf':0.0378,'05kl':0.0219,'05kz':0.046,'05la':0.0100,
                    '05mf':0.0153,'05nb':0.0320,'06F':0.1635,'06T':0.0647,
                    '06aj':0.1267,'06ba':0.0452,'06bf':0.0216,'06cb':0.0094,
                    '06ck':0.0245,'06el':0.0973,'06ep':0.0310,'06fo':0.0250,
                    '06gi':0.0205,'06ir':0.0393,'06jc':0.0173,'06lc':0.0556,
                    '06ld':0.0144,'06lv':0.0245,'06ss':0.0178,'07C':0.0363,
                    '07D':0.2881,'07I':0.0250,'07ag':0.0250,'07aw':0.0338,
                    '07bg':0.0179,'07ce':0.0200,'07cl':0.0370,'07gr':0.0535,
                    '07hb':0.0518,'07iq':0.1182,'07ke':0.0954,'07kj':0.0691,
                    '07ru':0.2254,'07rz':0.1723,'07uy':0.0194,'08D':0.0194,
                    '08an':0.0450,'08aq':0.0383,'08ax':0.0186,'08bo':0.0513,
                    '08cw':0.0060,'08hh':0.0427,'09bb':0.0844,'09er':0.0389,
                    '09iz':0.0729,'09jf':0.0971,'09mg':0.0388,'09K':0.0491,
                    '03bg':0.0197,'10as':0.1472,'11dh':0.0308,'11ei':0.0506,
                    '07Y':0.0184, '99ex':0.0172, '07c':0.0363,'07d':0.2881,
                    '07i':0.0250,'06f':0.1635,'06t':0.0647,'13df':0.017,
                    '98bw':0.0509, '03lw':0.9040,'10bh':0.1000,'13dx':0.0368,
                    '11bm':0.0289,'11fu':0.0664,'11hs':0.0107,'13cq':0.0174,
                    '12bz':0.0303,'PTF10vgv':0.1382,'PTF10qts':0.0252,'iPTF13bvn':0.0436}

#        self.ebmvhost={'02ap':0.03,'03jd':0.10,'04aw':0.352,'07Y':0.09,'07gr':0.038,'07ru':0.01,'08D':0.59,'08ax':0.28}
#        self.ebmvhost={'02ap':0.03,'03jd':0.10,'04aw':0.352,'07gr':0.038,'07ru':0.01,'08D':0.5406}#,'08ax':0.3814}
#        self.ebmvhost={'02ap':0.03,'04aw':0.352,'07gr':0.03,'08D':0.5406}
        self.ebmvhost={'02ap':0.03,'03jd':0.10,'04aw':0.352,'07gr':0.038,'99ex':1.0,#Hamuy02
                       '07ru':0.01,'08D':0.5406,'07Y':0.112,#Stritzinger09
                       '08ax':0.4, '03bg':0,'11ei':0.18,'11dh':0,#Taubenberger11
                       #'09jf':0.05, #Valenti11
#                       '08bo':0.24,
#                       '08ax':0.3814,
#                       '04dk':0.201,'04dn':0.5265,
#                       '04fe':0.294,
#                       #'04ff':0.274,
#                       '04gq':0.19,'05az':0.43,'05kz':0.47,
#                       '05hg':0.63,
#                       '05mf':0.383,
#                       '06el':0.21,
#                       #'06C':0.65,
#                       '05bf':0.007,
#                       '07uy':0.601, '09jf':0.0146
                       '11bm': 0.032# valenti13
                        }

        self.ebmvcfa={'02ap':0.03,'03jd':0.10,'04aw':0.352,
                      '07ru':0.01,'08D':0.5406,
                      '06el' : 0.147  , '06ep':   0.448  ,'06fo'  : 0.201,
                      '07gr':   0.0462,
                      '07kj' :  0.295  ,'07uy' :  0.378  ,'08bo' :  0.294,
                      '09er' :  0.110  ,'09iz' :  0.064  ,'09jf' :  0.045,
                      '05bf':   0.05  ,'05hg' :  0.244  ,'05kl' :  1.344,
                      '05kz' :  0.437  ,'05mf' :  0.231  ,'06aj' :  0.141,
                      '06bf' :  0.368  ,'06F' :   0.533  ,'06lv' :  0.574  ,
                      '06T' :   0.397  ,'07ag' :  0.627  ,'07C' :   0.650  ,
                      '07ce' :  0.082  ,'07cl' :  0.258  ,'08D' :   0.572  }

#'09iz':0.1,
#07grxt Drout reddening via Chen 2014 was 0.06, drout is 0.038
#05hg Cano reddening 0.63 WRONG! from drout photometry?
#09iz guessing the reddening
##'06aj':0.04, gives trouble although reddening should be well constrained!!
        self.AonEBmV={'U': 5.434, 'u':5.155, 'B': 4.315, 'V': 3.315, 'R': 2.673,
                      'I': 1.940, 'r': 2.751,'i': 2.086,
                      'J': 0.902,'H': 0.576, 'K': 0.367,
                      'uu':5.155,'vv':3.315,'w1':4.3,'w2':4.3,'m2':4.3}

        self.uberTemplate = {}
        self.meansmooth = {}
        for b in self.bands:
            self.meansmooth[b] = None
            self.uberTemplate[b] = {}

class atemplate:
     def __init__(self):
          self.median=None
          self.mean  =None
          self.std   =None
          self.x     =None
          self.tempfuncy=None

class templatesn:
     def __init__(self, sn, dist=0.0, e_dist=0.0, incl=0.0, ebv=0.0, sntype='',
                  mjdmax=0.0, e_mjdmax=0.0, ebmvhost=0.0, vmr=0.0,peak=0.0,
                  e_peak=0.0, peak_ap=0.0, e_peak_ap=0.0,
                  phot=None, absphot=None, normphot=None,
                  new_x=None, new_y=None, yfunc=None):
        self.sn=sn
        self.dist=dist
        self.e_dist=e_dist
        self.incl=incl
        self.ebv=ebv
        self.sntype=sntype
        self.mjdmax=mjdmax
        self.e_mjdmax=e_mjdmax
        self.ebmvhost=ebmvhost
        self.vmr=vmr
        self.peak=peak
        self.e_peak=e_peak
        self.peak_ap=peak_ap
        self.e_peak_ap=e_peak_ap

        self.phot = phot
        self.absphot = absphot
        self.normphot = normphot
        if new_x:
             self.new_x = new_x
        else:
             self.new_x = NEW_X
        self.new_y = new_y

        self.yfunc=yfunc

class gptemplclass:
    def __init__(self):
        self.su=setupvars()
        self.gptemplate = {}
        for b in self.su.bands:
             self.gptemplate[b]=atemplate()

        self.typetemplate= {'Ib':None,'Ic':None,'IIb':None,'Ic-bl':None,'IIb-n':None}

class Mytempclass:
    def __init__(self):
        self.su=setupvars()
        self.template = {}
        for b in self.su.bands:
             self.template[b]=atemplate()

        self.typetemplate= {'Ib':None,'Ic':None,'IIb':None,'Ic-bl':None,'IIb-n':None}

    def calctemplate(self,tptype=None):
         thissn = mysn(f, addlit=True)
         thissn.readinfofileall(verbose=False, earliest=False, loose=True)
         if tptype is None:
              tytype = self.typetemplate

    def loadtemplate(self, b, x=None,mean=None,median=None,std=None):
        if b in self.su.bands:
            print ("ready to load temaplte")
            self.template[b].x=x
            self.template[b].mean=mean
            self.template[b].median=median
            self.template[b].std=std
        else:
            print ("wrong band ", b)

    def gettempfuncy(self,b):
        if 1:#not self.template[b].tempfuncy():
            from scipy.interpolate import interp1d
            self.template[b].tempfuncy = interp1d(self.template[b].x,
                                                  self.template[b].mean, kind='cubic',
                                                  bounds_error=False)
            if np.sum(np.isnan(self.template[b].tempfuncy(self.template[b].x))) == \
               len(self.template[b].x) or \
               np.std(np.array(self.template[b].tempfuncy(self.template[b].x)\
                               [~np.isnan(self.template[b].tempfuncy(
                                    self.template[b].x))]))>10:
                self.template[b].tempfuncy = interp1d(self.template[b].x,
                                                      self.template[b].mean,
                                                      kind='linear', bounds_error=False)

    def tempfuncstd(self,b):
        from scipy.interpolate import interp1d
        tempfuncstd = interp1d(self.template[b].x,self.template[b].std,
                               kind='cubic', bounds_error=False)
        if np.sum(np.isnan(tempfuncstd(self.template[b].x)))==len(self.template[b].x) \
        or np.std(np.array(tempfuncstd(self.template[b].x)\
                           [~np.isnan(tempfuncstd(self.template[b].x))]))>10:
            tempfuncstd = interp1d(self.template[b].x,self.template[b].std,
                                   kind='linear', bounds_error=False)

    def loadtemplatefile(self, new=False, sntype=None):
        for b in self.su.bands:
             print (b, self.template[b].mean)
             if self.template[b].mean:
                 print ("template already read in")
                 continue
###preparing template functions"
             if new:
                  if not sntype:
                       print ("must input sn type")
                       sys.exit()
                  myfile ='templates/new/mytemplate'+b+'_'+sntype+'.pkl'
             else:
                  myfile='templates/mytemplate'+b+'.pkl'
             if not os.path.isfile(myfile):
                   print ("file not there: ", myfile)
                   continue
             pkl_file = open(myfile,'rb')
             self.template[b] = pickle.load(pkl_file)
             pprint.pprint(self.template[b])

    def templateupdate(self,s):
         if self.typetemplate[sn.sntype]==None:
              typetemplate[sn.sntype]=copy.deepcopy(template)
         else:

              for b in ['V','R']:#su.bands:
                new_x = NEW_X
                new_y = interp1d(sn.photometry[b]['mjd']-sn.Vmax, s['normphot'],
                                 kind='cubic', bounds_error=False)(s['new_x'])

                print (np.sum(np.isnan(np.array(new_y))), np.isnan(np.array(new_y).all()),
                       len(new_y))
                if np.sum(np.isnan(np.array(new_y)))== len(new_y):
                    print ("REDOING SPLINE WITH LINEAR")
                    new_y = interp1d(sn.photometry[b]['mjd']-sn.Vmax,
                                     sn.photometry[b]['mag']-sn.maxdata,
                                     kind='linear', bounds_error=False)(s['new_x'])

                if np.std(np.array(new_y[~np.isnan(new_y)]))>10:
                    new_y = interp1d(sn.photometry[b]['mjd']-sn.Vmax,
                                     sn.photometry[b]['mag']-sn.maxdata,
                                     kind='linear', bounds_error=False)(s['new_x'])
                print (s['new_y'], s['normphot'])
                print (s['sn'],np.min(s['new_y'][~np.isnan(s['new_y'])]))

                typetemplate[sn.sntype][b]=stats.stats.nanmean([new_y,
                                                              typetemplate[sn.sntype][b]],
                                                             axis=0)
                typetemplate[sn.sntype][b]=stats.stats.nanmean([new_y,
                                                              typetemplate[sn.sntype][b]],
                                                             axis=0)

def mycavvaccaleib(x,p, secondg=False,earlyg=False, verbose=False):
     if verbose:
          print ("\np,x",p,x)
     try:
          #needed in case i am passing an array of 1 element for x
          if verbose:
               print (len(x))
          if len(x)==1:
               x=x[0]
     except:
          pass
     latebump=False
     if p is None:
          return (x)*99e9
     if p[8]>p[1]:
          if verbose:
               print ("late bump")
          latebump=True

     '''
     fit the magnitudes with a vacca leibundgut (1997) analytical model

     p is the parameter list
     if secondg=1: secondgaussian added
     if secondg=0: secondgaussian not
     parameters are:
     p[0]=first gaussian normalization (negative if fitting mag)
     p[1]=first gaussian mean
     p[2]=first gaussian sigma
     p[3]=linear decay offset
     p[4]=linear decay slope
     p[5]=exponxential rise slope
     p[6]=exponential zero point
     p[7]=second gaussian normalization (negative if fitting mag)
     p[8]=second gaussian mean
     p[9]=second gaussian sigma
     '''
     g=p[4]*(x)+p[3]
     g+=p[0]*np.exp(-(x-p[1])**2/p[2]**2)
     g*=(np.exp(-p[5]*(x-p[6]))+1)
     if latebump and  earlyg:
          g*=1e5
     if secondg:
          g+=p[7]*np.exp(-(x-p[8])**2/p[9]**2)
     if latebump and p[8]-p[1]<15 :
          g+=p[7]*np.exp(-(x-p[8])**2/p[9]**2)
     try:
          len(g)
     except TypeError:
          g=[g]

          #(np.zeros(len(g),float)+1)
     if p[8]-p[1]>70:
          g+=(np.zeros(len(g),float)+1)
     return g

def loadlitlist(band):
    print (TEMPLDIR)
    try:
        f=TEMPLDIR+"/"+"templatelist"+band+".txt"
        templates=np.loadtxt(f,usecols=(0,1,2,3,4,5,7,8,9,10,13,14,15,16),
                             dtype={'names': ('sn','dist','e_dist','incl','ebv',
                                              'type','mjdmax','e_mjdmax',
                                              'peak_ap','e_peak_ap','ebmvhost',
                                              'vmr','peak','e_peak'),
                                    'formats': ('S6','f','f','f','f','S5','f',
                                                'f','f','f','f','f','f','f')},
                             skiprows=1, comments='#')

        print ("reading files list ",f," worked")
    except:
        print ("reading files list ",f," failed")

    sne=[]
    print ("the templates are: ",templates['sn'])

    for i,sn in enumerate(templates['sn']):
        sne.append(templatesn(sn,dist=templates[i]['dist'],
                              e_dist=templates[i]['e_dist'],
                              incl=templates[i]['incl'],
                              ebv=templates[i]['ebv'],
                              sntype=templates[i]['type'],
                              mjdmax=templates[i]['mjdmax'],
                              e_mjdmax=templates[i]['e_mjdmax'],
                              ebmvhost=templates[i]['ebmvhost'],
                              vmr=templates[i]['vmr'],
                              peak=templates[i]['peak'],
                              e_peak=templates[i]['e_peak'],
                              peak_ap=templates[i]['peak_ap'],
                              e_peak_ap=templates[i]['e_peak_ap']))

#    for i,sn in  enumerate(templates['sn']):
        sne[i].phot=(np.loadtxt(TEMPLDIR+"/"+"sn"+sn+".dat",
                                usecols=(0,bandscols[band], bandscols[band]+1),
                                skiprows=1, unpack=1))
        #dereddening
        sne[i].phot[1]+=3.2*sne[i].ebv

        sne[i].absphot=sne[i].phot[1]-(sne[i].peak_ap-sne[i].peak)
        sne[i].normphot=sne[i].absphot-sne[i].peak
        #sne[i]['phot'][1]-2.5*np.log10((sne[i]['dist']/1e6/10.0)**2)
#    mysn=np.where(templates['sn']=='1994I')[0]
#    print sne[mysn]
##        Mv = m - 2.5 log[ (d/10)2 ].
##        flux = 10**(-lc['mag']/2.5)*5e10
##        dflux = flux*lc['dmag']/LN10x2p5
#        print sn
#
    print ("loaded sne")
    return sne, templates

def splinetemplates(sne):
    for s in sne:
#        pl.figure()
#        minx=max(-20.1,min(s['phot'][0]-s['mjdmax']))+0.1
#        maxx=min(40.1,max(s['phot'][0]-s['mjdmax']))-0.1
        s.new_x = NEW_X
        s.new_y = interp1d(s.phot[0]-s.mjdmax, s.normphot, kind='cubic',
                           bounds_error=False)(s.new_x)
        NaN = float('nan')

        print (np.sum(np.isnan(np.array(s.new_y))), np.isnan(np.array(s.new_y).all()),
               len(s.new_y))
        if np.sum(np.isnan(np.array(s.new_y)))== len(s.new_y):
            print ("REDOING SPLINE WITH LINEAR")
            s.new_y = interp1d(s.phot[0]-s.mjdmax, s.normphot, kind='linear',
                               bounds_error=False)(s.new_x)

        if np.std(np.array(s.new_y[~np.isnan(s.new_y)]))>10:
            s.new_y = interp1d(s.phot[0]-s.mjdmax, s.normphot, kind='linear',
                               bounds_error=False)(s.new_x)
#        print (s['new_y'], s['normphot'])

        print (s.sn,np.min(s.new_y[~np.isnan(s.new_y)]))

    return sne


import torch
import torchvision
import torch.nn as nn

def get_vgg16():
    vgg16 = torchvision.models.vgg16(pretrained=False)
    weight = vgg16.features[0].weight.clone()
    vgg16.features[0] = nn.Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=1)
    with torch.no_grad():
        vgg16.features[0].weight[:, 0] = weight[:, 0]
    vgg16.classifier[6] = nn.Linear(in_features=4096, out_features=4, bias=True)
    return vgg16

def get_resnet50():
    resnet50 = torchvision.models.resnet50(pretrained=False)
    weight = resnet50.conv1.weight.clone()
    resnet50.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=3)
    with torch.no_grad():
        resnet50.conv1.weight[:, 0] = weight[:, 0]
    resnet50.fc = nn.Linear(in_features=2048, out_features=4, bias=True)
    return resnet50

def get_squeezenet():
    squeezenet = torchvision.models.squeezenet1_0(pretrained=False)
    weight = squeezenet.features[0].weight.clone()
    squeezenet.features[0] = nn.Conv2d(1, 96, kernel_size=(7, 7), stride=(1, 1), padding=1)
    with torch.no_grad():
        squeezenet.features[0].weight[:, 0] = weight[:, 0]
    squeezenet.classifier[1] = nn.Conv2d(512, 4, kernel_size=(1, 1), stride=(1, 1))
    return squeezenet

def get_resnext():
    resnext = torchvision.models.resnext50_32x4d(pretrained=False)
    weight = resnext.conv1.weight.clone()
    resnext.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=3)
    with torch.no_grad():
        resnext.conv1.weight[:, 0] = weight[:, 0]
    resnext.fc = nn.Linear(2048, 4, bias=True)
    return resnext

def get_densenet():
    densenet = torchvision.models.densenet161(pretrained=False)
    weight = densenet.features.conv0.weight.clone()
    densenet.features.conv0 = nn.Conv2d(1, 96, kernel_size=(7, 7), stride=(2, 2), padding=3)
    with torch.no_grad():
        densenet.features.conv0.weight[:, 0] = weight[:, 0]
    densenet.classifier = nn.Linear(2208, 4, bias=True)
    return densenet

def count_parameters(model):
        return sum(p.numel() for p in model.parameters() if p.requires_grad)


# 导入操作系统库
import os
# 更改工作目录
os.chdir(r"D:\softwares\applied statistics\pythoncodelearning\chap1\sourcecode")
# 导入基础计算库
import numpy as np
# 导入绘图库
import matplotlib.pyplot as plt
# 导入Lasso模型
from sklearn.linear_model import MultiTaskLasso, Lasso
# 导入绘图库中的字体管理包
from matplotlib import font_manager
# 实现中文字符正常显示
font = font_manager.FontProperties(fname=r"C:\Windows\Fonts\SimKai.ttf")
# 使用seaborn风格绘图
plt.style.use("seaborn-v0_8")
# 设置样本量，维度，回归模型中y的维度（响应变量的多元回归）
n_samples, n_features, n_tasks = 100, 30, 40
# 显著变量的个数
n_relevant_features = 5
# 初始化真实系数，是一个矩阵
coef = np.zeros((n_tasks, n_features))
# 时刻
times = np.linspace(0, 2 * np.pi, n_tasks)
# 设置随机数种子
np.random.seed(10)
# 生成真实系数
for k in range(n_relevant_features):
    coef[:, k] = np.sin(
        (1.0 + np.random.randn(1)) * times + 3 * np.random.randn(1)
    )
# 生成X
X = np.random.randn(n_samples, n_features)
# 生成Y
Y = np.dot(X, coef.T) + np.random.randn(n_samples, n_tasks)
print("查看多元响应变量Y的情况：", Y[:5, :2], sep="\n")
# 建立Lasso模型，分别对Y的每一个分量做，提取系数
coef_lasso_ = np.array(
    [
        Lasso(alpha=0.5).fit(X, y).coef_ for y in Y.T
    ]
)
# 建立MultiLasso模型，提取系数
coef_multi_task_lasso_ = MultiTaskLasso(alpha=1.0).fit(X, Y).coef_
# 开始绘图
fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(8, 5))
# 用于展示稀疏二维数组的图形
axs[0].spy(coef_lasso_)
axs[0].set_xlabel("Feature")
axs[0].set_ylabel("Time (or Task)")
axs[0].text(10, 5, "Lasso")
axs[1].spy(coef_multi_task_lasso_)
axs[1].set_xlabel("Feature")
axs[1].set_ylabel("Time (or Task)")
axs[1].text(10, 5, "MultiTaskLasso")
fig.suptitle("Coefficient non-zero location")
plt.show()
fig.savefig("../codeimage/code15.pdf")
# 绘制第一个特征前的系数
feature_to_plot = 0
# 开始绘图
fig1, ax = plt.subplots(figsize=(6,6), tight_layout=True)
# 绘制coef的线图
ax.plot(
    coef[:, feature_to_plot],
    color="seagreen",
    linewidth=2,
    label="Ground truth"
)
# 绘制coef_lass0的线图
ax.plot(
    coef_lasso_[:, feature_to_plot],
    color="cornflowerblue",
    linewidth=2,
    label="Lasso"
)
# 绘制coef_task_lasso的线图
ax.plot(
    coef_multi_task_lasso_[:, feature_to_plot],
    color="gold",
    linewidth=2,
    label="MultiTaskLasso",
)
# 显示图例
ax.legend(loc="best")
# 设置纵轴范围
ax.set_ylim([-1.1, 1.1])
plt.show()
fig1.savefig("../codeimage/code16.pdf")


from django.shortcuts import render, redirect, reverse
from django.contrib.auth.forms import AuthenticationForm
from django.contrib.auth import login as auth_login, logout as auth_logout
from django.contrib import messages
from django.utils.translation import ugettext as _

from .forms import RegistrationForm

def login(request):
    login_form = AuthenticationForm()

    if request.user.is_authenticated():
        return redirect('/')
    if request.POST:
        login_form = AuthenticationForm(request, data=request.POST)
        if login_form.is_valid():
            auth_login(request, login_form.get_user())
            return redirect('/')

    context = {
        'form': login_form,
    }
    return render(request, template_name='account/login.html', context=context)

def logout(request):
    auth_logout(request)
    messages.add_message(request, messages.WARNING, _('You\'ve been logged out.'))
    return redirect(reverse('login'))

def register(request):
    register_form = RegistrationForm()

    if request.user.is_authenticated():
        return redirect('/')
    if request.POST:
        register_form = RegistrationForm(request.POST)
        if register_form.is_valid():
            user = register_form.save()
            auth_login(request, user)
            messages.add_message(request, messages.SUCCESS, _('Your account has been registered.'))
            return redirect('/')

    context = {
        'form': register_form
    }

    return render(request, template_name='account/register.html', context=context)


"""A setuptools based setup module.
See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='mygrations',
    version='1.0.6',
    description='A general purpose migration tool for managing MySQL updates',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/cmancone/mygrations',
    author='Conor Mancone',
    author_email='cmancone@gmail.com',
    license='MIT',
    download_url='https://github.com/cmancone/mygrations/archive/v1.0.0.tar.gz',

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Build Tools',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3',
    ],
    keywords='setuptools development migrations mysql',
    packages=find_packages(exclude=['contrib', 'docs', 'tests']),
    python_requires=">=3.6",
    install_requires=[
        'PyMySQL',
    ],
    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
    # data_files=[('/usr/local/bin', ['mygrate.py'])],
)


from fastapi import APIRouter, Depends, HTTPException
from pyairtable.api.table import Table
from pyairtable.formulas import match
from sentry_sdk import capture_message

from app.auth.auth_bearer import JWTBearer
from app.routers.attendees import update_attendee
from ..dependencies import get_mobile_table, get_registration_table
from ..auth.auth_handler import verify_jwt, decode_jwt
from ..models.attendee import UpdatedAttendee, recordToAttendee

router = APIRouter(prefix="/verify", tags=["verify"])

@router.get("/discord", dependencies=[Depends(JWTBearer())])
async def verify_discord(
    email: str, disc_username: str, table: Table = Depends(get_mobile_table)
):
    res = []
    for i in table.all(formula=match({"Email": email})):
        res.append(recordToAttendee(i))

    if len(res) == 0:
        e = HTTPException(
            status_code=400,
            detail="No user with specified email found.",
        )
        capture_message("No user with specified email found.")
        raise e
    else:
        user = res[0]
        if user.discord_id:
            e = HTTPException(
                status_code=400,
                detail="Discord username already set.",
            )
            capture_message("Discord username already set.")
            raise e

        try:
            table.update(user.airtable_id, {"Discord ID": disc_username})
        except:
            raise HTTPException(status_code=500, detail="Updating attendee failed")

    return user.first_name, user.last_name

@router.get("/{token}")
async def verify_email(token: str, table: Table = Depends(get_registration_table)):
    # token is invalid or expired
    if not verify_jwt(token):
        e = HTTPException(
            status_code=403,
            detail="Expired or invalid verification link. Please try registering again!",
        )
        capture_message("Invalid or expired verification link.")
        raise e

    payload = decode_jwt(token)
    attendee_id = payload["id"]
    email_type = payload["type"]

    if not attendee_id or not email_type:
        # token does not contain payload id
        e = HTTPException(status_code=403, detail="Invalid verification link.")
        capture_message("Invalid verification link.")
        raise e

    field_name = "Parent Email Verified" if email_type == "parent" else "Email Verified"

    try:
        attendee = table.update(attendee_id, {field_name: True})

        parent_email_verified = (
            attendee["fields"]["Parent Email Verified"]
            if "Parent Email Verified" in attendee["fields"]
            else False
        )
        email_verified = (
            attendee["fields"]["Email Verified"]
            if "Email Verified" in attendee["fields"]
            else False
        )

        return {
            "parent_email_verified": parent_email_verified,
            "email_verified": email_verified,
        }
    except:
        # attendee does not exist
        e = HTTPException(status_code=403, detail="Invalid attendee ID.")
        capture_message("Invalid attendee ID.")
        raise e


from abc import ABC, abstractmethod
import logging

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.session import Session
from sqlalchemy.pool import NullPool
from os import getenv
from dotenv import load_dotenv
from models import BaseTable

class Collector(ABC):

    def __init__(self) -> None:
        self.session_factory = SessionHandler()

        # set up logging
        self.logger = logging.getLogger(__name__)
        logging.basicConfig(
            level=logging.INFO,
            format='%(levelname)s-%(threadName)s: %(message)s'
        )
    @abstractmethod
    def _collect_surfaces(self, bag_building_id: str):
        raise NotImplementedError('Implement this')

    @abstractmethod
    def _collect_images(self, model: BaseTable):
        raise NotImplementedError('Implement this')

class SessionHandler:
    engine = None

    def __init__(self, echo=False):
        load_dotenv()

        user = getenv('user')
        host = getenv('host')
        name = getenv('dbname')
        self.engine = create_engine(
            f'postgresql://{user}:@{host}/{name}',
            poolclass=NullPool,
            echo=echo
        )

    def build(self) -> Session:
        return sessionmaker(bind=self.engine)()


import os

import pytest
import textwrap

@pytest.fixture()
def sample_test(testdir):
    testdir.makefile(
        ".feature",
        scenario=textwrap.dedent(
            """\
            Feature: Testing
                Scenario: Test scenario
                    Given I have a scenario
                    When I start the test
                    And I know it will fails
                    Then It fails
            """
        ),
    )

    testdir.makepyfile(
        textwrap.dedent(
            """\
        import pytest
        from pytest_bdd import given, when, then, scenario

        @scenario("scenario.feature", "Test scenario")
        def test_scenario():
            pass

        @given("I have a scenario")
        def _():
            pass

        @when("I start the test")
        def _():
            pass

        @when("I know it will fails")
        def _():
            pass

        @then('It fails')
        def _():
            assert False

        """
        )
    )

    return testdir

def test_arguments_in_help(testdir):
    res = testdir.runpytest("--help")
    res.stdout.fnmatch_lines(
        [
            "*bdd-report*",
        ]
    )

def test_create_html_report_file(sample_test):
    sample_test.runpytest("--bdd-report=report.html")
    assert (sample_test.tmpdir / "report.html").exists()

def test_create_html_report_file_with_directory(sample_test):
    sample_test.runpytest("--bdd-report=./reports/report.html")
    assert (sample_test.tmpdir / "./reports/report.html").exists()

def test_create_html_report_file_with_directory_name(sample_test):
    sample_test.runpytest("--bdd-report=results/report.html")
    assert (sample_test.tmpdir / "results/report.html").exists()

def test_create_html_report_file_with_directory_and_subdirectory(sample_test):
    sample_test.runpytest("--bdd-report=./reports/year/report.html")
    assert (sample_test.tmpdir / "./reports/year/report.html").exists()

def test_content_in_report(sample_test):
    sample_test.runpytest("--bdd-report=report.html")
    content = ""
    with open((sample_test.tmpdir / "report.html"), "r") as f:
        content = f.read()
    assert content != ""

def test_information_in_report(sample_test):
    sample_test.runpytest("--bdd-report=report.html")
    content = ""
    with open((sample_test.tmpdir / "report.html"), "r") as f:
        content = f.read()
    assert "Testing" in content
    assert "Test scenario" in content
    assert "I have a scenario" in content
    assert "I start the test" in content
    assert "I know it will fails" in content
    assert " It fails" in content


import datetime
import inspect
import json
import boto3
from util import create_ec2_client, create_ec2_resource, print_response

def create_elastic_ip(ec2_client):
    # https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.allocate_address
    # DomainのvpcはVPC、standardはEC2-Classic向け
    response = ec2_client.allocate_address(Domain='vpc')
    print_response(inspect.getframeinfo(inspect.currentframe())[2], response)
    return response['AllocationId']

def create_nat_gateway(ec2_client, allocation_id, subnet_id):
    # https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.create_nat_gateway
    response = client.create_nat_gateway(
        AllocationId=allocation_id,
        SubnetId=subnet_id,
    )
    print_response(inspect.getframeinfo(inspect.currentframe())[2], response)
    return response['NatGateway']['NatGatewayId']

def describe_main_route_tables(ec2_client, vpc_id):
    # https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.describe_route_tables
    response = ec2_client.describe_route_tables(
        Filters=[
            {
                'Name': 'association.main',
                'Values': ['true'],
            },
            {
                'Name': 'vpc-id',
                'Values': [vpc_id],
            }
        ]
    )
    main_route_table_id = response['RouteTables'][0]['RouteTableId']
    print_response(inspect.getframeinfo(inspect.currentframe())[2], main_route_table_id)
    return main_route_table_id

def wait_nat_gateway_available(ec2_client, nat_gateway_id):
    # https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Waiter.NatGatewayAvailable
    print(f'NAT Gatewayがavailableになるまで待つ(開始)：{datetime.datetime.now()}')
    waiter = ec2_client.get_waiter('nat_gateway_available')
    response = waiter.wait(
        Filters=[{
            'Name': 'state',
            'Values': ['available']
        }],
        NatGatewayIds=[nat_gateway_id]
    )
    print_response(inspect.getframeinfo(inspect.currentframe())[2], response)
    print(f'NAT Gatewayがavailableになるまで待つ(終了)：{datetime.datetime.now()}')

def create_nat_gateway_route_in_route_table(ec2_resource, route_table_id, nat_gateway_id):
    # https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.RouteTable.create_route
    route_table = ec2_resource.RouteTable(route_table_id)
    route = route_table.create_route(
        DestinationCidrBlock='0.0.0.0/0',
        NatGatewayId=nat_gateway_id,
    )
    print_response(inspect.getframeinfo(inspect.currentframe())[2], route)
    return route

if __name__ == '__main__':
    session = boto3.Session(profile_name='my-profile')
    # 使用するクライアントとリソースを作成
    client = create_ec2_client(session)
    resource = create_ec2_resource(session)

    # AWSの各IDを取得する
    with open('aws.json', mode='r') as f:
        aws = json.load(f)

    # Elastic IPを取得する
    aws['allocation_id'] = create_elastic_ip(client)

    # パブリックサブネットにNATゲートウェイを置く
    aws['nat_gateway_id'] = create_nat_gateway(client, aws['allocation_id'], aws['public_subnet_id'])

    # NATゲートウェイはすぐに使うことができないため、availableになるまで待つ
    wait_nat_gateway_available(client, aws['nat_gateway_id'])

    # NATゲートウェイのエントリを追加するため、メインのルートテーブルのIDを取得する
    aws['main_route_table_id'] = describe_main_route_tables(client, aws['vpc_id'])

    # VPC領域2でメインのルートテーブルにNATゲートウェイのエントリを追加する
    create_nat_gateway_route_in_route_table(resource, aws['main_route_table_id'], aws['nat_gateway_id'])

    # ここまでのid情報をJSONとして上書き保存
    with open('aws.json', mode='w') as f:
        json.dump(aws, f)


from parlai.core.teachers import DialogTeacher
from .build import build

import json
import os
import glob

def _path(opt):
    build(opt)
    print('opt is', opt['datatype'])
    dt = opt['datatype'].split(':')[0]

    if dt == 'valid':
        dt = 'dev'
    elif dt != 'train' and dt != 'test':
        raise RuntimeError('Not valid datatype.')

    prefix = os.path.join(opt['datapath'], 'nlvr', 'nlvr-master', 'nlvr')
    questions_path = os.path.join(prefix, dt, dt + '.json')
    images_path = os.path.join(prefix, dt, 'images')

    return questions_path, images_path

class DefaultTeacher(DialogTeacher):
    # all possile answers for the questions
    cands = labels = ['true', 'false']

    def __init__(self, opt, shared=None):
        self.datatype = opt['datatype']
        data_path, self.images_path = _path(opt)
        opt['datafile'] = data_path
        self.id = 'nlvr'
        self.dt = opt['datatype'].split(':')[0]
        if self.dt == 'valid':
            self.dt = 'dev'

        super().__init__(opt, shared)

    def label_candidates(self):
        return self.cands

    def setup_data(self, path):
        print('loading: ' + path)

        for line in open(path, 'r'):
            ques = json.loads(line)

            image_path = os.path.join(self.images_path, ques['directory'])
            image_file_names = glob.glob(
                image_path + '/' + self.dt + '-' + ques['identifier'] + '*'
            )

            question = "True or False: " + ques['sentence']
            answer = [ques['label']] if self.dt != 'test' else None
            # print( answer)
            yield (question, answer, None, None, image_file_names[0]), True


from setuptools import find_packages, setup
from typing import List

import io
import os

def parse_requirements(filename: str) -> List[str]:
    required_packages = []
    with open(os.path.join(os.path.dirname(__file__), filename)) as req_file:
        for line in req_file:
            required_packages.append(line.strip())
    return required_packages

setup(
    name='ml_pet',
    packages=find_packages(),
    python_requires=">=3.7",
    install_requires=parse_requirements("requirements.txt"),
    extras_requires=parse_requirements("requirements-dev.txt"),
    version='0.1.0',
    description='A python repository of research and utility code for analysis of PET image reconstructions and automatic detection of parathyroid adenomas from [F-18]-FCH PET/CT',
    author='Filip Cvetko',
    license='MIT',
)


from collections import deque
# For debugging
import time

#-----
# Define Node class
#-----
class Node:
    def __init__(self, val, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

    def __repr__(self):
        return str(self.val)

#-----
# DFT - recursive
#-----
def dft(root, path=None):
    '''Depth-first traversal (recursively). Returns list w/ values in path.'''
    if not root:
        return path
    if not path:
        path = []

    print(root.val, end=' ')
    path.append(root.val)
    dft(root.left, path)
    dft(root.right, path)
    return path

#-----
# DFT - iterative
#-----
def dft_iter(root):
    '''Depth-first traversal (iteratively). Returns list w/ values in path.'''
    if root is None:
        return
    stack = [root]
    path = []
    while stack:
        node = stack.pop()
        path.append(node.val)
        print(node.val, end=' ')
        # Note we must add to the stack in *reverse* order to be processed (right then left)
        if node.right:
            stack.append(node.right)
        if node.left:
            stack.append(node.left)
    return path

#-----
# BFT - iterative
#-----
def bft(root):
    '''Breadth-first traversal, or "level-order", of tree'''
    if root is None:
        return path
    queue = deque([root])
    path = []
    while queue:
        node = queue.popleft()
        path.append(node.val)
        print(node.val, end=' ')
        if node.left:
            queue.append(node.left)
        if node.right:
            queue.append(node.right)
    return path

#-----
# BST insert
#-----
def insert_bst(root, val):
    '''Insert new node into BST, returning root node of updated tree'''
    if root is None:
        return Node(val)
    if val < root.val:
        root.left = insert_bst(root.left, val)
    else:
        root.right = insert_bst(root.right, val)
    return root

#-----
# BST lookup
#-----
def lookup_bst(root, val):
    '''Return first node found with given value'''

    if root is None:
        print(f'NO value {val} found in tree')
        return None
    if root.val == val:
        print(f'Found value {val} in tree; returning node')
        return root
    if val < root.val:
        return lookup_bst(root.left, val)
    else:
        return lookup_bst(root.right, val)

#-----
# BST delete
#-----
def delete_bst(root, val, parent=None):
    '''Delete first node found with given value; return root of updated tree.'''

    # Case: empty tree or val not found
    if root is None:
        print(f'DEBUG: value {val} not found in tree')
        return

    if root.val == val:
        # Case: leaf
        if root.left is None and root.right is None:
            # TODO: delete the parent's correct child reference
            # TODO: DEBUG this -- fails when parent is null
            if parent.val < val:
                parent.right = None
            else:
                parent.left = None

        # Case: 1 child
        elif root.left is not None:
            root.val = root.left.val
            root.left = None
        elif root.right is not None:
            root.val = root.right.val
            root.right = None

        # Case: 2 children
        else:
            #replacement_node = find_successor_bst(root.right, val)
            delete_bst(root.right, replacement_node.val)
            root.val = replacement_node.val

    elif val < root.val:
        delete_bst(root.left, val, root)
    else:
        delete_bst(root.right, val, root)

    return root

#-----
# Create test tree(s)
#-----

# EXAMPLE TREE 1
#
#       10
#     _/  \_
#    /      \
#   5        15
#  / \      /  \
# 3   7    13   17
#
node1_1 = Node(5, Node(3), Node(7))
node1_2 = Node(15, Node(13), Node(17))
tree1 = Node(10, node1_1, node1_2)

# EXAMPLE TREE X
#
#       5
#     _/ \_
#    /     \
#   5       5
#  / \       \
# 5   5       5
#

#---
# Run & assert correctness of DFT & BFT
# TODO: Add assertions and more test cases
#---
print('Calling dft()...')
assert dft(tree1) == [10, 5, 3, 7, 15, 13, 17]
print('\n')

print('Calling dft_iter()...')
assert dft_iter(tree1) == [10, 5, 3, 7, 15, 13, 17]
print('\n')

print('Calling bft()...')
assert bft(tree1) == [10, 5, 15, 3, 7, 13, 17]
print('\n')

print('Calling insert_bst(tree1, 6)...')
result = insert_bst(tree1, 6)
assert dft(result) == [10, 5, 3, 7, 6, 15, 13, 17]
print('\n')

print('Calling insert_bst(tree1, 20)...')
result = insert_bst(tree1, 20)
assert dft(result) == [10, 5, 3, 7, 6, 15, 13, 17, 20]
print('\n')

print('Calling insert_bst(None, 100)... [case: empty tree passed]')
result = insert_bst(None, 100)
assert dft(result) == [100]
print('\n')

print('Calling lookup_bst(tree1, 13)...')
result_node = lookup_bst(tree1, 13)
assert result_node.val == 13

print('Calling lookup_bst(tree1, 1000)... [case: not found]')
result_node = lookup_bst(tree1, 1000)
assert result_node == None
print('\n')

print('Calling delete_bst(tree1, 13)... [delete leaf (parent\'s left)]')
result_root = delete_bst(tree1, 13)
assert dft(result_root) == [10, 5, 3, 7, 6, 15, 17, 20]
print('\n')

print('Calling delete_bst(tree1, 20)... [delete leaf (parent\'s right)]')
result_root = delete_bst(tree1, 20)
assert dft(result_root) == [10, 5, 3, 7, 6, 15, 17]
print('\n')

print('Calling delete_bst(tree1, 17)... [delete node w/ one child]')
result_root = delete_bst(tree1, 17)
assert dft(result_root) == [10, 5, 3, 7, 6, 15]
print('\n')

print('Calling delete_bst(tree1, 5)... [delete node w/ two children]')
result_root = delete_bst(tree1, 5)
assert dft(result_root) == [10, 3, 7, 6, 15]
print('\n')

print('Calling delete_bst(Node(42), 42)... [delete only node]')
assert delete_bst(Node(42), 42) == None
print('\n')

print('Calling delete_bst(tree1, 42)... [node not found]')
assert delete_bst(tree1, 42) == None
print('\n')


import numpy as np
import matplotlib.pyplot as plt
import os
import pandas as pd
import csv

### NOTE: Need to make filepaths for the laptop now ###

def lagrange(X):
    '''
    Function for creating Lagrange basis functions.
    Inputs
    ------
    X: array-like. The elements nodal x-locations.
    Returns
    -------
    List of basis functions and a list of their derivatives. This function supports linear
    and quadratic basis functions.
    '''
    deg = len(X)-1
    if deg == 1:
        x0 = X[0]
        x1 = X[1]
        x = np.linspace(x0,x1)
        phi0 = (x-x1)/(x0-x1)
        phi1 = (x-x0)/(x1-x0)
        dphi0 = 1/(x0-x1)
        dphi1 = 1/(x1-x0)
        return [phi0, phi1], [dphi0*np.ones_like(x), dphi1*np.ones_like(x)], x
    elif deg ==2:
        x0, x1, x2 = X
        x = np.linspace(x0,x2)
        phi0 = (x-x1)*(x-x2)/((x0-x1)*(x0-x2))
        phi1 = (x-x0)*(x-x2)/((x1-x0)*(x1-x2))
        phi2 = (x-x0)*(x-x1)/((x2-x0)*(x2-x1))
        dphi0 = (2*x-x1-x2)/((x0-x1)*(x0-x2))
        dphi1 = (2*x-x0-x2)/((x1-x0)*(x1-x2))
        dphi2 = (2*x-x0-x1)/((x2-x0)*(x2-x1))
        return [phi0, phi1, phi2], [dphi0, dphi1, dphi2], x
    else:
        raise ValueError('Currently supporting only first or second order Lagrange basis functions.')

# def read_nodes(file):
#     path = os.getcwd()
#     if 'mattjwilliams' in path:
#         path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
#     node_df = pd.read_csv(os.path.join(path,file),delimiter=' ')
#     return node_df

def read_file(file):
    path = os.getcwd()
    if 'mattjwilliams' in path:
        path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
    else:
        path = '/Users/mattwilliams/Documents/PythonProjects/FEM/GangLi/IntroToFEM/Chapter4/HeatTransfer/ProgramFiles'
    df = pd.read_csv(os.path.join(path,file),delimiter=' ')
    return df

def mesher1D(xstart,xend,n_elements,deg=1):
    if deg == 1:
        n_nodes = n_elements + 1
        dx = (xend-xstart)/(n_elements)
        elements = np.arange(0,n_elements,dtype='int8')
        arr = np.zeros((n_elements,deg+2),dtype='object')
        node1 = np.arange(xstart,xend,dx)
        node2 = np.arange(xstart+dx,xend+dx,dx)
        arr[:,0] = elements
        arr[:,1] = node1
        arr[:,2] = node2
    else:
        n_nodes = 2*n_elements + 1
        dx = (xend-xstart)/(n_elements)
        elements = np.arange(0,n_elements,dtype='int8')
        arr = np.zeros((n_elements,deg+2),dtype='object')
        node1 = np.arange(xstart,xend,dx)
        node2 = np.arange(xstart+dx/2,xend+dx/2,dx)
        node3 = np.arange(xstart+dx,xend+dx,dx)
        arr[:,0] = elements
        arr[:,1] = node1
        arr[:,2] = node2
        arr[:,3] = node3
    path = os.getcwd()
    if 'mattjwilliams' in path:
        path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
    else:
        path = '/Users/mattwilliams/Documents/PythonProjects/FEM/GangLi/IntroToFEM/Chapter4/HeatTransfer/ProgramFiles'
    fname = 'nodes.txt'
    if deg == 1:
        headers = ['element','node0','node1']
    else:
        headers = ['element','node0','node1','node2']
    with open(os.path.join(path,fname),'w',newline='') as f:
        mywriter = csv.writer(f,delimiter=' ')
        mywriter.writerow(headers)
        mywriter.writerows(arr)
    return None

def element_indexer(n_elements,deg):
    arr = np.zeros((n_elements,deg+2),dtype='int8')
    arr[:,0] = np.arange(n_elements)
    if deg == 1:
        arr[:,1] = np.arange(0,n_elements,deg)
        arr[:,2] = np.arange(1,n_elements+1,deg)
        headers = ['element','node0','node1']
    else:
        n_nodes = 2*n_elements + 1
        arr[:,1] = np.arange(0,n_nodes-1,deg)
        arr[:,2] = np.arange(1,n_nodes,deg)
        arr[:,3] = np.arange(2,n_nodes+1,deg)
        headers = ['element','node0','node1','node2']

    path = os.getcwd()
    if 'mattjwilliams' in path:
        path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
    else:
        path = '/Users/mattwilliams/Documents/PythonProjects/FEM/GangLi/IntroToFEM/Chapter4/HeatTransfer/ProgramFiles'
    fname = 'elements.txt'
    with open(os.path.join(path,fname),'w',newline='') as f:
        mywriter = csv.writer(f,delimiter=' ')
        mywriter.writerow(headers)
        mywriter.writerows(arr)
    return None

def global_nodes(xstart,xend,n_elements,deg=1):
    if deg == 1:
        n_nodes = n_elements + 1
    else:
        n_nodes = 2*n_elements + 1
    arr = np.zeros((n_nodes,2),dtype='object')
    arr[:,0] = np.arange(n_nodes)
    dx = (xend-xstart)/(n_nodes-1)
    arr[:,1] = np.arange(xstart,xend+dx,dx)
    path = os.getcwd()
    if 'mattjwilliams' in path:
        path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
    else:
        path = '/Users/mattwilliams/Documents/PythonProjects/FEM/GangLi/IntroToFEM/Chapter4/HeatTransfer/ProgramFiles'
    fname = 'global_nodes.txt'
    headers = ['global node','x location']
    with open(os.path.join(path,fname),'w',newline='') as f:
        mywriter = csv.writer(f,delimiter=' ')
        mywriter.writerow(headers)
        mywriter.writerows(arr)
    return None

def boundary_conditions(n_elements,bcs,deg=1):
    '''
    Boundary conditions passed as a list:
    Left end type, left end value, right end type, right end value

    Type 1 == Dirichlet, Type 2 == Neumann or flux
    '''
    type1, val1, type2, val2 = bcs
    if deg == 1:
        n_nodes = n_elements + 1
    else:
        n_nodes = 2*n_elements + 1
    arr = np.zeros((n_nodes,3),dtype='object')
    arr[:,0] = np.arange(n_nodes)
    arr[0,1] = type1
    arr[0,2] = val1
    arr[-1,1] = type2
    arr[-1,2] = val2
    path = os.getcwd()
    if 'mattjwilliams' in path:
        path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
    else:
        path = '/Users/mattwilliams/Documents/PythonProjects/FEM/GangLi/IntroToFEM/Chapter4/HeatTransfer/ProgramFiles'
    fname = 'bcs.txt'
    headers = ['global node','type','value']
    with open(os.path.join(path,fname),'w',newline='') as f:
        mywriter = csv.writer(f,delimiter=' ')
        mywriter.writerow(headers)
        mywriter.writerows(arr)
    return None

def properties(n_elements,k1,k2):
    arr = np.zeros((n_elements,2),dtype='object')
    if n_elements%2 != 0:
        arr[:n_elements//2+1,1] = k1
        arr[n_elements//2+1:,1] = k2
        arr[:,0] = np.arange(n_elements)
    else:
        arr[:n_elements//2,1] = k1
        arr[n_elements//2:,1] = k2
        arr[:,0] = np.arange(n_elements)
    path = os.getcwd()
    if 'mattjwilliams' in path:
        path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
    else:
        path = '/Users/mattwilliams/Documents/PythonProjects/FEM/GangLi/IntroToFEM/Chapter4/HeatTransfer/ProgramFiles'
    fname = 'elprops.txt'
    headers = ['element','kcond']
    with open(os.path.join(path,fname),'w',newline='') as f:
        mywriter = csv.writer(f,delimiter=' ')
        mywriter.writerow(headers)
        mywriter.writerows(arr)
    return None

if __name__ == '__main__':
    ### PREPROCESSING SECTION ###
    # rod is 100 cm long.
    xleft = 0.0
    xright = 100.0
    n_elements = 8
    # degree of basis functions
    deg = 2
    # set my local working directory for retrieving and saving program files
    path = os.getcwd()
    if 'mattjwilliams' in path:
        path = '/Users/mattjwilliams/Documents/PythonStuff/FEM/GangLi/Chapter4/HeatTransfer/ProgramFiles'
    else:
        path = '/Users/mattwilliams/Documents/PythonProjects/FEM/GangLi/IntroToFEM/Chapter4/HeatTransfer/ProgramFiles'
    fnames = ['nodes.txt','elements.txt','global_nodes.txt','bcs.txt','elprops.txt','globals.txt']
    # run functions that create the files containing the mesh information
    mesher1D(xleft,xright,n_elements,deg)
    element_indexer(n_elements,deg)
    global_nodes(xleft,xright,n_elements,deg)
    boundary_conditions(n_elements,[2,0.1,1,0.0],deg)
    properties(n_elements,0.92,0.12)
    # load information from program files into dataframes
    df_dict = {}
    df_names = ['df_nodes','df_elems','df_glob_nodes','df_bcs','df_elprops','df_globals']
    for fname, df_name in zip(fnames,df_names):
        df_dict[df_name] = read_file(fname)

    ### MAIN LOOP ###
    n_nodes = n_elements*deg + 1
    K = np.zeros((n_nodes,n_nodes))
    F = np.zeros(n_nodes)
    Tenv, r, kconv = df_dict['df_globals'].iloc[0]
    for e in range(n_elements):
        node_locs = df_dict['df_nodes'].iloc[e,1:]
        N, Nx, xelem = lagrange(node_locs)
        kcond = df_dict['df_elprops'].iloc[e]['kcond']
        n = len(Nx)
        k = np.zeros((n,n))
        f = np.zeros(n)
        for row in range(n):
            I = 2*kconv*Tenv/r*N[row]
            f[row] = np.trapz(I,x=xelem)
            for col in range(n):
                # This currently only has the first integral on the LHS
                I1 = Nx[row]*Nx[col]*kcond
                I2 = 2*kconv/r*N[row]*N[col]
                k[row,col] = np.trapz(I1,x=xelem) + np.trapz(I2,x=xelem)
        glob_nodes = df_dict['df_elems'].iloc[e,1:]
        for i,row in enumerate(glob_nodes):
            F[row] += f[i]
            for j,col in enumerate(glob_nodes):
                K[row,col] += k[i,j]
    for i in range(n_nodes):
        bc_type = df_dict['df_bcs'].iloc[i]['type']
        if bc_type == 2:
            F[i] += df_dict['df_bcs'].iloc[i]['value']
        if bc_type == 1:
            penalty = abs(K[i,i]+1)*1e7
            K[i,i] = penalty
            F[i] = df_dict['df_bcs'].iloc[i]['value']*penalty
    ### END MAIN LOOP ###

    ### SOLUTION AND POST PROCESSING ###
    temps = np.linalg.solve(K,F)

    sol = []
    xvals = []
    for e in range(n_elements):
        node_locs = df_dict['df_nodes'].iloc[e,1:]
        N, Nx, xelem = lagrange(node_locs)
        val_arr = np.zeros_like(xelem)
        xvals.append(xelem)
        for i in range(deg+1):
            val_arr += N[i]*temps[deg*e+i]
        sol.append(val_arr)

sol_temps = np.stack(sol).flatten()
sol_xvals = np.stack(xvals).flatten()
plt.plot(sol_xvals,sol_temps)
plt.grid()
plt.xlim(sol_xvals[0],sol_xvals[-1])
plt.ylim(0,30)
plt.xlabel('Distance along bar (cm)')
plt.ylabel('Temperature ($^oC$)')
plt.title(f'FEM Solution with {n_elements} Elements and Degree {deg} Interpolants')
plt.show()


from orders import get_model
from flask import Blueprint, redirect, render_template, request, url_for

giftcards_crud = Blueprint('giftcards_crud', __name__)

builtin_list = list

# [START list]
@giftcards_crud.route("/")
def list():
    token = request.args.get('page_token', None)
    if token:
        token = token.encode('utf-8')

    gcs, next_page_token = get_model().gclist(cursor=token)

    return render_template(
        "gclist.html",
        gcs=gcs,
        next_page_token=next_page_token)
# [END list]

# [START view]
@giftcards_crud.route('/<id>')
def view(id):
    gc = get_model().gcread(id)
    return render_template("gcview.html", gc=gc)
# [START view]

# [START create]
@giftcards_crud.route('/add', methods=['GET', 'POST'])
def add():
    if request.method == 'POST':
        data = request.form.to_dict(flat=True)

        gc = get_model().gccreate(data)

        return redirect(url_for('.view', id=gc['id']))

    return render_template("gcform.html", action="Add", gc={})
# [END create]

# [START edit]
@giftcards_crud.route('/<id>/edit', methods=['GET', 'POST'])
def edit(id):
    gc = get_model().gcread(id)

    if request.method == 'POST':
        data = request.form.to_dict(flat=True)

        gc = get_model().gcupdate(data, id)

        return redirect(url_for('.view', id=gc['id']))

    return render_template("gcform.html", action="Edit", gc=gc)
# [END edit]

# [START delete]
@giftcards_crud.route('/<id>/delete')
def delete(id):
    get_model().gcdelete(id)
    return redirect(url_for('.list'))
# [END delete]


import argparse
import datetime

from analyzer.AntiPatterns import find_anti_patterns
from analyzer.config import PROVIDERS, ANTI_PATTERNS
from analyzer.CIDetector import detect_ci_tools
from analyzer.BuildCollector import collect_builds
from analyzer.Output import create_json, create_text_files, create_images
from analyzer.utils import format_date_str
from time import time

# ARGUMENT PARSER

def repository_slug_type(arg):
    parsed = arg.split('/')
    provider_included = len(parsed) > 2

    # provider is included in slug
    if provider_included:
        known_provider = parsed[0] in PROVIDERS
    else:
        known_provider = True

    if known_provider and len(parsed) >= 2:
        return arg
    else:
        raise argparse.ArgumentTypeError(
            f"Unknown provider '{parsed[0]}'" if known_provider else f"Invalid slug '{arg}'"
        )

def provider_type(p):
    if p in PROVIDERS:
        return p
    raise argparse.ArgumentTypeError(
        f"Unknown provider type '{p}' allowed values are '{PROVIDERS}'"
    )

def antipattern_type(a):
    if a in ANTI_PATTERNS:
        return a
    raise argparse.ArgumentTypeError(
        f"Unknown anti-pattern '{a}', allowed values are '{ANTI_PATTERNS}'"
    )

def start_date_type(d):
    try:
        d = datetime.datetime.strptime(d, "%Y-%m-%d")
        return format_date_str(d)
    except ValueError:
        print(f"'{d}' is not in the format YYYY-MM-DD, using no date instead...")
        return None

parser = argparse.ArgumentParser()
parser.add_argument('repository_slugs', nargs='+', type=repository_slug_type,
                    help='One or more repository slugs. A slug is constructed as follows:'
                         '[{provider}/]{username}/{repository_name}'
                         'The provider is optional. If none is given, the default provider is assumed (see -p)')
parser.add_argument('-p', '--default-provider', default='github', type=provider_type,
                    help=f'Default provider. Allowed values are {PROVIDERS}')
parser.add_argument('-do', '--detect-only', action='store_true', default=False,
                    help='Only detect CI tools in the specified repositories')
parser.add_argument('-a', '--anti-patterns', nargs='+', type=antipattern_type,
                    help=f'Select anti-patterns to detect, allowed values are {ANTI_PATTERNS}')
parser.add_argument('-nc', '--no-cache', action='store_true', default=False,
                    help='Use this flag to disable cache usage')
parser.add_argument('-ncc', '--no-create-cache', action='store_true', default=False,
                    help='Use this flag to disable cache creation')
parser.add_argument('-od', '--out-dir', type=str, help='Output path')
parser.add_argument('-v', '--verbose', action='store_true', default=False,
                    help='Provide more information in console')
parser.add_argument('-d', '--start-date', type=start_date_type,
                    help='Date to start collecting data from, if none is provided, the latest three months are '
                         'collected. Date should be formatted as YYYY-MM-DD')

# GENERAL FUNCTIONS

def analyze_repo(
        repo,
        anti_patterns=None,
        detect_only=False,
        use_cache=True,
        create_cache=True,
        verbose=False,
        out_dir=None,
        start_date=None,
        to_date=None
):
    start = time()
    print(f"===============\nStarting analysis on '{repo.path}'\n===============")
    print("Detecting CI...", end='') if verbose else None
    detected = detect_ci_tools(repo, use_cache, create_cache)
    print(f"Done ({round(time() - start, 2)}s)") if verbose else None

    if detect_only:
        print("Detected CI tools: ", detected)
        print(f"==== DONE, TOOK {round(time() - start, 2)}s ====\n")
        return detected

    print("Collecting builds...", end='') if verbose else None
    builds = collect_builds(repo, use_cache, create_cache, start_date, to_date)
    print(f"Done ({round(time() - start, 2)}s)") if verbose else None

    print("Gathering branch information...", end='') if verbose else None
    branch_info = repo.branch_information(use_cache, create_cache)
    default_branch = repo.default_branch
    print(f"Done ({round(time() - start, 2)}s)") if verbose else None

    print("Analyzing anti-patterns...", end='') if verbose else None
    anti_patterns = find_anti_patterns(builds, branch_info, default_branch, restriction=anti_patterns)
    print(f"Done ({round(time() - start, 2)}s)") if verbose else None

    print("Creating output files...", end='') if verbose else None
    if out_dir:
        create_images(anti_patterns, repo, out_dir)
        create_json(anti_patterns, repo, out_dir)
        create_text_files(anti_patterns, repo, out_dir)
    else:
        create_images(anti_patterns, repo)
        create_json(anti_patterns, repo)
        create_text_files(anti_patterns, repo)
    print(f"Done ({round(time() - start, 2)}s)") if verbose else None

    print(f"==== DONE, TOOK {round(time() - start, 2)}s ====\n")
    return detected, anti_patterns, branch_info, default_branch


from flask import (
    Flask,
    render_template,
    request
)
from bs4 import BeautifulSoup
import requests
from urllib.request import Request, urlopen
from flask_cors import CORS
from textblob import TextBlob

# Create the application instance
app = Flask(__name__, template_folder="templates")
CORS(app)

# Create a URL route in our application for "/"
@app.route('/')
def home():
    url = request.args.get('url')
    sauce = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
    soup =BeautifulSoup(urlopen(sauce).read(), 'html.parser')
    result = ""
    for paragraph in soup.find_all('p'):
        result += paragraph.text
    print("request to " + url)

    return result

# If we're running in stand alone mode, run the application
if __name__ == '__main__':
    app.run(debug=True)


from django.contrib import admin
from django.urls import path
from Bank import views

urlpatterns = [
    path("", views.homepage, name='homepage_bank'),
    path("feedback", views.feedback, name='feedback'),
    path("SignUp", views.SignUp, name='SignUp'),
    path("LogIn", views.LogIn, name='LogIn'),
    path('LogIn-Active', views.login_active, name='login-active'),
    path("LogIn-Active/actions", views.deposit, name='deposit'),
    path("LogIn-Active/actions", views.withdraw, name='withdraw'),
    path("LogIn-Active/actions", views.transfer, name='transfer'),
    path("LogIn-Active/register", views.register, name='register'),

]


import pygame, random, sys

class Ball:
    def __init__(self):
        self.x = 380.0
        self.y = 280.0
        self.radius = 20
        self.velocityX = 0.0
        self.velocityY = 0.0
        self.speed = 150.0
    def draw(self, window):
        pygame.draw.circle(window, (255, 255, 255), (int(self.x), int(self.y)), self.radius, 0)
    def update(self, deltaTime, player):
        if self.velocityX == 0 and self.velocityY == 0:
            direction = random.randint(0, 3)
            if direction == 1:
                self.velocityX = -self.speed
                self.velocityY = self.speed
            if direction == 2:
                self.velocityY = self.speed
            if direction == 3:
                self.velocityX = self.speed
                self.velocityY = self.speed
        self._check_bounds(player)
        self.x += self.velocityX*deltaTime
        self.y += self.velocityY*deltaTime
        if (self.y+self.radius) >= 600:
            print("You lost!")
            pygame.quit()
            sys.exit()

    def _check_bounds(self, player):
        if (self.x-self.radius) <= 0:
            self.velocityX = -self.velocityX
        if (self.x+self.radius) >= 800:
            self.velocityX = -self.velocityX
        if (self.y-self.radius) <= 0:
            self.velocityY = -self.velocityY
        if (self.y+self.radius) >= 560:
            if ((self.x+self.radius) >= player.x) and ((self.x-self.radius) <= player.x+150):
                self.velocityY = -self.velocityY


# coding: utf-8
## Author: Jiayi Chen
## Time-stamp:  11/26/2018

import argparse
import time
import math
import os
import torch
import torch.nn as nn
import torch.onnx

parser = argparse.ArgumentParser(description='PyTorch Wikitext-2 RNN/LSTM Language Model')
parser.add_argument('--data', type=str, default='trn-wiki.txt',
                    help='location of the data corpus')
parser.add_argument('--model', type=str, default='LSTM',
                    help='type of recurrent net (RNN_TANH, RNN_RELU, LSTM, GRU)')
parser.add_argument('--emsize', type=int, default=32,
                    help='size of word embeddings')
parser.add_argument('--nhid', type=int, default=32,
                    help='number of hidden units per layer')
parser.add_argument('--nlayers', type=int, default=1,
                    help='number of layers')
parser.add_argument('--lr', type=float, default=0.7,
                    help='initial learning rate')
parser.add_argument('--clip', type=float, default=3,
                    help='gradient clipping')
parser.add_argument('--epochs', type=int, default=50,
                    help='upper epoch limit')
parser.add_argument('--batch_size', type=int, default=16, metavar='N',
                    help='batch size')
parser.add_argument('--bptt', type=int, default=35,
                    help='sequence length')
parser.add_argument('--dropout', type=float, default=0.0,
                    help='dropout applied to layers (0 = no dropout)')
parser.add_argument('--tied', action='store_true',
                    help='tie the word embedding and softmax weights')
parser.add_argument('--seed', type=int, default=1111,
                    help='random seed')
parser.add_argument('--cuda', action='store_true',
                    help='use CUDA')
parser.add_argument('--log-interval', type=int, default=200, metavar='N',
                    help='report interval')
parser.add_argument('--save', type=str, default='model.pt',
                    help='path to save the final model')
parser.add_argument('--onnx-export', type=str, default='',
                    help='path to export the final model in onnx format')
args = parser.parse_args()

class Dictionary(object):
    def __init__(self):
        self.word2idx = {}
        self.idx2word = []

    def add_word(self, word):
        if word not in self.word2idx:
            self.idx2word.append(word)
            self.word2idx[word] = len(self.idx2word) - 1
        return self.word2idx[word]

    def __len__(self):
        return len(self.idx2word)

class Corpus(object):
    def __init__(self, path):
        self.dictionary = Dictionary()
        # self.train = self.tokenize(os.path.join(path, 'train.txt'))
        # self.valid = self.tokenize(os.path.join(path, 'valid.txt'))
        # self.test = self.tokenize(os.path.join(path, 'test.txt'))
        self.train = self.tokenize('trn-wiki.txt')
        self.dev = self.tokenize('dev-wiki.txt')
        self.test = self.tokenize('tst-wiki.txt')
    def tokenize(self, path):
        """Tokenizes a text file."""
        assert os.path.exists(path)
        # Add words to the dictionary
        with open(path, 'r', encoding="utf8") as f:
            tokens = 0
            for line in f:
                words = line.split() + ['<eos>']
                tokens += len(words)
                for word in words:
                    self.dictionary.add_word(word)

        # Tokenize file content
        with open(path, 'r', encoding="utf8") as f:
            ids = torch.LongTensor(tokens)
            token = 0
            for line in f:
                words = line.split() + ['<eos>']
                for word in words:
                    ids[token] = self.dictionary.word2idx[word]
                    token += 1

        return ids

class RNNModel(nn.Module):
    """Container module with an encoder, a recurrent module, and a decoder."""

    def __init__(self, rnn_type, ntoken, ninp, nhid, nlayers, dropout=0.5, tie_weights=False):
        super(RNNModel, self).__init__()
        self.drop = nn.Dropout(dropout)
        self.encoder = nn.Embedding(ntoken, ninp)
        self.rnn = getattr(nn, 'LSTM')(ninp, nhid, nlayers, dropout=dropout)

        self.decoder = nn.Linear(nhid, ntoken)

        # Optionally tie weights as in:
        # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016)
        # https://arxiv.org/abs/1608.05859
        # and
        # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016)
        # https://arxiv.org/abs/1611.01462
        if tie_weights:
            if nhid != ninp:
                raise ValueError('When using the tied flag, nhid must be equal to emsize')
            self.decoder.weight = self.encoder.weight

        self.rnn_type = rnn_type
        self.nhid = nhid
        self.nlayers = nlayers

    def forward(self, input, hidden):
        emb = self.drop(self.encoder(input))#input(seq, batch)    emb(seq, batch, featuresz)

        output, hidden = self.rnn(emb, hidden)# hidden(1, batch, featuresz)     output(seq, batch, featuresz)
        output = self.drop(output)
        decoded = self.decoder(output.view(output.size(0)*output.size(1), output.size(2)))# decoded(seq*batch, featuresz)
        return decoded.view(output.size(0), output.size(1), decoded.size(1)), hidden# decoded(seq, batch, V)

    def init_hidden(self, bsz):
        weight = next(self.parameters())
        if self.rnn_type == 'LSTM':
            return (weight.new_zeros(self.nlayers, bsz, self.nhid),
                    weight.new_zeros(self.nlayers, bsz, self.nhid))
        else:
            return weight.new_zeros(self.nlayers, bsz, self.nhid)

def batchify(data, bsz):
    # Work out how cleanly we can divide the dataset into bsz parts.
    nbatch = data.size(0) // bsz
    # Trim off any extra elements that wouldn't cleanly fit (remainders).
    data = data.narrow(0, 0, nbatch * bsz)
    # Evenly divide the data across the bsz batches.
    data = data.view(bsz, -1).t().contiguous()
    return data.to(device)

def repackage_hidden(h):
    """Wraps hidden states in new Tensors, to detach them from their history."""
    if isinstance(h, torch.Tensor):
        return h.detach()
    else:
        return tuple(repackage_hidden(v) for v in h)

def get_batch(source, i):
    seq_len = min(args.bptt, len(source) - 1 - i)
    data = source[i:i+seq_len]#  [ seq , batchsz ]
    target = source[i+1:i+1+seq_len].view(-1)#  batchsz * seq
    return data, target

def evaluate(data_source):
    # Turn on evaluation mode which disables dropout.
    model.eval()
    total_loss = 0.
    ntokens = len(corpus.dictionary)
    hidden = model.init_hidden(eval_batch_size)
    with torch.no_grad():
        for i in range(0, data_source.size(0) - 1, args.bptt):
            data, targets = get_batch(data_source, i)
            output, hidden = model(data, hidden)
            output_flat = output.view(-1, ntokens)#p(wn|wn−1,…,w1,START)=softmax(Whn−1+b), where W is the weight matrix, hn−1 is the last hidden state from RNN and b is the bias term.
            total_loss += len(data) * criterion(output_flat, targets).item()
            hidden = repackage_hidden(hidden)
    return total_loss / (len(data_source) - 1)

def train():
    # Turn on training mode which enables dropout.
    model.train()
    total_loss = 0.
    start_time = time.time()
    ntokens = len(corpus.dictionary)
    hidden = model.init_hidden(args.batch_size)
    for batch, i in enumerate(range(0, train_data.size(0) - 1, args.bptt)):
        data, targets = get_batch(train_data, i)
        # Starting each batch, we detach the hidden state from how it was previously produced.
        # If we didn't, the model would try backpropagating all the way to start of the dataset.
        hidden = repackage_hidden(hidden)
        model.zero_grad()

        output, hidden = model(data, hidden)

        loss = criterion(output.view(-1, ntokens), targets)

        loss.backward()# call BBTT

        optimizer.step()

        # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
        torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
        for p in model.parameters():
            p.data.add_(-lr, p.grad.data)

        total_loss += loss.item()

        if batch % args.log_interval == 0 and batch > 0:
            cur_loss = total_loss / args.log_interval
            elapsed = time.time() - start_time
            print('| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | '
                    ' cur_avg_loss {:5.2f} | current_perplexity {:8.2f}'.format(
                epoch, batch, len(train_data) // args.bptt, lr,
                 cur_loss, math.exp(cur_loss)))
            total_loss = 0
            start_time = time.time()

def export_onnx(path, batch_size, seq_len):
    print('The model is also exported in ONNX format at {}'.
          format(os.path.realpath(args.onnx_export)))
    model.eval()
    dummy_input = torch.LongTensor(seq_len * batch_size).zero_().view(-1, batch_size).to(device)
    hidden = model.init_hidden(batch_size)
    torch.onnx.export(model, (dummy_input, hidden), path)

# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
    if not args.cuda:
        print("WARNING: You have a CUDA device, so you should probably run with --cuda")

device = torch.device("cuda" if args.cuda else "cpu")

###############################################################################
# Load data
###############################################################################

corpus = Corpus(args.data)

eval_batch_size = args.batch_size
train_data = batchify(corpus.train, args.batch_size)
dev_data = batchify(corpus.dev, eval_batch_size)
test_data = batchify(corpus.test, eval_batch_size)

###############################################################################
# Build the model
###############################################################################

ntokens = len(corpus.dictionary)
model = RNNModel(args.model, ntokens, args.emsize, args.nhid, args.nlayers, args.dropout, args.tied).to(device)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
criterion = nn.CrossEntropyLoss()# no need softmax

###############################################################################
# Training code
###############################################################################
# Loop over epochs.
lr = args.lr
best_val_loss = None

# At any point you can hit Ctrl + C to break out of training early.
try:
    for epoch in range(1, args.epochs+1):
        epoch_start_time = time.time()
        train()
        trn_ppl = evaluate(train_data)
        dev_ppl = evaluate(dev_data)

        print('End of epoch {:3d}  | Perplexity on training data is {:8.2f}'.format(epoch, math.exp(trn_ppl)))

        print('End of epoch {:3d}  | Perplexity on development data is {:8.2f}\n'.format(epoch, math.exp(dev_ppl)))

        # Save the model if the validation loss is the best we've seen so far.
        if not best_val_loss or dev_ppl < best_val_loss:
            with open(args.save, 'wb') as f:
                torch.save(model, f)
            best_val_loss = dev_ppl
        else:
            # Anneal the learning rate if no improvement has been seen in the validation dataset.
            lr /= 4.0
except KeyboardInterrupt:

    print('Exiting from training early')

# Load the best saved model.
with open(args.save, 'rb') as f:
    model = torch.load(f)
    # after load the rnn params are not a continuous chunk of memory
    # this makes them a continuous chunk, and will speed up forward pass
    model.rnn.flatten_parameters()

# Run on test data.
dev_loss = evaluate(dev_data)

print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(
    dev_loss, math.exp(dev_loss)))

if len(args.onnx_export) > 0:
    # Export the model in ONNX format.
    export_onnx(args.onnx_export, batch_size=1, seq_len=args.bptt)


import datetime as dt
from peewee import DataError
from playhouse.postgres_ext import DateTimeTZField

class DateTimeTzField(DateTimeTZField):
    def db_value(self, value):
        if value is not None:
            if value.tzinfo is None:
                raise DataError(f'Cannot use naive datetime "{value}" in DateTimeTzField')
            value = value.astimezone(dt.timezone.utc).replace(tzinfo=None)
        return super(DateTimeTzField, self).db_value(value)

    def python_value(self, value):
        value = super(DateTimeTzField, self).python_value(value)
        if value is not None:
            value = value.replace(tzinfo=dt.timezone.utc)
        return value


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu May 24 23:37:57 2018

@author: pt
"""

for i in range(1,10) :
    print(str(i) + '\r')

from IPython.display import clear_output

for f in range(10):
    clear_output(wait=True)
    print(f)
    time.sleep(.1)

import sys
import time

for f in range(10):
    #delete "\r" to append instead of overwrite
    sys.stdout.write("\r" + str(f))
    sys.stdout.flush()
    time.sleep(.1)


import time
import asyncio
import logging
from config import CONFIG
from proxy import get_proxy
from errors import (
    BadStatusLine, BadResponseError, ErrorOnStream,
    NoProxyError, ProxyRecvError, ProxyTimeoutError)
from utils import parse_headers, parse_status_line

logger = logging.getLogger(__name__)
request_logger = logging.getLogger('proxy_request')

global_requests = []

CONNECTED = b'HTTP/1.1 200 Connection established\r\n\r\n'

class Server:
    """Server distributes incoming requests to its pool of proxies.
    Each instance of this calss is a 'pool' which has proxies.
    TODOs:
    - The pool should at all times have calculated stats about the proxies in its pool

    """

    def __init__(self, host, port, timeout=30, loop=None):
        self.host = host
        self.port = int(port)
        self._loop = loop or asyncio.get_event_loop()
        self._timeout = timeout

        self._server = None
        self._connections = {}

    def start(self):
        srv = asyncio.start_server(
            self._accept, host=self.host, port=self.port, loop=self._loop)
        self._server = self._loop.run_until_complete(srv)

        logger.info('Listening established on {0}'.format(
            self._server.sockets[0].getsockname()))

    def stop(self):
        if not self._server:
            return
        for conn in self._connections:
            if not conn.done():
                conn.cancel()
        self._server.close()
        if not self._loop.is_running():
            self._loop.run_until_complete(self._server.wait_closed())
            # Time to close the running futures in self._connections
            self._loop.run_until_complete(asyncio.sleep(0.5))
        self._server = None
        self._loop.stop()
        logger.info('Server is stopped')

    def _accept(self, client_reader, client_writer):
        def _on_completion(f):
            reader, writer = self._connections.pop(f)
            writer.close()
            logger.debug('client: %d; closed' % id(client_reader))
            try:
                exc = f.exception()
            except asyncio.CancelledError:
                logger.error('CancelledError in server._handle:_on_completion')
                exc = None
            if exc:
                if isinstance(exc, NoProxyError):
                    self.stop()
                else:
                    raise exc
        f = asyncio.ensure_future(self._handle(client_reader, client_writer))
        f.add_done_callback(_on_completion)
        self._connections[f] = (client_reader, client_writer)

    async def _handle(self, client_reader, client_writer):
        logger.debug(f"Accepted connection from {client_writer.get_extra_info('peername')}")

        time_of_request = int(time.time())  # The time the request was requested
        request, headers = await self._parse_request(client_reader)
        scheme = self._identify_scheme(headers)
        client = id(client_reader)
        error = None
        stime = 0
        proxy, pool = await get_proxy(headers['Host'], self.port)
        proto = self._choice_proto(proxy, scheme)
        logger.debug(f'client: {client}; request: {request}; headers: {headers}; '
                     f'scheme: {scheme}; proxy: {proxy}; proto: {proto}')
        try:
            await proxy.connect()

            if proto in ('CONNECT:80', 'SOCKS4', 'SOCKS5'):
                if scheme == 'HTTPS' and proto in ('SOCKS4', 'SOCKS5'):
                    client_writer.write(CONNECTED)
                    await client_writer.drain()
                else:  # HTTP
                    await proxy.send(request)
            else:  # proto: HTTP & HTTPS
                await proxy.send(request)

            stime = time.time()
            stream = [asyncio.ensure_future(self._stream(reader=client_reader, writer=proxy.writer)),
                      asyncio.ensure_future(self._stream(reader=proxy.reader, writer=client_writer, scheme=scheme))
                      ]
            await asyncio.gather(*stream, loop=self._loop)

        except asyncio.CancelledError:
            logger.error('Cancelled in server._handle')
            error = 'Cancelled in server._handle'

        except ErrorOnStream as e:
            logger.error(f'client: {client}; EOF: {client_reader.at_eof()}; Error: {e}')
            for task in stream:
                if not task.done():
                    task.cancel()
            if client_reader.at_eof() and 'Timeout' in repr(e):
                # Proxy may not be able to receive EOF and will raise a
                # TimeoutError, but all the data has already successfully
                # returned, so do not consider this error of proxy
                error = 'TimeoutError'

            if scheme == 'HTTPS':  # SSL Handshake probably failed
                error = 'SSL Error'

        except ProxyTimeoutError:
            logger.error("Proxy timeout")
            error = 'Proxy Timeout'
            # TODO: Send client a 408 status code

        except Exception as e:
            # Catch anything that falls through
            logger.exception("Catch all in server")
            error = repr(e)

        finally:
            proxy.log(request.decode(), stime)
            # At this point, the client has already disconnected and now the stats can be processed and saved
            try:
                if CONFIG.get('Server', {}).get('Log_Requests', True):
                    proxy_url = f'{proxy.host}:{proxy.port}'
                    path = None
                    # Can get path for http requests, but not for https
                    if '/' in headers.get('Path', ''):
                        path = '/' + headers.get('Path', '').split('/')[-1]

                    try:
                        status_code = parse_status_line(stream[1].result().split(b'\r\n', 1)[0].decode()).get('Status')
                    except Exception as e:
                        logger.warning(f"Issue saving status code: proxy={proxy_url}; host={headers.get('Host')}")
                        status_code = None
                        if error is None:
                            error = repr(e)

                    try:
                        proxy_bandwidth_up = len(stream[0].result()) + proxy.stats.get('bandwidth_up', 0)
                        proxy_bandwidth_down = len(stream[1].result()) + proxy.stats.get('bandwidth_down', 0)
                    except Exception:
                        # Happens if something goes wrong with the connection
                        logger.warning(f"Issue saving bandwidth: proxy={proxy_url}; host={headers.get('Host')}")
                        proxy_bandwidth_up = None
                        proxy_bandwidth_down = None

                    request_log = {'host': headers.get('Host'),
                                   'proxy': proxy_url,
                                   'path': path,
                                   'scheme': scheme,
                                   'bw_up': proxy_bandwidth_up,
                                   'bw_down': proxy_bandwidth_down,
                                   'status_code': status_code,
                                   'error': error,
                                   'total_time': proxy.stats['total_time'],
                                   'ts': time_of_request,
                                   'pool_name': pool,
                                   'proxy_port': self.port
                                   }
                    request_logger.info('Request made', extra=request_log)

            except Exception:
                logger.exception("Failed to save request data")

            proxy.close()

    async def _parse_request(self, reader, length=65536):
        request = await reader.read(length)
        headers = parse_headers(request)
        if headers['Method'] == 'POST' and request.endswith(b'\r\n\r\n'):
            # For aiohttp. POST data returns on second reading
            request += await reader.read(length)
        return request, headers

    def _identify_scheme(self, headers):
        if headers['Method'] == 'CONNECT':
            return 'HTTPS'
        else:
            return 'HTTP'

    def _choice_proto(self, proxy, scheme):
        if scheme == 'HTTP':
            if 'CONNECT:80' in proxy.types:
                proto = 'CONNECT:80'
            else:
                relevant = ({'HTTP', 'CONNECT:80', 'SOCKS4', 'SOCKS5'} &
                            proxy.types)
                proto = relevant.pop()
        else:  # HTTPS
            relevant = {'HTTPS', 'SOCKS4', 'SOCKS5'} & proxy.types
            proto = relevant.pop()
        return proto

    async def _stream(self, reader, writer, length=65536, scheme=None):
        checked = False
        total_data = b''
        try:
            while not reader.at_eof():
                data = await asyncio.wait_for(reader.read(length), self._timeout)
                if not data:
                    writer.close()
                    break

                elif scheme and not checked:
                    self._check_response(data, scheme)
                    checked = True

                total_data += data
                writer.write(data)
                await writer.drain()

        except (asyncio.TimeoutError, ConnectionResetError, OSError,
                ProxyRecvError, BadResponseError) as e:
            raise ErrorOnStream(e)

        return total_data

    def _check_response(self, data, scheme):
        if scheme.startswith('HTTP'):
            # Check both HTTP & HTTPS requests
            line = data.split(b'\r\n', 1)[0].decode()
            try:
                parse_status_line(line)
            except BadStatusLine:
                raise BadResponseError