from __future__ import print_function

import numpy as np
import scipy.ndimage as simage
import astropy.io.fits as fits
import astropy.stats as astats
import math as mt
from .instrument import Instrument
#import astropysics.ccd as ccd
import photutils as pu
#import skimage as skim
#from skimage.transform import hough_circle
#from skimage.util import img_as_ubyte
from . import darkbias
from . import flatfield
#from image_registration.register_images import register_images
#from skimage.feature import peak_local_max, canny
import matplotlib.pyplot as plt
import matplotlib.colors as cols
from astropy.convolution import interpolate_replace_nans, Gaussian2DKernel
#import skimage.transform as tf
import pickle
import os
import datetime
from .__init__ import __version__ #hack, replace later

#print(__version__)
eso_fits_keywords = ["ORIGIN",
                     "DATE",
                     "TELESCOP",
                     "INSTRUME",
                     "OBJECT",
                     "RA",
                     "DEC",
                     "EQUINOX",
                     "RADECSYS",
                     #EXPTIME
                     #MJD-OBS
                     #DATE-OBS
                     #UTC
                     #LST
                     "ESO DPR CATG",
                     "ESO DPR TECH",
                     "ESO DPR TYPE",
                     "ESO INS COMB ICOR",
                     "ESO INS COMB IFLT",
                     "ESO INS COMB POLA",
                     "ESO INS COMB VCOR",
                     "ESO INS DATE",
                     "ESO INS1 FILT ID",
                     "ESO INS1 FILT NAME",
                     "ESO INS1 FILT NO",
                     "ESO INS1 ID",
                     "ESO INS1 MODE",
                     "ESO INS1 OPTI1 ID",
                     "ESO INS1 OPTI1 NAME",
                     "ESO INS1 OPTI1 NO",
                     "ESO INS1 OPTI1 TYPE",
                     "ESO INS1 OPTI2 ID",
                     "ESO INS1 OPTI2 NAME",
                     "ESO INS1 OPTI2 NO",
                     "ESO INS1 OPTI2 TYPE",
                     "ESO OBS ID",
                     "ESO OBS NAME",
                     "ESO OBS NTPL",
                     "ESO OBS PROG ID",
                     "ESO OBS START",
                     "ESO SEQ ARM",
                     "ESO SEQ CORO CPI_ND",
                     "ESO SEQ CORO DATE",
                     "ESO SEQ CORO XC",
                     "ESO SEQ CORO YC",
                     "ESO TEL ALT",
                     "ESO TEL AMBI RHUM",
                     "ESO TEL AMBI TAU0",
                     "ESO TEL AMBI TEMP",
                     "ESO TEL AMBI WINDDIR",
                     "ESO TEL AMBI WINDSP",
                     "ESO TEL AZ",
                     "ESO TEL DATE",
                     "ESO TEL DID",
                     "ESO TEL GEOELEV",
                     "ESO TEL GEOLAT",
                     "ESO TEL GEOLON",
                     "ESO TEL IA FWHM",
                     "ESO TEL IA FWHMLIN",
                     "ESO TEL IA FWHMLINOBS",
                     "ESO TEL ID",
                     "ESO TEL MOON DEC",
                     "ESO TEL MOON RA",
                     "ESO TEL OPER",
                     "ESO TEL TARG ALPHA",
                     "ESO TEL TARG COORDTYPE",
                     "ESO TEL TARG DELTA",
                     "ESO TEL TARG EPOCH",
                     "ESO TEL TARG EPOCHSYSTEM",
                     "ESO TEL TARG EQUINOX",
                     "ESO TEL TARG PARALLAX",
                     "ESO TEL TARG PMA",
                     "ESO TEL TARG PMD",
                     "ESO TEL TARG RADVEL",
                     "ESO TEL TH M1 TEMP",
                     "ESO TEL TRAK STATUS",
                     "ESO TPL DID",
                     "ESO TPL EXPNO",
                     "ESO TPL ID",
                     "ESO TPL NAME",
                     "ESO TPL NEXP",
                     "ESO TPL PRESEQ",
                     "ESO TPL START",
                     "ESO TPL VERSION",
                     "PIXSCAL"
                     #ARCFILE

                     ]

#need a list of filters, conoragraphs, etc
class Irdis(Instrument):
    def __init__(self, observation,#,scifiles=None,badpixfile=None,skyfiles=None,
                 #flatfiles=None,darkfiles=None,mode=None,optics=None,
                 options=None, interProd=False,prefix="", **kwargs):
        self.__version__ = __version__
        self.science=observation.obs_main
        self.datadir=observation.datadir
        self.mode=observation.obs_type
        self.prefix=prefix
        self.darks=observation.cals['IRD_DARK']
        if 'CLI' in self.mode:
            self.flats=observation.cals['IRD_CLI_FLAT']
            try:
                self.sky=observation.cals['IRD_SCI_CLI_SKY']  #or does this need to be 'IRD_SKY_BG_RAW'? #Sky
            except KeyError:
                try:
                    self.sky=observation.cals['IRD_SKY_BG_RAW']  #or does this need to be 'IRD_SKY_BG_RAW'? #Sky
                except KeyError:
                    print("No sky frame found. Please check your association tree contains all required calibrations")
                    print("Do not continue with reduction!")
                    #return
        elif 'DBI' in self.mode:
            self.flats=observation.cals['IRD_DBI_FLAT']
            try:
                self.sky=observation.cals['IRD_SCI_CLI_SKY']  #or does this need to be 'IRD_SKY_BG_RAW'? #Sky
            except KeyError:
                try:
                    self.sky=observation.cals['IRD_SKY_BG_RAW']  #or does this need to be 'IRD_SKY_BG_RAW'? #Sky
                except KeyError:
                    print("No sky frame found. Please check your association tree contains all required calibrations")
                    print("Do not continue with reduction!")
                    #return

        try:
            self.flux=observation.cals['IRD_FLUX_CALIB_CORO_RAW'] #flux
        except KeyError:
            self.flux=None #flux observations were not taken (e.g. this may be a reference psf)
#        self.scifiles=scifiles
#        self.badpixfile=badpixfile
#        self.skyfiles=skyfiles
#        self.flatfiles=flatfiles
#        self.flatcals
#        self.darkfiles=darkfiles
#        self.mode=mode
#        self.optics=optics
        self.options=options
        self.interProd=interProd #write out intermediate products? (master dark, etc)
        #calibration angles
        self.pupilOffset=-135.99 #Pupil offset angle
        self.tn=-1.764 #True North offset
        #astrometric calibration numbers - probably won't use these and just say that pipeline is not suitable for high-precision astrometry
        self.plateScale=12.251 #IRDIS plate scale (after distortion correction) of irdis detector
        self.Anamorph=1.0062 #anamophic distortion multiplier for Y direction of detector
        self.logfile = open(str(datetime.datetime.now()).split(' ')[0]+"_"+prefix+"_precision.log",'wb+')

    def __repr__(self):
        return "<IRDIS mode: %s science data: %s sky: %s flux: %s flats: %s darks: %s options: %s>" % (self.mode, self.science, self.sky, self.flux, self.flats, self.darks)

    def __str__(self):
        return "IRDIS mode: %s science data: %s sky: %s flux: %s flats: %s darks: %s options: %s" % (self.mode, self.science, self.sky, self.flux, self.flats, self.darks, self.options)

    def findSources(self,sourcefinder,data,window,guess,median,interact=True,**kwargs):
        sources=sourcefinder.find_stars(data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,
                                              np.int(guess[1])-window:np.int(np.ceil(guess[1]))+window] - median)
        sources.sort('flux')
        if interact:
            sources.pprint(max_lines=-1)
            plt.imshow(data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,
                            np.int(guess[1])-window:np.int(np.ceil(guess[1]))+window] - median,
                       origin='lower')
            plt.show(block=False)
            try:
                a=raw_input('What is the ID of the source at the centre of rotation?\n')
            except NameError:
                a=input('What is the ID of the source at the centre of rotation?\n')
            plt.close()
            if int(a) < 0:
                sources,new_window=self.findSources(sourcefinder,data,int(1.5*window),guess,median)
            else:
                b=(sources['id'] == int(a))
                return sources[:][b],window
        return sources,window#sources

    def findcentre(self,data,guess,interact=True,window=60,**kwargs):
        print(data.shape,guess)
        print(np.int(guess[0]))
        print(np.int(np.ceil(guess[1])))
        #help(np.int(np.ceil(guess[1])) + window)
        #help(data)
        mean,median,std=astats.sigma_clipped_stats(data,sigma=3.0)
        sourcefinders=pu.DAOStarFinder( #data[500:550,450:500] - median, # assumes datacube #[500:550,450:500]
                           fwhm = 2.0,
                           threshold=std#,
                           #sharplo=0.3,
                           #sharphi=0.5,
                           #roundhi=0.3,
                           #roundlo=-0.3
                           )
        #sources
        #b=
        self.central,new_window=self.findSources(sourcefinders,data,window,guess,median,interact)
#        sources.sort('flux')#'peak') #take brightest source
        #try making this interactive?
        #if interact:
#            print sources
        #    sources.pprint(max_lines=-1)
        #    plt.imshow(data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,
        #                    np.int(guess[1])-window:np.int(np.ceil(guess[1]))+window] - median)
        #    plt.show(block=False)
            #try:
        #    a=raw_input('What is the ID of the source at the centre of rotation?\n')
            #except ValueError:
        #    if int(a) < 0:
        #        sources=self.findSources(sourcefinders,data,int(1.5*window),guess,median)
        #        sources.pprint(max_lines=-1)
        #        plt.imshow(data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,
        #                        np.int(guess[1])-window:np.int(np.ceil(guess[1]))+window] - median)
        #        plt.show(block=False)
                #try:
        #        a=raw_input('What is the ID of the source at the centre of rotation?\n')
                #            print a,sources.dtype
        #    b=(sources['id'] == int(a))
#            print b
#            print sources[b]
            #try:
                #self.central.append(sources[:][b])
            #except:
        #self.central=sources[:][b]
        #    plt.close()
        #try:
        #    self.central=sources[:][-1]
        #except:
        #    self.central=sources
#        self.central=central
        #self.sources=sources
        #return central,sources
        #print
        self.centre=np.array([self.central['xcentroid'].data+(np.int(guess[1])-new_window),self.central['ycentroid'].data+(np.int(guess[0])-new_window)])
#        pass
        #how on earth do I do this? try finding the coronagraph?
        #in case of ALC look for central spot using photutils
        #else use scikit-image to find circles for CLC
 #       tempimg=img_as_byte(frame)
 #       edges=canny(tempimg)
 #       hough_radii=1 #needs to depend on coronagraph choice
 #       hough_res=hough_circle(edges,hough_radii)
        #now need to use info to align both cubes
        #return centres of circles

    def splitchannels(self,data,**kwargs):
        #call this after dark/flat/sky correction and centre finding but before anything else - will probably end up most used...
        return data[:,:,0:1024],data[:,:,1024:2048]

    def derotate(self,data,angle,**kwargs):
        """ Function to derotate images. Assumes images are already centred """
        #derotate
        return simage.interpolation.rotate(data,angle,reshape=False)

    def CI(self,**kwargs): #test on VY CMa
        print("Combining images for CI")
        self.finalNoDerot=np.mean(self.medianNoDerot,axis=0) #take median of all images at each pixel
        #self.finalvarNoDerot=np.mean(self.varNoDerot,axis=0)         #small number approximation -
        self.finalvarNoDerot=(np.mean(self.varNoDerot,axis=0) / #variance on mean = mean of variances / N
                              (self.varNoDerot.shape[0]))#(2*self.varNoDerot.shape[0]))          #or alternatively
                                                                     #uncertainty on mean = mean of uncertainties / sqrt(N)
        #need to fix uncertainty tracking...current version should underestimate variances, only alteratives (apart from MC) appear to overestimate them
        if self.rot=='PUPIL':
            self.finalDerot=np.mean(self.medianDerot,axis=0) #take median of all derotated images at each pixel
            #self.finalVarDerot=np.mean(self.varDerot,axis=0)         #small number approximation -
            self.finalVarDerot=(np.mean(self.varDerot,axis=0) / #variance on mean = mean of variances / N
                                  (self.varDerot.shape[0]))          #or alternatively
        return

    def SDI(self,**kwargs): #test on GD50, VY CMa
        self.CI()
        print("Combining images for SDI")
        self.SDINoDerot=np.mean(self.medianNoDerot,axis=0) #produces L- and R- channel images
        self.SDINoDerot=self.SDINoDerot[1]-self.SDINoDerot[0] #check which way round this is supposed to be! and scaling!

        self.varSDINoDerot=np.mean(self.varNoDerot,axis=0) / self.varNoDerot.shape[0]
        self.varSDINoDerot=np.sum(self.varSDINoDerot, axis=0)

        if self.rot=='PUPIL':
            self.SDIDerot=np.mean(self.medianDerot,axis=0)
            self.SDIDerot=self.SDIDerot[1]-self.SDIDerot[0]
            self.varSDIDerot=np.mean(self.varDerot,axis=0) / self.varDerot.shape[0]
            self.varSDIDerot=np.sum(self.varSDIDerot,axis=0)
        return

    def ADI(self,**kwargs): #test on GD50
        """ This method perfoms classical ADI by taking the median non-derotated frame and subtracting it from each non-derotated frame. It then derotates all frames and collapses the whole cube """
        self.medianADI=np.array([])
        self.varADI=np.array([])
        isci=0
        for f in self.scifiles:
            #read data
            data,header=self.readdata(f)
            data=self.badpixcorrect(data)
            data=self.skysub(data,self.mastersky)
            #undither data
            data=simage.interpolation.shift(data,[header['HIERARCH ESO INS1 DITH POSX'], #check sign
                                                  header['HIERARCH ESO INS1 DITH POSY'] #of shifts
                                                  ]
                                            )
            #split channels and align
            datal,datar=self.splitchannels(data)
            #Centres were found before by CI algorithm, so shift each cube
            datal=simage.interpolation.shift(datal,
                                             np.array([0,
                                                       self.shiftl[isci][1][0],
                                                       self.shiftl[isci][0][0]
                                             ])
            )
            datar=simage.interpolation.shift(datar,
                                             np.array([0,
                                                       self.shiftr[isci][1][0],
                                                       self.shiftr[isci][0][0]
                                             ])
            )
            #subtract speckle image (non-derotated median frames)
            datal=datal-self.finalNoDerot
            datar=datar-self.finalNoDerot
            #derotate channels
            #then calculate rotation as a function of time
            self.parang=[header['HIERARCH ESO TEL PARANG START'],header['HIERARCH ESO TEL PARANG START']]
            self.pdelt=(self.parang[1]-self.parang[0])/data.shape[0]
            for i in range(data.shape[0]):
                angle=-1.*self.parang[0]-self.parangInit + (i+0.5)*self.pdelt #rotation angle at centre of exposure relative to beginning of entire sequence - add absolute rotations as well!
                    #then derotate each frame of each half of the detector
                datal[i,:,:]=self.derotate(datal[i,:,:],angle)

            #then calculate rotation as a function of time
            self.parang=[header['HIERARCH ESO TEL PARANG START'],header['HIERARCH ESO TEL PARANG START']]
            self.pdelt=(self.parang[1]-self.parang[0])/data.shape[0]
            for i in range(data.shape[0]):
                angle=-1.*self.parang[0]-self.parangInit + (i+0.5)*self.pdelt #rotation angle at centre of exposure relative to beginning of entire sequence - add absolute rotations as well!
                    #then derotate each frame of each half of the detector
                datar[i,:,:]=self.derotate(datar[i,:,:],angle)
            #now build ADI medians for each dither position
            try:
                self.medianADI=np.r_[self.medianADI,
                                     [np.nanmedian(datal,axis=0),np.nanmedian(datar,axis=0)]
                ]
                self.varADI=np.r_[self.varADI,
                                  [(np.pi/(2.*datal.shape[0]))*(np.std(datal,axis=0))**2,
                                   (np.pi/(2.*datar.shape[0]))*(np.std(datar,axis=0))**2
                                  ]
                ]
            except:
                self.medianADI=np.array([np.nanmedian(datal,axis=0),np.nanmedian(datar,axis=0)])
                self.varADI = np.array(
                                       [
                                           (np.pi/(2.*datal.shape[0]))*(np.std(datal,axis=0))**2,
                                           (np.pi/(2.*datar.shape[0]))*(np.std(datar,axis=0))**2
                                        ]
                                       )
            isci+=1
        #average all frames ('cADI')
        self.finalADI=np.mean(self.medianADI,axis=0) #produces L- and R- channel images and combines
        #self.finalADI=np.mean(self.finalADI,axis=0) #combine both channels
        #self.finalVarADI=np.mean(self.varADI,axis=0)         #small number approximation -
        self.finalVarADI=(np.mean(self.varADI,axis=0) / #variance on mean = mean of variances / N
                            (self.varADI.shape[0]))

        if mode=='SDI':
            self.finalSADI=np.mean(self.medianSADI[0:-1:2,:,:],axis=0) - np.mean(self.medianSADI[1::2,:,:],axis=0)    #produces L- and R- channel images and subtracts them
            #self.finalSADI=self.finalSADI[1]-self.finalSADI[0]#
            self.finalVarSADI=np.mean(self.varSADI,axis=0)    #small number approximation -
            self.finalVarSADI=np.sum(self.finalVarSADI,axis=0)#variance on mean = mean of variances / N
        pass

    def LOCI(self,**kwargs):
        raise NotImplementedError("LOCI is not implemented yet.")#        pass

    def DPI(self,**kwargs): #test on HR 3090
        self.CI()

        #split L and R channels into O- and E-rays respectively
        self.MeanOray=np.mean(self.medianNoDerot[:,0,:,:],axis=0) #add an additional axis for DPI observations, so that it contains [+Q, -Q, +U, -U] or as many as are available
        self.MeanEray=np.mean(self.medianNoDerot[:,1,:,:],axis=0)# self.splitchannels()

        #calculate Stokes parameter(s) the old fashioned way
        #concatenate channels and collapse for I
        self.I=self.MeanOray + self.MeanEray

        #O - E for Q/U
        self.pol=self.MeanOray - self.MeanEray #this all needs updating to enable it to process an entire DPI observation, correctly interpreting what combinations of +/- Q/U to do

        #if double difference, do +Q - -Q

        #compute fractional pol

        #do ratio method too

        pass

    def makesky(self,**kwargs):
        print(self.sky)
        if len(self.sky.obs_main) == 1:
            f=self.sky.datadir+self.sky.obs_main[0][:][1]+'.fits'
            data,header=self.readdata(f)
            data=self.badpixcorrect(data)
            self.skyframes=data
        else:
            for f in self.sky.obs_main[0][:][1]:
                f=self.sky.datadir+f+'.fits'
                data,header=self.readdata(f)
                data=self.badpixcorrect(data)
                self.skyframes=np.r_[self.skyframes,data]
        #now median combine all sky frames
        if (len(self.skyframes.shape) == 3):
            self.mastersky=np.nanmedian(self.skyframes,axis=0)
            self.skyvar=(np.pi/(2.*self.skyframes.shape[0]))*(np.std(self.skyframes,axis=0))**2
            self.skyfiles=None
        else:
            self.mastersky=self.skyframes
            self.skyvar=self.mastersky
        pass

    def reduce(self,**kwargs):
        self.status=0
        if self.darks is not None:
            self.masterdark,self.darkvar,self.RON=darkbias.makemasterdark(self.darks,**kwargs)
        else:
            self.masterdark=None

        if self.flats is not None:
            self.masterflat,self.flatvar,self.badpixmap=flatfield.makemasterflat(self.flats,**kwargs)#,self.masterdark,**kwargs)
        else:
            self.masterflat=None
        if self.sky is not None:
            self.makesky()
        if self.flux is not None:
            self.sciFlux()
            #exit()

        #print np.sum(np.isnan(self.mastersky))
        self.medianNoDerot=np.array([])
        self.medianDerot=np.array([])
        self.varNoDerot=np.array([])
        self.varDerot=np.array([])
        self.headers=[]#np.array([])
        isci=-1
        print(self.science[0][1])
        #exit()
        self.centrel=[]
        self.centrer=[]
        self.shiftl=[]
        self.shiftr=[]
        for f in self.science:
            isci+=1
            f=self.datadir+f[1]+'.fits'
            #read data
            data,header=self.readdata(f)
            #print data
            if isci==0:
                #pull important info out of header from first science file
                self.rot=header['HIERARCH ESO INS4 COMB ROT']
                self.parangInit=header['HIERARCH ESO TEL PARANG START']
                self.optics={'filt': [header['HIERARCH ESO INS1 FILT NO'],header['HIERARCH ESO INS1 FILT ID'],header['HIERARCH ESO INS1 FILT NAME']],'opti':[header['HIERARCH ESO INS1 OPTI2 NO'],header['HIERARCH ESO INS1 OPTI2 ID'],header['HIERARCH ESO INS1 OPTI2 NAME']],'stop': [header['HIERARCH ESO INS1 OPTI1 NO'],header['HIERARCH ESO INS1 OPTI1 ID'],header['HIERARCH ESO INS1 OPTI1 NAME']]}
                #coros and stops could be in IRDIS (INS1) or in CPI (INS4)
                pass
            #intermediate processing
            data=self.badpixcorrect(data)
            #print data
            #print np.sum(np.isfinite(data))
            data=self.skysub(data,self.mastersky)
            #print data
            #print np.sum(np.isfinite(data))
            mask=np.logical_not(np.isfinite(data))
            data[mask]=np.nan
            if (len(data.shape) == 3): #datacube
                for i in range(data.shape[0]):
                    data[i,:,:] = interpolate_replace_nans(data[i,:,:],Gaussian2DKernel(stddev=1))
            else:
                data[:,:] = interpolate_replace_nans(data[:,:],Gaussian2DKernel(stddev=1))
            #split channels and align
            datal,datar=self.splitchannels(data)

            #----------------------------NOW THINGS ARE DIFFERENT DEPENDING ON METHOD!!--------------------------
            #Undithering data:
            try:
                self.medianNoDerot=np.r_[self.medianNoDerot,
                                         [simage.interpolation.shift(np.nanmedian(datal,axis=0),
                                                                     #[0,
                                                                     [header['HIERARCH ESO INS1 DITH POSX'],
                                                                      header['HIERARCH ESO INS1 DITH POSY']
                                                                     ]
                                         ),
                                          simage.interpolation.shift(np.nanmedian(datar,axis=0),
                                                                     #[0,
                                                                     [header['HIERARCH ESO INS1 DITH POSX'],
                                                                      header['HIERARCH ESO INS1 DITH POSY']])
                                         ]
                                        ] #make sure the sign is right here (by inspection!!)
                self.varNoDerot=np.r_[self.varNoDerot,
                                      [simage.interpolation.shift((np.pi/(2.*datal.shape[0]))*(np.std(datal,axis=0))**2,
                                                                  #[0,
                                                                  [header['HIERARCH ESO INS1 DITH POSX'],
                                                                   header['HIERARCH ESO INS1 DITH POSY']]),
                                       simage.interpolation.shift((np.pi/(2.*datar.shape[0]))*(np.std(datar,axis=0))**2,
                                                                  #[0,
                                                                  [header['HIERARCH ESO INS1 DITH POSX'],
                                                                   header['HIERARCH ESO INS1 DITH POSY']])
                                      ]
                                     ]
            except:
                self.medianNoDerot=np.array(
                                         [simage.interpolation.shift(np.nanmedian(datal,axis=0),
                                                                     #[0,
                                                                     [header['HIERARCH ESO INS1 DITH POSX'],
                                                                      header['HIERARCH ESO INS1 DITH POSY']
                                                                     ]
                                         ),
                                          simage.interpolation.shift(np.nanmedian(datar,axis=0),
                                                                     #[0,
                                                                     [header['HIERARCH ESO INS1 DITH POSX'],
                                                                      header['HIERARCH ESO INS1 DITH POSY']])
                                         ]
                                        ) #make sure the sign is right here (by inspection!!)
                self.varNoDerot=np.array(
                                      [simage.interpolation.shift((np.pi/(2.*datal.shape[0]))*(np.std(datal,axis=0))**2,
                                                                  #[0,
                                                                   [header['HIERARCH ESO INS1 DITH POSX'],
                                                                   header['HIERARCH ESO INS1 DITH POSY']]),
                                       simage.interpolation.shift((np.pi/(2.*datar.shape[0]))*(np.std(datar,axis=0))**2,
                                                                 # [0,
                                                                  [header['HIERARCH ESO INS1 DITH POSX'],
                                                                   header['HIERARCH ESO INS1 DITH POSY']])
                                      ]
                                     )

            #plt.imshow(self.medianNoDerot[0,:,:])
            print(np.max(self.medianNoDerot))
            #plt.show()
            #exit()
            self.headers.append(header)
            #print header
            print(self.medianNoDerot.shape)
            #if isci == 0:
            centreGuess=[header['HIERARCH ESO SEQ CORO XC'],header['HIERARCH ESO SEQ CORO YC']]
            self.findcentre(self.medianNoDerot[isci*2],centreGuess,interact=True)
            self.centrel.append(self.centre)#[self.central['xcentroid'],self.central['ycentroid']]
            #centreGuess.append([header['HIERARCH ESO SEQ CORO XC'],header['HIERARCH ESO SEQ CORO YC']])
            self.findcentre(self.medianNoDerot[isci*2+1],centreGuess,interact=True)
            self.centrer.append(self.centre)#[self.central['xcentroid'],self.central['ycentroid']]
            print(self.centrel[isci],self.centrer[isci])
            self.shiftl.append(np.array(511 - self.centrel[isci]))
            self.shiftr.append(np.array(511 - self.centrer[isci]))
            print(self.shiftl[isci],self.shiftl[isci][0],self.shiftl[isci][1][0])
            #tform=tf.SimilarityTransform(scale=1,rotation=0,translation=(self.shiftl[isci][1],self.shiftl[isci][0]))
            self.medianNoDerot[isci*2]=simage.interpolation.shift(self.medianNoDerot[isci*2],np.array([self.shiftl[isci][1][0],self.shiftl[isci][0][0]]))#tf.warp(self.medianNoDerot[isci*2 -1],tform)# [yshift, xshift]
            self.medianNoDerot[isci*2+1]=simage.interpolation.shift(self.medianNoDerot[isci*2 + 1],np.array([self.shiftr[isci][1][0],self.shiftr[isci][0][0]]))
            self.varNoDerot[isci*2]=simage.interpolation.shift(self.varNoDerot[isci*2],np.array([self.shiftl[isci][1][0],self.shiftl[isci][0][0]]))#tf.warp(self.medianNoDerot[isci*2 -1],tform)# [yshift, xshift]
            self.varNoDerot[isci*2+1]=simage.interpolation.shift(self.varNoDerot[isci*2 + 1],np.array([self.shiftr[isci][1][0],self.shiftr[isci][0][0]]))
            print(isci*2,isci*2+1)
            temp=self.medianNoDerot[isci*2]+self.medianNoDerot[isci*2+1]

            #now derotate cube if pupil stabilised
            if self.rot=='PUPIL':
                #first find centre of rotation

                #then calculate rotation as a function of time
                self.parang=[header['HIERARCH ESO TEL PARANG START'],header['HIERARCH ESO TEL PARANG START']]
                self.pdelt=(self.parang[1]-self.parang[0])/data.shape[0]
                #centre cube:
                datal=simage.interpolation.shift(datal,
                                                np.array(
                                                    [
                                                        0,
                                                        self.shiftl[isci][1][0],
                                                        self.shiftl[isci][0][0]
                                                    ]
                                                 )
                                                )
                #shift data to common reference frame
                for i in range(data.shape[0]):
                    angle=-1.*self.parang[0] + (i+0.5)*self.pdelt + self.pupilOffset + self.tn #rotation angle at centre of exposure relative to beginning of entire sequence - add absolute rotations as well! ##-self.parangInit is not required
                    #then derotate each frame of each half of the detector
                    datal[i,:,:]=self.derotate(datal[i,:,:],angle) #simage.interpolation.rotate(datal[i,:,:],angle,reshape=False) #somehow I must be able to pass in the centre of rotation...I guess I could also shift it so that it is centred correctly first.
                #then calculate rotation as a function of time
                self.parang=[header['HIERARCH ESO TEL PARANG START'],header['HIERARCH ESO TEL PARANG START']]
                self.pdelt=(self.parang[1]-self.parang[0])/data.shape[0]
                datar=simage.interpolation.shift(datar,
                                                np.array(
                                                    [
                                                        0,
                                                        self.shiftr[isci][1][0],
                                                        self.shiftr[isci][0][0]
                                                    ]
                                                 )
                                                )
                for i in range(data.shape[0]):
                    angle=-1.*self.parang[0] + (i+0.5)*self.pdelt + self.pupilOffset + self.tn #rotation angle at centre of exposure relative to beginning of entire sequence - add absolute rotations as well! ##-self.parangInit is not required
                    #then derotate each frame of each half of the detector
                    datar[i,:,:]=self.derotate(datar[i,:,:],angle)
                try:
                    self.medianDerot=np.r_[self.medianDerot,
                                           [np.nanmedian(datal,axis=0),
                                            np.nanmedian(datar,axis=0)]
                                          ]

                    self.varDerot=np.r_[self.varDerot,
                                        [(np.pi/(2.*datal.shape[0]))*(np.std(datal,axis=0))**2,
                                         (np.pi/(2.*datar.shape[0]))*(np.std(datar,axis=0))**2]
                                        ]
                except:
                    self.medianDerot=np.array([np.nanmedian(datal,axis=0),
                                               np.nanmedian(datar,axis=0)
                                                 ]
                                              )
                    self.varDerot=np.array(
                                           [(np.pi/(2.*datal.shape[0]))*(np.std(datal,axis=0))**2,
                                            (np.pi/(2.*datar.shape[0]))*(np.std(datar,axis=0))**2]
                                          )

        if self.mode=='IRD_SCI_CLI_OBJ':#'CI':
            self.CI()
        elif self.mode=='DPI':
            self.DPI()
        elif self.mode=='IRD_SCI_DBI_OBJ':
            self.SDI()
        else:
            print('IRDIS mode not recognised')

        self.writeFinalProducts()
        return self.status

    def writeFinalProducts(self,**kwargs):
        """ Write out final data reduction products to single multi-extension fits file and to separate files """
        hdu=fits.PrimaryHDU()
        for key in eso_fits_keywords: #add important keywords from observed data
            hdu.header.append((key,self.headers[0][key]))
        hdu.header.append(("Pipeline","precision"))
        hdu.header.append(("PipeAuth","P. Scicluna"))
        hdu.header.append(("PipeVers",__version__))
        if self.flux is not None:
            hdu.header.append(("Flux factor",self.flux.flux,"Divide images by this amount to convert to contrast"))
        else:
            hdu.header.append(("Flux factor",1,"No flux frames observed, no conversion necessary"))
        #hdu.header.append(("COMMENT",self.__str__().encode("ascii"))) #full list of files and associations used to produce this dataset - fix in future versions to produce a more useful list which can be used to reconstruct the processing without the association tree ... fix later...
        hdulist=fits.HDUList(hdu)
        """ CI outputs are always required because it always gets done  """
        hdu=fits.ImageHDU(self.finalNoDerot,name="Image (No Derotation)")
        hdulist.append(hdu)
        #How should this be structured?
        #if self.mode=='IRD_SCI_CLI_OBJ':#'CI':
            #CI - required outputs, finalNoDerot,finalDerot,ADI,fluxcal,photcal, uncertainty frames
        hdu=fits.ImageHDU(self.finalvarNoDerot, name="variance (no derotation)")
        hdulist.append(hdu)
        if self.rot=='PUPIL':
            #Derotated images
            hdulist.append(fits.ImageHDU(self.finalDerot,name="Image (derotated)"))
            hdulist.append(fits.ImageHDU(self.finalVarDerot, name="variance (derotated)"))
            #pass
        elif self.mode=='DPI':
            """ Special outputs just for DPI """
            pass
        elif self.mode=='IRD_SCI_DBI_OBJ':
            """ Special outputs just for DBI """
            hdulist.append(self.SDINoDerot, name='SDI')
            hdulist.append(self.varSDINoDerot, name='var SDI')
            if self.rot=='PUPIL':
                hdulist.append(self.SDIDerot, name='SDI derotated')
                hdulist.append(self.varSDIDerot, name='var SDI derotated')
            pass
        logtime = str(datetime.datetime.now()).replace(" ","_")
        hdulist.writeto(self.prefix+"_"+logtime+"_prod.fits")
        pass

    def writeInterProd(self,**kwargs):
        """ Write out intermediate products """
        #What intermediate products are required?
        #Master dark + unc
        #Master flat + unc
        #Bad-pixel map
        #Dark/flat corrected cubes?
        #Bad-pixel corrected cubes?
        #Derotated cubes?
        #Collapsed/aligned cubes?
        #Other calibrations (Flux frames, phot.cals. etc) after calibration
        #Try to make this flexible where you pass info in and it gets turned into a fits file
        #hdulist.writeto(self.outfile+"_interprod.fits")
        pass

    def sciFlux(self,**kwargs):
        '''
        Take non-coronographic observation to compute contrast and facilitate flux calibration
        '''
        #read flux file and calibrate it
        filename=self.flux.datadir+self.flux.obs_main[0][:][1]+".fits"
        fluxframes,fluxhdr=self.readdata(filename)

        #scale for ND filters #and exposure time
        filt=fluxhdr['HIERARCH ESO INS COMB IFLT'].split('_')[1].strip(' ') #self.optics['filt'][2].split('_')[1].strip(' ')#
        ND=fluxhdr['HIERARCH ESO INS4 FILT2 NAME'].split('_')[1].strip(' ')
        print(ND)
        ND=np.int(np.float64(ND))
        #fluxframes=fluxframes * 10**(ND) #very approximate, try to get the ND transmission curves.
        #Scrape filter curves from https://www.eso.org/sci/facilities/paranal/instruments/sphere/inst/filters/*.dat
        #then construct look-up table with approximate throughput of each filter/ND combination
        #read in lookup table
        path=os.path.abspath(__file__)
        print(os.path.dirname(path))
        with open(os.path.dirname(path)+"/IRDIS_filter_throughputs.pkl","rb") as f:
            tab=pickle.load(f, encoding='latin1', fix_imports=True)
            tp=tab[filt][ND]
            print(tp)
            fluxframes=fluxframes / tp
        #extract point-source counts and peak counts for target
        #check for multiple frames:
        if len(fluxframes.shape) == 3:
            masterflux=np.median(fluxframes,axis=0)
        else:
            masterflux=fluxframes

        mean,median,std=astats.sigma_clipped_stats(masterflux,sigma=3.0)
        sourcefinders=pu.DAOStarFinder( #data[500:550,450:500] - median, # assumes datacube #[500:550,450:500]
                           fwhm = 2.0,
                           threshold=std#,
                           #sharplo=0.3,
                           #sharphi=0.5,
                           #roundhi=0.3,
                           #roundlo=-0.3
                           )
        #self.findsources(sourcefinders,masterflux,100
        sources=sourcefinders.find_stars(masterflux[400:600,400:600] - median#data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,#daofind(data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,
                                         #     np.int(guess[1])-window:np.int(np.ceil(guess[1]))+window] - median)
        )
        sources.sort('flux')
        sources.pprint(max_lines=-1)
        peakl=sources[-1]['peak']
        sources=sourcefinders.find_stars(masterflux[400:600,1350:1550] - median#data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,#daofind(data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,
                                         #     np.int(guess[1])-window:np.int(np.ceil(guess[1]))+window] - median)
        )
        sources.sort('flux')
        sources.pprint(max_lines=-1)
        peakr=sources[-1]['peak']
        peak=0.5*(peakl+peakr)
        #print peak
        self.flux.flux=peak
        #plt.imshow(masterflux[400:600,1350:1550] - median,#data[np.int(guess[0])-window:np.int(np.ceil(guess[0]))+window,
                   #         np.int(guess[1])-window:np.int(np.ceil(guess[1]))+window] - median,
        #               origin='lower')
        #plt.show()

    def photCal(self,**kwargs):
        '''
        Perform photometric calibration with standard star
        '''
        pass

    def readdata(self,filename,**kwargs):
        #print filename

        hdu=fits.open(filename)
        cube=hdu[0].data #extract data itself
        exptime=hdu[0].header['EXPTIME']
        #then extract important header info ? (might have already done this before, not sure about architecture yet)
        cube=cube/exptime #?
        if self.masterdark is not None:
            cube=cube-self.masterdark
        if self.masterflat is not None:
            cube=cube/self.masterflat
        header=hdu[0].header
        extra='??'
        hdu.close()
        return cube,header#,extra

    def output(self,**kwargs):
        self.outfile=(self.headers[0]['HIERARCH ESO OBS NAME'] + '_' +
                      self.optics['filt'][2] + '_' +
                      self.headers[0]['HIERARCH ESO DET SEQ1 DIT'] + '_' +
                      self.headers[0]['HIERARCH ESO OBS START'] +
                      '.fits')
        print('Writing reduced data to file ',self.outfile)
        if self.interProd: #write intermediate products too
            pass
        fits.writeto(self.outfile ,self.finalNoDerot,self.outheader)
        fits.update(self.outfile , self.finalvarNoDerot,'var CI')
        if self.rot=='PUPIL':
            fits.update(self.outfile , self.finalDerot, 'CI derotated')
            fits.update(self.outfile , self.finalvarDerot, 'var CI derotated')
            fits.update(self.outfile , self.finalADI, 'Classical ADI')
            fits.update(self.outfile , self.finalvarADI, 'var Classical ADI')
        if self.mode=='SDI':
            fits.update(self.outfile , self.SDINoDerot, 'SDI')
            fits.update(self.outfile , self.varSDINoDerot, 'var SDI')
            if self.rot=='PUPIL':
                fits.update(self.outfile , self.SDIDerot, 'SDI derotated')
                fits.update(self.outfile , self.varSDIDerot, 'var SDI derotated')
                fits.update(self.outfile , self.finalSADI, 'SADI')
                fits.update(self.outfile , self.finalvarSADI, 'var SADI')
        if self.mode=='DPI':
            pass
#        fits.update( , self.badpixmap, 'bad pixel map')
        pass

class IrdisLSS(Irdis):
    pass


# /usr/bin/env python
# coding=utf8

import http.client
import hashlib
from urllib import request
import random
import json
import csv
import time
import pandas as pd

appid = ''  # 你的appid
secretKey = ''  # 你的密钥

def baidu_translate(sentence):
    httpClient = None
    myurl = '/api/trans/vip/translate'
    q = sentence
    fromLang = 'en'
    toLang = 'zh'
    salt = random.randint(32768, 65536)

    sign = appid + q + str(salt) + secretKey
    m1 = hashlib.md5()
    m1.update(sign.encode("utf8"))
    sign = m1.hexdigest()
    myurl = myurl + '?appid=' + appid + '&q=' + request.quote(q) + '&from=' + fromLang + '&to=' + toLang + '&salt=' + str(salt) + '&sign=' + sign

    httpClient = http.client.HTTPConnection('api.fanyi.baidu.com')
    httpClient.request('GET', myurl)

    # response是HTTPResponse对象
    response = httpClient.getresponse()
    result = response.read()
    result_json = json.loads(result)

    print(result_json)

    translate_str = result_json['trans_result'][0]['dst']

    return translate_str

# with open('./data/msr_label.csv', "r") as f, open('./data/msr_label_translate.csv', 'a+') as f2:
#
#     f_csv = csv.reader(f)
#     next(f_csv, None)
#
#     w_csv = csv.writer(f2)
#
#     for row in f_csv:
#         similay = row[0]
#         sentence1 = row[1]
#         sentence2 = row[2]
#
#         print(similay, sentence1, sentence2)
#
#         sentence1_translate = baidu_translate(sentence1)
#
#         time.sleep(1)
#
#         sentence2_translate = baidu_translate(sentence2)
#
#         w_csv.writerow([sentence1_translate, sentence2_translate, similay])
#
#         time.sleep(1)

train = pd.read_table('datadeal/knowledge.csv', header=None, encoding='utf-8-sig', sep=',')

train.reset_index(inplace=True)

train[0].to_frame().to_csv('datadeal/knowledge_question.csv', header=0, index=0)

# if httpClient:
#     httpClient.close()


from django.urls import path
from django.contrib.auth import views as auth_views
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    # path('cadastro/', views.criar_conta, name='criar_conta'),
    path('cadastro/', views.CriarConta.as_view(), name='criar_conta'),
    path('usuarios/<int:pk>/editar',
         views.EditarConta.as_view(), name='editar_conta'),
    path('usuarios/<int:pk>/delete/',
         views.DeletarUser.as_view(), name='deletar_conta'),

    path('login/', views.login_page, name='login'),
    path('logout/', auth_views.LogoutView.as_view(), name='logout'),

    path('jogos/cadastrar_jogo/', views.CriarJogo.as_view(), name='cadastrar_jogo'),
    path('jogos/<int:pk>/', views.DetalheJogo.as_view(), name='detail_jogo'),
    path('jogos/<int:pk>/editar/', views.EditarJogo.as_view(), name='editar_jogo'),
    path('jogos/<int:pk>/delete/',
         views.DeletarJogo.as_view(), name='deletar_jogo'),
]


import json
import base64
import boto3

personalize = boto3.client('personalize')
personalize_runtime = boto3.client('personalize-runtime')

def lambda_handler(event, context):
    describe_campaign_response = personalize.describe_campaign(
        campaignArn = event['campaign_arn']
    )
    status = describe_campaign_response["campaign"]["status"]
    print("Campaign: {}".format(status))

    return {
        'status': status,
        'campaign_arn': event['campaign_arn']
        #'o': status,
        #'datasetGroupArn': datasetGroupArnVal

        #'body': json.dumps('Hello from Lambda!')
    }


'''
From https://github.com/makinacorpus/easydict.
'''

class EasyDict(dict):
    """
    Get attributes

    >>> d = EasyDict({'foo':3})
    >>> d['foo']
    3
    >>> d.foo
    3
    >>> d.bar
    Traceback (most recent call last):
    ...
    AttributeError: 'EasyDict' object has no attribute 'bar'

    Works recursively

    >>> d = EasyDict({'foo':3, 'bar':{'x':1, 'y':2}})
    >>> isinstance(d.bar, dict)
    True
    >>> d.bar.x
    1

    Bullet-proof

    >>> EasyDict({})
    {}
    >>> EasyDict(d={})
    {}
    >>> EasyDict(None)
    {}
    >>> d = {'a': 1}
    >>> EasyDict(**d)
    {'a': 1}

    Set attributes

    >>> d = EasyDict()
    >>> d.foo = 3
    >>> d.foo
    3
    >>> d.bar = {'prop': 'value'}
    >>> d.bar.prop
    'value'
    >>> d
    {'foo': 3, 'bar': {'prop': 'value'}}
    >>> d.bar.prop = 'newer'
    >>> d.bar.prop
    'newer'

    Values extraction

    >>> d = EasyDict({'foo':0, 'bar':[{'x':1, 'y':2}, {'x':3, 'y':4}]})
    >>> isinstance(d.bar, list)
    True
    >>> from operator import attrgetter
    >>> map(attrgetter('x'), d.bar)
    [1, 3]
    >>> map(attrgetter('y'), d.bar)
    [2, 4]
    >>> d = EasyDict()
    >>> d.keys()
    []
    >>> d = EasyDict(foo=3, bar=dict(x=1, y=2))
    >>> d.foo
    3
    >>> d.bar.x
    1

    Still like a dict though

    >>> o = EasyDict({'clean':True})
    >>> o.items()
    [('clean', True)]

    And like a class

    >>> class Flower(EasyDict):
    ...     power = 1
    ...
    >>> f = Flower()
    >>> f.power
    1
    >>> f = Flower({'height': 12})
    >>> f.height
    12
    >>> f['power']
    1
    >>> sorted(f.keys())
    ['height', 'power']
    """
    def __init__(self, d=None, **kwargs):
        if d is None:
            d = {}
        if kwargs:
            d.update(**kwargs)
        for k, v in d.items():
            setattr(self, k, v)
        # Class attributes
        for k in self.__class__.__dict__.keys():
            if not (k.startswith('__') and k.endswith('__')):
                setattr(self, k, getattr(self, k))

    def __setattr__(self, name, value):
        if isinstance(value, (list, tuple)):
            value = [self.__class__(x)
                     if isinstance(x, dict) else x for x in value]
        elif isinstance(value, dict) and not isinstance(value, self.__class__):
            value = self.__class__(value)
        super(EasyDict, self).__setattr__(name, value)
        super(EasyDict, self).__setitem__(name, value)

    __setitem__ = __setattr__

def load_json(filename):
    import json
    with open(filename) as f:
        return EasyDict(json.load(f))

if __name__ == "__main__":
    import doctest
    doctest.testmod()


import os
import subprocess
import numpy as np
import matplotlib.pyplot as plt
import mplbm_utils as mplbm

def download_geometry(filename, url):

    download_command = f'wget {url} -O {filename}'
    try:
        subprocess.run(download_command.split(' '))
    except FileNotFoundError:
        raise InterruptedError(f'wget was not found. Please make sure it is installed on your system.')
    return

def run_2_phase_sim(inputs):
    # Steps
    # 1) create geom for palabos
    # 2) create palabos input file
    # 3) run 2-phase sim

    if inputs['simulation type'] == '1-phase':
        raise KeyError('Simulation type set to 1-phase...please change to 2-phase.')
    sim_directory = inputs['input output']['simulation directory']

    # 1) Create Palabos geometry
    print('Creating efficient geometry for Palabos...')
    mplbm.create_geom_for_palabos(inputs)

    # 2) Create simulation input file
    print('Creating input file...')
    mplbm.create_palabos_input_file(inputs)

    # 3) Run 2-phase simulation
    print('Running 2-phase simulation...')
    num_procs = inputs['simulation']['num procs']
    input_dir = inputs['input output']['input folder']
    simulation_command = f"mpirun -np {num_procs} ../../src/2-phase_LBM/ShanChen {input_dir}2_phase_sim_input.xml"
    file = open(f'{sim_directory}/{input_dir}run_shanchen_sim.sh', 'w')
    file.write(f'{simulation_command}')
    file.close()

    simulation_command_subproc = f'bash {sim_directory}/{input_dir}run_shanchen_sim.sh'
    subprocess.run(simulation_command_subproc.split(' '))

    return

def run_rel_perm_sim(inputs):

    # Rel Perm Steps
    # 1) create 1-phase palabos input file for relperms
    # 2) create geoms for rel perm
    # 3) run 1-phase sim to get rel perms

    if inputs['simulation type'] == '1-phase':
        raise KeyError('Simulation type set to 1-phase...please change to 2-phase.')
    sim_directory = inputs['input output']['simulation directory']

    # 1) Create geoms for rel perm
    print('Creating rel perm geometries...')
    user_geom_name = inputs['domain']['geom name']
    inputs = mplbm.create_geom_for_rel_perm(inputs)

    # 2) Create simulation input file
    print('Creating input file...')
    inputs['simulation type'] = 'rel perm'
    mplbm.create_palabos_input_file(inputs)
    inputs['domain']['geom name'] = user_geom_name

    # 3) Write rel perm bash file
    print('Running rel perm simulation...')
    num_procs = inputs['simulation']['num procs']
    input_dir = inputs['input output']['input folder']
    output_dir = inputs['input output']['output folder']
    simulation_command = f"mpirun -np {num_procs} ../../src/1-phase_LBM/permeability {input_dir}relperm_input.xml"
    file = open(f'{sim_directory}/{input_dir}run_relperm_sim.sh', 'w')
    file.write(f'{simulation_command}')
    file.close()

    simulation_command_subproc = f'bash {sim_directory}/{input_dir}run_relperm_sim.sh'
    make_4relperm_folder = f'mkdir {sim_directory}/{output_dir}/4relperm'
    subprocess.run(make_4relperm_folder.split(' '))
    subprocess.run(simulation_command_subproc.split(' '))

    return

def process_and_plot_results(inputs):

    # Process data
    mplbm.create_pressure_data_file(inputs)
    mplbm.create_relperm_data_file(inputs)

    # Load and plot data
    sim_dir = inputs['input output']['simulation directory'] + '/'
    output_dir = inputs['input output']['output folder']
    Sw = np.loadtxt(f'{sim_dir + output_dir}data_Sw.txt')
    Pc = np.loadtxt(f'{sim_dir + output_dir}data_Pc.txt')
    krw = np.loadtxt(f'{sim_dir + output_dir}data_krw.txt')
    krnw = np.loadtxt(f'{sim_dir + output_dir}data_krnw.txt')

    plt.figure(figsize=[10,8])
    mplbm.plot_capillary_pressure_data(Sw, Pc)
    plt.savefig(sim_dir + 'pc_curve.png', dpi=300)

    plt.figure(figsize=[10,8])
    mplbm.plot_rel_perm_data(Sw, krw, krnw)
    # plt.yscale('log')
    plt.savefig(sim_dir + 'relperm_curve.png', dpi=300)

    plt.figure(figsize=[6, 10])
    mplbm.plot_pc_and_rel_perm(Sw, Pc, krw, krnw)
    plt.savefig(sim_dir + 'pc_and_relperm_curve.png', dpi=300)

    return

drp_url = 'www.digitalrocksportal.org/projects/65/images/71108/download/'
file_name = 'input/rg_theta60_phi10.raw'
download_geometry(file_name, drp_url)

input_file = 'input.yml'
inputs = mplbm.parse_input_file(input_file)  # Parse inputs
inputs['input output']['simulation directory'] = os.getcwd()  # Store current working directory
run_2_phase_sim(inputs)  # Run 2 phase sim
run_rel_perm_sim(inputs)  # Run rel perm
process_and_plot_results(inputs)  # Plot results

# plt.show()


"""Functions for loading data files from the SP experiment

"""
import numpy as np
from os import path
import os
import re
from collections import namedtuple
import yaml
import MT
import convert

relaxedNames="""Tot Pos Det
Tot Pos Det 2
Tot Pos Det 3
Cell Temp
Ferrite Temp
Oven Temp
Temperature Controller Output
Breadboard Temp
Room Temp
G10 Lid Temp
Tilt Sensor 1
Tilt Sensor 2
WavelengthErr
Pump Current Mon
Pump Temp Mon
Probe Current Mon
Probe Temp Mon
Water Flow
Apparatus Position
""".splitlines()
mediumNames="""Accelerometer
Position Purple
Position Green
Fluxgate X
Fluxgate Y
Fluxgate Z
Tiltmeter X
Tiltmeter Y
Vert Pos Det
Horiz Pos Det
Vert Pos Det 2
Horiz Pos Det 2
Vert Pos Det 3
Horiz Pos Det 3
""".splitlines()

def getSpinningDataNames(timestamp, bMotorData=False):
    if not bMotorData:
        base_name='SP_aparatus {0}'.format(timestamp)
    else:
        base_name='SP_motor {0}'.format(timestamp)
    base_dir=os.path.join(os.getenv('SP_DATA_DIR'), base_name + ' Bundle/')
    return base_dir, base_name

def getAllTimeStamps(bApparatus=True, bMotor=False):
    data_dir=os.path.join(os.getenv('SP_DATA_DIR'))
    ap_dir_regex=re.compile('SP_aparatus (\d+(?:\.\d*)?|\.\d+) Bundle$')
    mo_dir_regex=re.compile('SP_motor (\d+(?:\.\d*)?|\.\d+) Bundle$')

    regExL=[]
    if bApparatus:
        regExL.append(ap_dir_regex)
    if bMotor:
        regExL.append(mo_dir_regex)
    outL=[]
    for reg in regExL:
        match_list=[ap_dir_regex.match(line) for line in os.listdir(data_dir)]
        stamps=[m.groups()[0] for m in match_list if m is not None]
        stamps.sort()
        outL.append(stamps)
    if len(outL)==1:
        outL=outL[0]
    return outL

#old
def loadSet(timestamp, cal=None, subL=["Fast", "Medium", "Relaxed"], startTime=-np.inf, endTime=np.inf):
    """OLD Load all the data associated with @timestamp
    """
    if cal is None:
        print("Haven't implemented loading of calibration data yet, Your data will stay in Volts")
        cal=1
    base_dir, base_name=getSpinningDataNames(timestamp)

    def splitRecArray(rA):
        t=rA['sd']
        names=list(rA.dtype.names)
        names.remove('sd')
        y=rA[names].view(rA.dtype[-1]).reshape(rA.size,-1)
        return [t, y.T, names]

    #startI=0
    fastD=None;
    medD=None;
    relD=None;
    if "Fast" in subL:
        fastRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Fast'), totalDataType=[('sd','>f8'), ('s1', '>f4')])
        if fastRec is not None:
            fastD=splitRecArray(fastRec)
            fastD[1]=fastD[1].squeeze()
            fastD[1]*=cal

            startI, endI=fastD[0].searchsorted([startTime, endTime])
            fastD[0]=fastD[0][startI:endI]
            fastD[1]=fastD[1][startI:endI]

    if "Medium" in subL:
        medRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Medium'), useMetaFile=True)
        if medRec is not None:
            renameFieldMaybe(medRec, 'Sidereal Days', 'sd')
            medD=splitRecArray(medRec)
            startI, endI=medD[0].searchsorted([startTime, endTime])
            medD[0]=medD[0][startI:endI]
            medD[1]=medD[1][:,startI:endI]

    if "Relaxed" in subL:
        relRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Relaxed'), useMetaFile=True)
        if relRec is not None:
            renameFieldMaybe(relRec, 'Sidereal Days', 'sd')
            relD=splitRecArray(relRec)
            startI, endI=relD[0].searchsorted([startTime, endTime])
            relD[0]=relD[0][startI:endI]
            relD[1]=relD[1][:,startI:endI]
    if "Slow" in subL:
        slRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Slow'), useMetaFile=True)
        if slRec is not None:
            renameFieldMaybe(slRec, 'Sidereal Days', 'sd')
            slD=splitRecArray(slRec)

    return SPDataSet(fastD, medD=medD, relD=relD)

set_info_defaults={
            'bUseful': 'unknown',
            'bDataConfirmed':False,
            'acquisition':{'sampleRate': 50, 'timeConstant': 0.003},
            #'rotationSpec':{'delay': 9,
            #                'interval':30,
            #                'numRotations':20,
            #                'rotAngles': [-180],
            #                'extraRotAngle': -90.,
            #                'startingAngle': 0,
            #                },
            'timeStamp':'unknown',
            'notes': 'Default',
            #'windows':{}
            'windows':{
                'sig':{
                    'duration': 4,
                    'offset': -6.6,
                    },
                'bg':{
                    'duration': 1,
                    'offset': 0,
                    },
                    }
            }
def loadSetInfo(dir_name):
    ts=dir_name.split()[-2]
    bWriteNew=False
    filepath=os.path.join(dir_name, 'set_info.yaml')
    try:
        f=open(filepath)
        infoD=yaml.load(f)
        f.close()
    except IOError:
        print("No set_info.yaml exists in {}, making defaults".format(dir_name))
        bWriteNew=True
        infoD= {}

    loadedKeys=infoD.keys()
    if 'bandwidth' in loadedKeys: #It's a matlab generated yaml file
        print("updating matlab generated yaml")
        #newD=copy(set_info_defaults)
        newD={
            'bUseful':infoD['bUseful'],
            'bDataConfirmed':infoD['bDataConfirmed'],
            'acquisition':{'sampleRate': infoD['bandwidth'][1],
                            'timeConst': infoD['bandwidth'][0]
                         },
            'rotationSpec':{'delay': infoD['delay'], 'interval':infoD['interval'],
                            'numRotations':infoD['numRotations'],
                            'rotAngles': [infoD['rotationAngles'][0]],
                            'extraRotAngle':infoD['rotationAngles'][1],
                            'startingAngle':MT.cardinalToDeg(infoD['startPosition']),
                            },
            'timeStamp':infoD['timestamp'],
            'notes':infoD['notes']+'\nConverted from matlab',
            }
        infoD=newD
        writeSetInfo(dir_name, infoD)

    if infoD.has_key('rotationSpec') and infoD['rotationSpec'].has_key('startPosition'):
        infoD['rotationSpec']['startingAngle']= MT.cardinalToDeg(infoD['rotationSpec']['startPosition'])
        infoD['rotationSpec'].pop('startPosition')

    newLoadedKeys=infoD.keys()
    for key in set_info_defaults.keys():
        if key not in newLoadedKeys:
            infoD[key]=set_info_defaults[key]
            print("Loading {0} from defaults, it may not be right. Go fix this in set_info.yaml!".format(key))

    if bWriteNew:
        infoD['timeStamp']=ts
        writeSetInfo(dir_name, infoD)
    return infoD

def writeSetInfo(dir_name, D):
        filepath=os.path.join(dir_name, 'set_info.yaml')
        print("writing new set_info.yaml file...")
        fout=open(filepath,'w')
        fout.write(yaml.dump(D))
        fout.close()
        print("{0} was written.".format(filepath))

def loadZeroFile(fname):
    proc_dict={
            1: 'cBz',
            2: 'By',
            3: 'Bx',
            35: 'Cal',
            29: 'Pol',
    }
    dat=np.loadtxt(fname).T;
    t=dat[0]
    proc_num=dat[1]
    val=dat[2]
    slope=dat[3]
    sigma=dat[4]
    ZeroProc=namedtuple('ZeroProc', ['t', 'val', 'slope', 'sigma'])
    zD={}
    for num, name in proc_dict.iteritems():
        I=np.where(proc_num==float(num))[0]
        zD[name]=ZeroProc(t=t[I], val=val[I], slope=slope[I], sigma=sigma[I])
        #DataFrame =namedtuple('DataFrame', ['sd', 'y'])
    return zD

def getAllSPSensor(names, slc=None, timeRange=None):
    data_dir=os.path.join(os.getenv('SP_DATA_DIR'))
    ap_dir_regex=re.compile('SP_aparatus (\d+(?:\.\d*)?|\.\d+) Bundle$')
    mo_dir_regex=re.compile('SP_motor (\d+(?:\.\d*)?|\.\d+) Bundle$')

    ap_match_list=[ap_dir_regex.match(line) for line in os.listdir(data_dir)]
    ap_stamps=[m.groups()[0] for m in ap_match_list if m is not None]
    mo_match_list=[mo_dir_regex.match(line) for line in os.listdir(data_dir)]
    mo_stamps=[m.groups()[0] for m in mo_match_list if m is not None]

    mo_stamps.sort()
    ap_stamps.sort()
    ioff()
    sdL=[]
    if slc is None:
        [firstI, lastI]=searchsorted(ap_stamps, timeRange)
        slc=slice(firstI, lastI)
    #slc=slice(-20,-15)
    relIdxs=[relaxedNames.index(name) for name in names if name in relaxedNames]
    medIdxs=[mediumNames.index(name) for name in names if name in mediumNames]
    #slIdxs=[slowNames.index(name) for name in names if name is in slowNames]
    #medList=[idx for name in names if name is in mediumNames]
    #slList=[idx for name in names if name is in slNames]

    allMedL=[]
    allRelL=[]
    subSetL=[]
    if medIdxs:
        subSetL.append("Medium")
    if relIdxs:
        subSetL.append("Relaxed")

    for stamp in ap_stamps[slc]:
        thisTsYL=[]
        dat=loadSet(stamp, subL=subSetL)
        relL=[]
        medL=[]
        #tsL=[]
        for idxs, D, l in [(relIdxs, dat.relD, relL), (medIdxs, dat.medD, medL)]:
            if idxs and D:
                l.append([D[0]]+[D[1][idx] for idx in idxs])
                #tsL.append(D[0])
            #else:
                #tsL.append(None)
                #l.append(None)

        if medL:
            allMedL.append(medL)
        if relL:
            allRelL.append(relL)

        if 0:
            try:
                dat=loadSet(stamp, subL=subSet)
                if subSet=='Relaxed':
                    D=dat.relD
                elif subSet=='Medium':
                    D=dat.medD
                elif subSet=='Slow':
                    D=dat.slowD
                if D:
                    sd=D[0].copy()
                    y=D[1][D[2].index(name)].copy()
                    del dat
                    sdL.append(sd)
                    yL.append(y)

            except ValueError:
                print("\nTimestamp {0} didn't load, maybe no '{1}' data? \n \n".format(stamp, subSet))
        if allRelL:
            allRelL=dstack(allRelL).squeeze()
        if allMedL:
            allMedL=dstack(allMedL).squeeze()
    return allMedL, allRelL

def loadBins(root_path, useMetaFile=True, nVars=1, endian='', dType=None, totalDataType=None):
    """Load all the binary files in that match the root path.

    They match the root path with regex 'root_path( \d+)?.bin',
    that is anything that starts with root_path, ends with .bin, and optionally has a number after a space. If there are numbers after the space, the files are loaded and the data appended in the correct order.
    """
    dirname=path.dirname(root_path)
    rootname=path.basename(root_path)
    r=re.compile('{0}( \d+)?\.bin'.format(rootname))
    dlist=os.listdir(dirname);
    dlist=[st for st in dlist if r.match(st)]
    if not dlist:
        print("No data for {0}".format(root_path))
        return None
    if len(dlist)>1:
        dlist.sort( key=lambda st: int(r.match(st).groups()[0]) ) # dlist is now sorted

    dat_list=[loadABin(path.join(dirname, fname), useMetaFile=useMetaFile, nVars=nVars, endian=endian, totalDataType=totalDataType) for fname in dlist]
    # Combine these into a single file...

    dat=np.hstack(dat_list)
    #datnames=dat_list[0].keys()
    return dat

def loadABin(filepath, nVars=1, useMetaFile=True, endian='', totalDataType=None):
    """Load a single binary file at filepath
    """
    #This is all just establishing how the data is stored in the file
    if totalDataType is None: #If a data type wasn't given, we'll try to work it out:
        if useMetaFile: #Usually the data type is stored in a meta file:
            fname_components=path.splitext(filepath)[0].split(' ')
            if fname_components[-1].isdigit():
                fname_components.pop()
            meta_file_path=' '.join(fname_components) + '.npmeta'
            npmeta_path=' '.join(fname_components) + '.npmeta'
            if path.exists(npmeta_path):
                mfile=open(npmeta_path)
                lines=mfile.readlines()
                pairs=[line.strip().split(',') for line in lines if not line.isspace() and not line.startswith('#') ]# should be tuples of e.g. ('signal', 'f8');
                defaultDtype='>f4'
                totalDataType=[]
                for p in pairs:
                    name=p[0].strip()
                    if len(p)>1:
                        dt=p[1].strip()
                        defaultDtype=dt;
                    else:
                        dt=defaultDtype;
                    totalDataType.append((name, dt));
            else: #If there was no appropriate unambiguous meta-file, look for the old style:
                meta_file_path=' '.join(fname_components) + '.meta'
                if not path.exists(meta_file_path): #Sometimes they have the extension .meta.txt for no good reason, so check for that too
                    meta_file_path=meta_file_path+'.txt'
                if path.exists(meta_file_path):
                    convert.spmeta2npmeta(meta_file_path) #This convert the old style to the new (Corrects for some changes, but not all)
                    print("Found an old meta-file only. Will attempt to make something useful out of it (old style is ambiguous)")
                    return loadABin(filepath, nVars, useMetaFile) #We'll run the function again with the new meta file exists
                    #mfile=open(meta_file_path)
                else:
                    raise IOError("Looking for a meta-file (for {0}) but none exists".format(' '.join(fname_components)))
                sigNames=mfile.readlines()
                sigNames=[name.strip() for name in sigNames]
                if sigNames[0]=='Sidereal Days':
                    sigNames=sigNames[1:]
                nVars=len(sigNames)
                sigDataType=[(name, endian+'f8') for name in sigNames]
                timeDataType=[('sd', endian+'f8')]
                totalDataType=timeDataType+sigDataType

        else:
            timeDataType=[('sd', endian+'f8')]
            sigDataType=[('s{0}'.format(n), endian+'f8') for n in range(1,nVars+1)]
            totalDataType=timeDataType+sigDataType

    # This is where we actually read it
    dat=np.fromfile(filepath, totalDataType)

    return dat;

if __name__=="__main__":
    #ddir='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_motor 5253.67 Bundle/'
    #fpath=ddir+'SP_motor 5253.67 sensors 8.bin'
    ddir='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_aparatus 5256.38 Bundle/'
    #ddir='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_motor 5256.38 Bundle/'
    fname='SP_aparatus 5256.38 Medium 6.bin'
    fname_f='SP_aparatus 5256.38 Fast'
    ddir2='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_motor 5253.67 Bundle/'
    fname2='SP_motor 5253.67 sensors'
    fpath=ddir+fname
    fpath_f=ddir+fname_f

    dat=loadABin(fpath, useMetaFile=True);
    dat_f=loadBins(fpath_f, useMetaFile=False, totalDataType=[('sd','>f8'), ('s1', '>f4')]);
    dat2=loadBins(ddir+'SP_aparatus 5256.38 Medium')
    dat3=loadBins(ddir2+fname2, useMetaFile=True)


"""Setup physlearn package."""

# Author: Alex Wozniakowski
# Licence: MIT

import os
import sys

from distutils.file_util import copy_file
from setuptools import setup, find_packages

DISTNAME = 'scikit-physlearn'
DESCRIPTION = 'A machine learning library for regression.'

MAINTAINER = 'Alex Wozniakowski'
MAINTAINER_EMAIL = 'wozn0001@e.ntu.edu.sg'
URL = 'https://github.com/a-wozniakowski/scikit-physlearn'
DOWNLOAD_URL = 'https://github.com/a-wozniakowski/scikit-physlearn'
PROJECT_URLS = dict(Paper='https://iopscience.iop.org/article/10.1088/2632-2153/ac1ee9')

SETUPTOOLS_COMMANDS = set(['develop', 'release', 'bdist_egg', 'bdist_rpm',
                           'bdist_wininst', 'install_egg_info', 'build_sphinx',
                           'egg_info', 'easy_install', 'upload', 'bdist_wheel',
                           '--single-version-externally-managed'])

if SETUPTOOLS_COMMANDS.intersection(sys.argv):
    extra_setuptools_args = dict(zip_safe=False,
                                 include_package_data=True,
                                 extras_require=dict(docs=['sphinx >= 3.0.3',
                                                           'sphinx-gallery >= 0.7.0']))
else:
    extra_setuptools_args = dict()

LICENSE = 'MIT'
CLASSIFIERS = ['Intended Audience :: Science/Research',
               'Intended Audience :: Developers',
               'Programming Language :: Python',
               'Topic :: Software Development',
               'Topic :: Scientific/Engineering :: Artificial Intelligence',
               'Operating System :: Microsoft :: Windows',
               'Operating System :: POSIX',
               'Operating System :: Unix',
               'Operating System :: MacOS',
               'Programming Language :: Python :: 3.6',
               'Programming Language :: Python :: 3.7',
               'Programming Language :: Python :: 3.8']
PACKAGES = find_packages()

INSTALL_REQUIRES = ['matplotlib',
                    'numpy',
                    'scipy',
                    'scikit-learn',
                    'pandas',
                    'shap',
                    'ipython',
                    'bayesian-optimization',
                    'catboost',
                    'xgboost',
                    'lightgbm',
                    'mlxtend',
                    'python-Levenshtein']

if __name__ == '__main__':
    CURRENT_DIR = os.path.abspath(os.path.dirname(__file__))
    if os.path.isfile(os.path.join(CURRENT_DIR, 'VERSION.txt')):
        copy_file(os.path.join(CURRENT_DIR, 'VERSION.txt'),
                  os.path.join(CURRENT_DIR, 'physlearn', 'VERSION.txt'),
                  verbose=0)
    VERSION = open(os.path.join(CURRENT_DIR, 'physlearn', 'VERSION.txt'), encoding='utf-8').read().strip()
    LONG_DESCRIPTION = open(os.path.join(CURRENT_DIR, 'README.md'), encoding='utf-8').read()

    sys.path.insert(0, CURRENT_DIR)

    metadata = dict(name=DISTNAME,
                    maintainer=MAINTAINER,
                    maintainer_email=MAINTAINER_EMAIL,
                    description=DESCRIPTION,
                    long_description=LONG_DESCRIPTION,
                    long_description_content_type='text/markdown',
                    url=URL,
                    download_url=DOWNLOAD_URL,
                    project_urls=PROJECT_URLS,
                    version=VERSION,
                    license=LICENSE,
                    classifiers=CLASSIFIERS,
                    packages=PACKAGES,
                    package_data={'': ['*.json', '*.csv']},
                    install_requires=INSTALL_REQUIRES,
                    **extra_setuptools_args)

    setup(**metadata)


import streamlit as st
import pandas as pd
from PIL import Image

def app():
    st.title('ABOUT')

    st.markdown("Automation to authenticate Indihome installation evidence")

    image_back = Image.open(r"data/slide_img/background.png")
    st.image(image_back, caption='Background', use_column_width="auto")

    st.markdown("We need a system that **automatically verify** incoming installation evidence by **classifying whether** that image is accepted **(OK)** or rejected **(NOT_OK)**")

    image_obj = Image.open(r"data/slide_img/objective.png")
    st.image(image_obj, caption='Objective', use_column_width="auto")


from . import AbstractExperiment
import experimentalCollections as tc
import networkx as nx
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import cm
import json
import pandas as pd
import convLib
from convLib import Annotator
import utils
import time
from retrieval.Analyzers import StandardAnalyzer

from convLib import contexts, Annotator

import experimentalCollections as EC

from sklearn.preprocessing import normalize
from sklearn.metrics import r2_score

from scipy.stats import gaussian_kde, pearsonr
from scipy.spatial.distance import jensenshannon
from collections import Counter

import seaborn as sns
import random

from retrieval.CAsTutils import loadQueries
#'first_query', r"FU",
systems_order = ['context_query', 'linear_prev',  'RM3_prev', 'RM3_seq', 'allennlp_simple']
systems_labels = [ r"CU", r"LP", r"RM3p", r"RM3s", r"anCB"]

class Intratopic_analysis(AbstractExperiment):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def run_experiment(self):
        sns.set_theme(style="whitegrid")
        #sns.set_context("paper",
        #                rc={"font.size": 24, "axes.titlesize": 24, "xtick.labelsize": 24, "ytick.labelsize": 24,
        #                     "legend.fontsize":24, "axes.labelsize": 24})

        measures = pd.read_csv(f"../../data/measures/full_valid_100.csv")

        avgs_all = measures\
                        [['topic', 'perm', 'qtype', 'name', 'nDCG@3']]\
                        .groupby(['topic', 'qtype', 'name', 'perm'])\
                        .aggregate(['mean'])\
                        .droplevel(1, axis=1)\
                        .reset_index() \

        avgs_all = avgs_all.groupby(['name', 'qtype', 'perm']).aggregate(['mean']).droplevel(1, axis=1).reset_index()

        avgs = measures[measures['name'].isin(systems_order)]\
                        [['topic', 'perm', 'qtype', 'name', 'nDCG@3']]\
                        .groupby(['topic', 'qtype', 'name', 'perm'])\
                        .aggregate(['mean'])\
                        .droplevel(1, axis=1)\
                        .reset_index()

        '''
        corrs = []
        for qtype in ['original', 'resolved']:
            avgs_t = avgs[avgs['qtype']==qtype][['name', 'topic', 'perm', 'nDCG@3']].copy()
            corr = avgs_t.groupby(['topic']).apply(rankingSimilarity)
            corr = pd.DataFrame([[qtype] + l for l in list(corr)], columns = ['qtype', 'topic', 'correlation', 'pvalue'])
            corrs.append(corr)
        corrs = pd.concat(corrs)
        fname = "../../data/correlation_valid_permutations.pdf"

        fig, ax = plt.subplots(figsize=(18, 12))
        h = sns.barplot(data=corrs, x='topic', y='correlation', hue='qtype', ax=ax)
        handles, labels = ax.get_legend_handles_labels()
        ax.legend(handles=handles[0:], labels=labels[0:])
        ax.set(xlabel= "Conversation id", ylabel=r"Correlation (Spearman's $\rho$)")
        plt.savefig(fname)
        '''
        #for qtype in ['original', 'resolved']:
        for qtype in ['original']:
            avgs_t = avgs[(avgs['qtype']==qtype) & (avgs['topic'].isin(["32", "33", "37", "56", "58", "59", "61", "67", "69", "77", "78", "79"]))][['name', 'topic', 'perm', 'nDCG@3']].copy()
            Z = avgs_t.groupby(["topic"]).apply(pairwise_best).reset_index()
            print(Z)
            Z = Z.drop(columns=["level_1"])
            Z = Z.rename({"topic":"Conv. id"}, axis='columns')
            print(Z)
            grid = sns.FacetGrid(Z, col="Conv. id", col_wrap=4)
            grid.map(draw_heatmap, 's1', 's2', 'n_best', cbar=False, annot=True,  fmt='.3g')
            grid.set_titles(size=18)
            fname = f"../../data/figures/best_between_two_{qtype}.pdf"
            plt.savefig(fname)

def draw_heatmap(*args, **kwargs):
    data = pd.concat(args, axis=1)
    d = data.pivot(index=args[0].name, columns=args[1].name, values=args[2].name)
    d = d.sort_values('s1', key=make_sorter(systems_order))[systems_order]
    g = sns.heatmap(d, **kwargs)
    g.set_xticklabels(systems_labels, fontsize=18)
    g.set_yticklabels(systems_labels, fontsize=18)
    g.set_ylabel("")
    g.set_xlabel("")

def rankingSimilarity(scores, sim='pearson', index='name'):
    old_scores = scores[scores['perm']==0][['name', 'nDCG@3']]
    new_scores = scores[scores['perm']!=0][['name', 'nDCG@3']]

    temp_scores = pd.merge(old_scores, new_scores, how="inner", on=["name"])

    return [scores['topic'].unique()[0]] + list(pearsonr(temp_scores['nDCG@3_x'], temp_scores['nDCG@3_y']))

def pairwise_best(avgs_t):
    models = avgs_t['name'].unique()
    tmp = avgs_t.pivot_table(index='perm', columns='name', values='nDCG@3').reset_index()

    output=[[i, j, (np.sum(tmp[i]>tmp[j]))] for i in models for j in models if i!=j]
    output = pd.DataFrame(output, columns=["s1", "s2", "n_best"])

    return output

def make_sorter(l):
    """
    Create a dict from the list to map to 0..len(l)
    Returns a mapper to map a series to this custom sort order
    """
    sort_order = {k:v for k,v in zip(l, range(len(l)))}
    return lambda s: s.map(lambda x: sort_order[x])


from tkinter import *
from PIL import ImageTk, Image
from tkinter import filedialog

root = Tk()

root.title('File dialog')
root.iconbitmap('quakechamps.ico')

def open():
    global image
    root.filename = filedialog.askopenfilename(initialdir="C:/Users/alexh/Desktop/pythontut/tkintertut/images", title="Select a File", filetypes=(("png files", "*.png"), ("all files", "*.*")))
    label = Label(root, text=root.filename).pack()

    image = ImageTk.PhotoImage(Image.open(root.filename))
    image_label = Label(image=image).pack()

button = Button(root, text="Open file", command=open).pack()

root.mainloop()


import mxnet as mx
import gluoncv as gcv
from mxnet import gluon
from gluoncv.data.transforms.presets.ssd import SSDDefaultValTransform
from gluoncv.data.transforms.presets.rcnn import FasterRCNNDefaultValTransform
from gluoncv.data.batchify import Tuple, Stack, Pad
from gluoncv.data import batchify

def validate_ssd(net, val_data, ctx, eval_metric):
    """Test on validation dataset."""
    eval_metric.reset()
    # set nms threshold and topk constraint
    net.set_nms(nms_thresh=0.45, nms_topk=400)
    net.hybridize(static_alloc=True, static_shape=True)
    for batch in val_data:
        data = gluon.utils.split_and_load(batch[0], ctx_list=ctx, batch_axis=0, even_split=False)
        label = gluon.utils.split_and_load(batch[1], ctx_list=ctx, batch_axis=0, even_split=False)
        det_bboxes = []
        det_ids = []
        det_scores = []
        gt_bboxes = []
        gt_ids = []
        gt_difficults = []
        for x, y in zip(data, label):
            # get prediction results
            ids, scores, bboxes = net(x)
            det_ids.append(ids)
            det_scores.append(scores)
            # clip to image size
            det_bboxes.append(bboxes.clip(0, batch[0].shape[2]))
            # split ground truths
            gt_ids.append(y.slice_axis(axis=-1, begin=4, end=5))
            gt_bboxes.append(y.slice_axis(axis=-1, begin=0, end=4))
            gt_difficults.append(y.slice_axis(axis=-1, begin=5, end=6) if y.shape[-1] > 5 else None)

        # update metric
        eval_metric.update(det_bboxes, det_ids, det_scores, gt_bboxes, gt_ids, gt_difficults)
    return eval_metric.get()

def validate_frcnn(net, val_data, ctx, eval_metric):
    """Test on validation dataset."""
    clipper = gcv.nn.bbox.BBoxClipToImage()
    eval_metric.reset()
    net.hybridize(static_alloc=True)
    for batch in val_data:
        batch = split_and_load(batch, ctx_list=ctx)
        det_bboxes = []
        det_ids = []
        det_scores = []
        gt_bboxes = []
        gt_ids = []
        gt_difficults = []
        for x, y, im_scale in zip(*batch):
            # get prediction results
            ids, scores, bboxes = net(x)
            det_ids.append(ids)
            det_scores.append(scores)
            # clip to image size
            det_bboxes.append(clipper(bboxes, x))
            # rescale to original resolution
            im_scale = im_scale.reshape((-1)).asscalar()
            det_bboxes[-1] *= im_scale
            # split ground truths
            gt_ids.append(y.slice_axis(axis=-1, begin=4, end=5))
            gt_bboxes.append(y.slice_axis(axis=-1, begin=0, end=4))
            gt_bboxes[-1] *= im_scale
            gt_difficults.append(y.slice_axis(axis=-1, begin=5, end=6) if y.shape[-1] > 5 else None)

        # update metric
        for det_bbox, det_id, det_score, gt_bbox, gt_id, gt_diff in zip(det_bboxes, det_ids,
                                                                        det_scores, gt_bboxes,
                                                                        gt_ids, gt_difficults):
            eval_metric.update(det_bbox, det_id, det_score, gt_bbox, gt_id, gt_diff)

    return eval_metric.get()

def get_val_frcnn_dataloader(val_dataset, batch_size,
                   num_workers, short=600, max_size=800):
    """Get dataloader."""
    val_bfn = batchify.Tuple(*[batchify.Append() for _ in range(3)])
    val_loader = mx.gluon.data.DataLoader(
        val_dataset.transform(FasterRCNNDefaultValTransform(short, max_size)),
        batch_size, False, batchify_fn=val_bfn, last_batch='keep', num_workers=num_workers)
    return val_loader

def split_and_load(batch, ctx_list):
    """Split data to 1 batch each device."""
    num_ctx = len(ctx_list)
    new_batch = []
    for i, data in enumerate(batch):
        new_data = [x.as_in_context(ctx) for x, ctx in zip(data, ctx_list)]
        new_batch.append(new_data)
    return new_batch

def get_val_ssd_dataloader(val_dataset, width, height, batch_size, num_workers):
    """Get dataloader."""
    # use fake data to generate fixed anchors for target generation
    val_batchify_fn = Tuple(Stack(), Pad(pad_val=-1))
    val_loader = gluon.data.DataLoader(
        val_dataset.transform(SSDDefaultValTransform(width, height)),
        batch_size, False, batchify_fn=val_batchify_fn, last_batch='keep', num_workers=num_workers)
    return val_loader


import cv2
import glob
import os

inputFolder='C:/Users/hp/OneDrive/Desktop/Leaf'
os.mkdir('C:/Users/hp/OneDrive/Desktop/Resize_300')

i=0
SIZE=300

for img in glob.glob(inputFolder+"/*.JPG"):
    image=cv2.imread(img)
    imgResized=cv2.resize(image,(SIZE,SIZE))
    cv2.imwrite('C:/Users/hp/OneDrive/Desktop/Resize_300/image%i.JPG' %i,imgResized)

    i+=1
    cv2.imshow('image',imgResized)
    cv2.waitKey(30)

cv2.destroyAllWindows()


import math
import random
from coloraide import Color
from rpi_ws281x import Color as PixelColor

# Coloraide color to pixel
def color_to_pixel(color, gamma_correction):
    c_dict = color.to_dict()
    return PixelColor(
        math.ceil(gamma_correct(c_dict["r"], gamma_correction) * 255),
        math.ceil(gamma_correct(c_dict["g"], gamma_correction) * 255),
        math.ceil(gamma_correct(c_dict["b"], gamma_correction) * 255),
    )

# Hex to pixel
def hex_to_pixel(hex_string, gamma_correction):
    return color_to_pixel(Color(hex_string), gamma_correction)

# Spread out colors into strip
def spread_colors(colors, pixel_length, id, max_spread):
    random.seed(f"{id}-spread")
    spread = random.randint(1, max_spread)
    full_colors = []

    # Fill in colors.  Probably a more efficient way to do this.
    spread_place = 0
    color_place = 0
    for i in range(pixel_length):
        full_colors.append(colors[color_place])

        # Spread place used to know how many times to repeat the color place
        spread_place = 0 if spread_place == spread - 1 else spread_place + 1

        # When the spread place starts, increment the color.
        if i > 0 or spread == 1:
            color_place = color_place + 1 if spread_place == 0 else color_place
            color_place = 0 if color_place >= len(colors) else color_place

    return full_colors

# Set all colors in a strip
def strip_set_colors(strip, colors, gamma_correction):
    for i in range(len(colors)):
        if isinstance(colors[i], Color):
            strip.setPixelColor(i, color_to_pixel(colors[i], gamma_correction))
        else:
            strip.setPixelColor(i, hex_to_pixel(colors[i], gamma_correction))

# Correct gamma for a value.  Value should be value between 0 and 1.
# Inspiration:
# https://github.com/adafruit/Adafruit_CircuitPython_FancyLED/blob/main/adafruit_fancyled/adafruit_fancyled.py#L372
def gamma_correct(value, gamma_correction):
    if not gamma_correction:
        return value

    return pow(value, gamma_correction)


from selenium import webdriver
from selenium.webdriver.safari.webdriver import WebDriver
from features.pages.quiz_home_page import QuizHomePage

def before_all(context):
    driver: WebDriver = webdriver.Safari()
    driver.maximize_window()
    quiz_page = QuizHomePage(driver)
    context.driver = driver
    context.quiz_page = quiz_page

def after_all(context):
    context.driver.quit()


from re import sub
from utils import time_and_date
from wikitools import wiki

verbose = False
LANGS = ['ar', 'cs', 'da', 'de', 'es', 'fi', 'fr', 'hu', 'it', 'ja', 'ko', 'nl', 'no', 'pl', 'pt', 'pt-br', 'ro', 'ru', 'sv', 'tr', 'zh-hans', 'zh-hant']

def main(w):
  all_pages = {language: set() for language in LANGS}
  all_english_pages = set()
  for page in w.get_all_pages(namespaces=['Main', 'Help']):
    if 'OTFWH' in page.title: # ETF2L Highlander Community Challenge/OTFWH
      pass # Do not translate
    elif page.lang == 'en':
      all_english_pages.add(page)
    else:
      all_pages[page.lang].add(page)

  outputs = []
  for language in LANGS:
    output = """\
{{{{DISPLAYTITLE: {count} pages in {{{{lang name|name|{lang}}}}}}}}}
All articles in {{{{lang info|{lang}}}}}; '''<onlyinclude>{count}</onlyinclude>''' in total. Data as of {date}.

; See also
* [[Project:Reports/Missing translations/{lang}|Missing translations in {{{{lang name|name|{lang}}}}}]]
* [[Project:Reports/Missing categories/{lang}|Missing categories in {{{{lang name|name|{lang}}}}}]]
* [[Special:RecentChangesLinked/Project:Reports/All articles/{lang}|Recent changes to articles in {{{{lang name|name|{lang}}}}}]]

== List ==""".format(
      lang=language,
      count=len(all_pages[language]),
      date=time_and_date())

    if language == 'en': # Fixup, because english doesn't have translations
      output = output.replace('{{lang name|name|en}}', 'English')
      output = sub('\n.*?Missing translations/en.*?\n', '\n', output)

    for page in sorted(all_pages[language]):
      output += f'\n# [[{page.title}]]'
    outputs.append([language, output])

  english_output = """\
{{{{DISPLAYTITLE: {count} pages in {{{{lang name|name|en}}}}}}}}
List of all English articles; <onlyinclude>{count}</onlyinclude> in total. Data as of {date}.

* ''See also:'' [[Special:RecentChangesLinked/Team Fortress Wiki:Reports/All articles/en|Recent changes to English articles]]

== List ==""".format(
    count=len(all_english_pages),
    date=time_and_date())
  for page in sorted(all_english_pages):
    english_output += f'\n# [[{page.title}]]'

  outputs.append(['en', english_output])

  return outputs

if __name__ == '__main__':
  verbose = True
  w = wiki.Wiki('https://wiki.teamfortress.com/w/api.php')
  with open('wiki_all_articles.txt', 'w') as f:
    for lang, output in main(w):
      f.write(f'\n===== {lang} =====\n')
      f.write(output)
  print(f'Article written to {f.name}')


#!/usr/bin/env python

import argparse
import logging
import os
import sys
from gdutils import GdacClient
from gdutils.plot.plotter import ErddapPlotter

def main(args):
    """Request and download a map of the profile positions for the specified dataset_id"""
    # Set up logger
    log_level = getattr(logging, args.loglevel.upper())
    log_format = '%(asctime)s:%(module)s:%(levelname)s:%(message)s [line %(lineno)d]'
    logging.basicConfig(format=log_format, level=log_level)

    dataset_id = args.dataset_id
    img_path = args.directory
    img_type = args.img_type
    marker_color = args.color
    colorbar = args.colorbar
    profiles = args.profiles
    no_legend = args.no_legend
    debug = args.debug
    dataset_variable = args.dataset_variable
    plot_all = args.plot_all
    hours = args.hours
    start_date = args.start_date
    end_date = args.end_date

    # Connect to the GDAC ERDDAP server
    client = GdacClient()
    # Fetch the dataset
    client.search_datasets(dataset_ids=dataset_id)
    if client.datasets.empty:
        logging.error('Dataset not found: {:}'.format(dataset_id))
        return 1

    # Create the ploter
    plotter = ErddapPlotter(client.server, response=img_type)

    # Configure the plot parameters
    plotter.set_colorbar(colorbar=colorbar)
    plotter.set_marker_color(marker_color)
    plotter.set_y_range(min_val=0)
    if no_legend:
        plotter.set_legend_loc('Off')
        plotter.set_trim_pixels()

    # Set up time window
    if not plot_all:
        if not start_date and not end_date:
            plotter.add_constraint('time>=', 'max(time)-{:}hours'.format(hours))
        else:
            if start_date:
                plotter.add_constraint('time>=', start_date)
            if end_date:
                plotter.add_constraint('time<=', end_date)

    ext = img_type[-3:].lower()

    img_url = plotter.build_image_request(dataset_id, 'time', 'depth', dataset_variable)
    image_name = os.path.join(img_path, '{:}_{:}_ts_{:}.{:}'.format(dataset_id, dataset_variable, img_type, ext))
    if profiles:
        img_url = plotter.build_image_request(dataset_id, dataset_variable, 'depth', 'time')
        image_name = os.path.join(img_path, '{:}_{:}_profiles_{:}.{:}'.format(dataset_id, dataset_variable, img_type, ext))
        if marker_color:
            img_url = plotter.build_image_request(dataset_id, dataset_variable, 'depth')

    if debug:
        logging.info('Image request: {:}'.format(img_url))
    else:
        logging.info('Requesting and downloading image {:}'.format(image_name))
        logging.debug('Image url: {:}'.format(img_url))
        img_path = plotter.download_image(img_url, image_name)
        if img_path:
            sys.stdout.write('{:}\n'.format(img_path))

    return 0

if __name__ == '__main__':
    arg_parser = argparse.ArgumentParser(description=main.__doc__,
                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    arg_parser.add_argument('dataset_id',
                            help='ERDDAP glider dataset id',
                            type=str)

    arg_parser.add_argument('dataset_variable',
                            help='Dataset variable to plot',
                            type=str)

    arg_parser.add_argument('-a', '--all',
                            dest='plot_all',
                            help='Plot the entire time series',
                            action='store_true')

    arg_parser.add_argument('--hours',
                            help='Plot the last hours of the time series',
                            default=24)

    arg_parser.add_argument('--start_date',
                            help='Plot data >= the specified date')

    arg_parser.add_argument('--end_date',
                            help='Plot data <= the specified date')

    arg_parser.add_argument('-d', '--directory',
                            help='Directory to write the images',
                            type=str)

    arg_parser.add_argument('-f', '--format',
                            help='Image type',
                            dest='img_type',
                            choices=['largePng', 'png', 'smallPng', 'largePdf', 'pdf', 'smallPdf', 'transparentPng'],
                            default='largePng')

    arg_parser.add_argument('-c', '--color',
                            help='Plot the positions using the specified color instead of color coding by timestamp',
                            type=str)

    arg_parser.add_argument('--colorbar',
                            help='Any valid ERDDAP plotting colorbar',
                            type=str,
                            default='Rainbow2')

    arg_parser.add_argument('-p', '--profiles',
                            help='Plot profiles',
                            action='store_true')

    arg_parser.add_argument('--no-legend',
                            action='store_true',
                            dest='no_legend',
                            help='Do not include a legend')

    arg_parser.add_argument('-x', '--debug',
                            help='Debug mode. No operations performed',
                            action='store_true')

    arg_parser.add_argument('-l', '--loglevel',
                            help='Verbosity level',
                            type=str,
                            choices=['debug', 'info', 'warning', 'error', 'critical'],
                            default='info')

    parsed_args = arg_parser.parse_args()

    # sys.stdout.write('{:}'.format(parsed_args))
    # sys.exit(13)

    sys.exit(main(parsed_args))


from __future__ import print_function, division

import os
import sys
import json
import pickle
import argparse

import torch
import torchvision
import numpy as np
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision.datasets import ImageFolder
from torchvision import transforms, utils, datasets

from model import *

def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")

def load_data(opts):

    transform = transforms.Compose(
                          [transforms.Scale((256, 256)),
                           transforms.ToTensor(),
                           transforms.Normalize(mean = opts.mean, std = opts.std)])

    dataset = ImageFolder(root=opts.dataset_path,
                           transform=transform)
    dataloader = DataLoader(dataset, batch_size=opts.batch_size,
                             shuffle=True, num_workers=opts.num_workers)

    return dataloader

def train(opts):
    encoder = Encoder()
    decoder = Decoder()
    # discriminator = Discriminator()
    if torch.cuda.is_available():
        encoder = encoder.cuda()
        decoder = decoder.cuda()
        # discriminator = discriminator.cuda()
    # Set learning rates
    gen_lr, reg_lr = 0.0006, 0.0008

    # Set optimizators
    dec_optimizer = optim.Adam(decoder.parameters(), lr=gen_lr)
    enc_optimizer = optim.Adam(encoder.parameters(), lr=gen_lr)
    gen_optimizer = optim.Adam(encoder.parameters(), lr=reg_lr)
    disc_optimizer = optim.Adam(discriminator.parameters(), lr=reg_lr)

    dataloader = load_data(opts)

    for epoch in range(opts.epochs):  # loop over the dataset multiple times
        # exp_lr_scheduler.step()

        print('Running epoch %d' % epoch)
        for i, data in enumerate(dataloader):

            if i%15 == 0:
                print('Batch id : %d' % i)

            # get the inputs
            X, Y = data
            # wrap them in Variable
            X, Y = Variable(X), Variable(Y)
            # convert to cuda if available
            if opts.cuda:
                X = X.cuda()
                Y = Y.cuda()

            # zero the parameter gradients
            dec_optimizer.zero_grad()
            enc_optimizer.zero_grad()
            gen_optimizer.zero_grad()
            disc_optimizer.zero_grad()

            # reconstruction loss
            z_sample = encoder(X)
            X_sample = decoder(z_sample)
            recon_loss = F.binary_cross_entropy(X_sample, X)
            recon_loss.backward()
            dec_optimizer.step()
            enc_optimizer.step()

            # generate fake samples from encoder
            encoder.eval()
            z_real_gauss = Variable(torch.randn(X.size(0), 256))
            if torch.cuda.is_available():
                z_real_gauss = z_real_gauss.cuda()
            z_fake_gauss = encoder(X)

            # train discriminator
            discriminator.train()
            D_real_gauss, D_fake_gauss = discriminator(z_real_gauss), discriminator(z_fake_gauss)
            D_loss = -torch.mean(torch.log(D_real_gauss) + torch.log(1 - D_fake_gauss))
            D_loss.backward()       # Backpropagate loss
            disc_optimizer.step()   # Apply optimization step

            # train generator
            encoder.train()   # Back to use dropout
            z_fake_gauss = encoder(X)
            D_fake_gauss = discriminator(z_fake_gauss)

            G_loss = -torch.mean(torch.log(D_fake_gauss))
            G_loss.backward()
            gen_optimizer.step()

            if i % 15 == 0:
                sample = Variable(torch.randn(16, 256))
                if opts.cuda:
                    sample = sample.cuda()
                sample = decoder(sample).cpu()
                sample = (sample * opts.std) + opts.mean
                img_name = 'results/sample_' + str(epoch) + '_' + str(i) + '.png'
                utils.save_image(sample.data.view(16, 3, 256, 256),
                           os.path.join(opts.save_path, img_name))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset_path', type=str, default='clock_data/')
    parser.add_argument('--epochs', type=int, default=50)
    parser.add_argument('--lr', type=float, default=1e-3)
    parser.add_argument('--momentum', type=float, default=0.9)
    parser.add_argument('--batch_size', type=int, default=100)
    parser.add_argument('--num_workers', type=int, default=2)
    parser.add_argument('--cuda', type=str2bool, default=False)
    parser.add_argument('--save_path', type=str, default='')
    parser.add_argument('--load_model', type=str, default='')
    parser.add_argument('--lr_step_size', type=int, default=10, help='step size for LR scheduler')
    parser.add_argument('--weight_decay', type=float, default=0.0)
    parser.add_argument('--lr_scale', type=float, default=0.1)

    opts = parser.parse_args()
    opts.mean = [0.485, 0.456, 0.406]
    opts.std = [0.229, 0.224, 0.225]

    json.dump(vars(opts), open(os.path.join(opts.save_path, 'opts.json'), 'w'))
    train(opts)