'''Functions used in river_tracker1.py

Author: guangzhi XU (xugzhi1987@gmail.com; guangzhi.xu@outlook.com)
Update time: 2019-05-10 11:03:36.
'''

from __future__ import print_function
import numpy as np
import pandas as pd
import networkx as nx
from skimage import measure
from skimage import morphology
from scipy import ndimage
import matplotlib.pyplot as plt

import cdms2 as cdms
import MV2 as MV
from genutil import statistics as stats
import cdutil

from utils import rdp
from utils import funcs
from utils import peak_prominence2d as pp2d

NX_VERSION=nx.__version__[0]

def plotGraph(graph,ax=None,show=True):
    '''Helper func to plot the graph of an AR coordinates
    '''

    if ax is None:
        fig=plt.figure()
        ax=fig.add_subplot(111)

    pos=[(ii[1],ii[0]) for ii in graph.nodes()]  # x,y
    pos_dict=dict(zip(graph.nodes(),pos))
    nx.draw(graph,ax=ax,pos=pos_dict,node_size=15,node_color='darkgray',
            edge_color='dimgray')
    if show:
        plt.show(block=False)
    return

def areaFilt(mask,area,min_area=None,max_area=None):
    '''Filter AR binary masks by region areas

    Args:
        mask (ndarray): 2D binary mask with detected objects shown as 1s.
        area (ndarray): 2D map showing grid cell areas in km^2.
        min_area (float or None): if not None, minimum area to filter objects
                                  in <mask>.
        max_area (float or None): if not None, maximum area to filter objects
                                  in <mask>.

    Returns:
        result (ndarray): 2D binary mask with objects area-filtered.
    '''

    if min_area is None and max_area is None:
        return mask

    labels=measure.label(mask,connectivity=1)
    n=labels.max()+1
    areas=ndimage.sum(area,labels,np.arange(n))
    sel=np.ones(n,bool)
    if min_area is not None:
        sel=np.where(areas<min_area,0,sel)
    if max_area is not None:
        sel=np.where(areas>max_area,0,sel)
    # remove background area
    sel[0]=0
    result=sel[labels]

    return result

def spherical2Cart(lat,lon):
    clat=(90-lat)*np.pi/180.
    lon=lon*np.pi/180.
    x=np.cos(lon)*np.sin(clat)
    y=np.sin(lon)*np.sin(clat)
    z=np.cos(clat)

    return np.array([x,y,z])

def cart2Spherical(x,y,z, shift_lon):

    r=np.sqrt(x**2+y**2+z**2)
    clat=np.arccos(z/r)/np.pi*180
    lat=90.-clat
    lon=np.arctan2(y,x)/np.pi*180
    lon=(lon+360)%360

    lon=np.where((lon>=0) & (lon<shift_lon),lon+360,lon)

    return np.array([lat,lon,np.ones(lat.shape)])

def computeTheta(p1,p2):
    '''Tangent line to the arc |p1-p2|
    <p1>,<p2>: (lat,lon) coordinates
    '''
    p1=spherical2Cart(p1[0],p1[1])
    p2=spherical2Cart(p2[0],p2[1])
    theta=p2-np.dot(p1,p2)*p1
    norm=np.linalg.norm(theta)
    if norm>0:
        theta=theta/norm
    return theta

def wind2Cart(u,v,lats,lons):
    '''Convert u,v winds to Cartesian, consistent with spherical2Cart.
    '''

    latsr=lats*np.pi/180
    lonsr=lons*np.pi/180
    vh=v*np.sin(latsr)

    ux=-u*np.sin(lonsr) - vh*np.cos(lonsr)
    uy=u*np.cos(lonsr) - vh*np.sin(lonsr)
    uz=v*np.cos(latsr)
    vs=np.array([ux,uy,uz])

    return vs

def cart2Wind(vs,lats,lons):
    '''Convert winds in Cartesian to u,v, inverse to wind2Cart.
    '''

    latsr=lats*np.pi/180
    lonsr=lons*np.pi/180

    #ux=vs[0]
    uy=vs[1]
    uz=vs[2]

    u=uy/np.cos(lonsr) + uz*np.tan(latsr)*np.tan(lonsr)
    v=uz/np.cos(latsr)

    return u,v

def maskToGraph(mask, quslab, qvslab, costhetas, sinthetas, edge_eps,
        connectivity=2):
    '''Create graph from AR mask

    Args:
        mask (ndarray): 2D binary map showing the location of an AR with 1s.
        quslab (cdms.TransientVariable): 2D map of u-flux.
        qvslab (cdms.TransientVariable): 2D map of v-flux.
        costhetas (cdms.TransientVariable): (n * m) 2D slab of grid cell shape:
                                          cos=dx/sqrt(dx^2+dy^2).
        sinthetas (cdms.TransientVariable): (n * m) 2D slab of grid cell shape:
                                          sin=dy/sqrt(dx^2+dy^2).
        edge_eps (float): float in (0,1), minimal proportion of flux component
                          in a direction to total flux to allow edge building
                          in that direction. Defined in Global preamble.
        connectivity (int): 1 or 2. 4- or 8- connectivity in defining neighbor-
                            hood relationship in a 2D square grid.
    Returns:
        g (networkx.DiGraph): directed planar graph constructed from AR mask
                              and flows.
    '''

    quslab=np.array(quslab)
    qvslab=np.array(qvslab)
    wsslab=np.sqrt(quslab**2+qvslab**2)

    g=nx.DiGraph()

    # 1 connectivity edges
    # the list approach
    '''
    y,x=np.where(mask)
    zipcor=zip(y,x)
    right=[(yi,xi) for yi,xi in zipcor if (yi,xi+1) in zipcor]
    left=[(yi,xi) for yi,xi in zipcor if (yi,xi-1) in zipcor]
    up=[(yi,xi) for yi,xi in zipcor if (yi+1,xi) in zipcor]
    down=[(yi,xi) for yi,xi in zipcor if (yi-1,xi) in zipcor]

    # nodes to the right/left/up/down
    right0=[(yi,xi+1) for yi,xi in right]
    left0=[(yi,xi-1) for yi,xi in left]
    up0=[(yi+1,xi) for yi,xi in up]
    down0=[(yi-1,xi) for yi,xi in down]
    '''

    # the shifting approach
    right=np.roll(mask, -1, axis=1)*mask
    left=np.roll(mask, 1, axis=1)*mask
    up=np.roll(mask, -1, axis=0)*mask
    down=np.roll(mask, 1, axis=0)*mask

    def addWeightedEdges2(nodes1,speedslab,d):
        '''Add directed edges to graph. For shifting approach
        '''

        ratio=np.where(wsslab==0., 0, speedslab/wsslab)
        idx=np.where(ratio>=edge_eps, 1, 0)*nodes1
        idx=zip(*np.where(idx>0))

        for ii, (yii,xii) in enumerate(idx):
            # nii: start, nii2: end
            yii=int(yii)
            xii=int(xii)
            nii=(yii,xii)
            if d=='r':
                nii2=(yii,xii+1)
            elif d=='l':
                nii2=(yii,xii-1)
            elif d=='u':
                nii2=(yii+1,xii)
            elif d=='d':
                nii2=(yii-1,xii)
            elif d=='tr':
                nii2=(yii+1,xii+1)
            elif d=='br':
                nii2=(yii-1,xii+1)
            elif d=='tl':
                nii2=(yii+1,xii-1)
            elif d=='bl':
                nii2=(yii-1,xii-1)

            meanivt=speedslab[yii,xii]

            g.add_edge(nii,nii2,
                    weight=np.exp(-meanivt/1e2),
                    ivt=meanivt)

    def addWeightedEdges(nodes1,nodes2,speedslab):
        '''Add directed edges to graph. For the list approach
        '''

        # nii: start, nii2: end
        for nii,nii2 in zip(nodes1,nodes2):
            if speedslab[nii]/wsslab[nii]>=edge_eps:
                meanivt=speedslab[nii]

                g.add_edge(nii,nii2,
                        weight=np.exp(-meanivt/1e2),
                        ivt=meanivt)

    # add 1 connectivity edges
    #addWeightedEdges(right,right0,quslab)
    #addWeightedEdges(left,left0,-quslab)
    #addWeightedEdges(up,up0,qvslab)
    #addWeightedEdges(down,down0,-qvslab)
    addWeightedEdges2(right,quslab,'r')
    addWeightedEdges2(left,-quslab,'l')
    addWeightedEdges2(up,qvslab,'u')
    addWeightedEdges2(down,-qvslab,'d')

    # 2 connectivity edges
    if connectivity==2:
        # the list approach
        '''
        tr=[(yi,xi) for yi,xi in zipcor if (yi+1,xi+1) in zipcor]
        br=[(yi,xi) for yi,xi in zipcor if (yi-1,xi+1) in zipcor]
        tl=[(yi,xi) for yi,xi in zipcor if (yi+1,xi-1) in zipcor]
        bl=[(yi,xi) for yi,xi in zipcor if (yi-1,xi-1) in zipcor]

        tr0=[(yi+1,xi+1) for yi,xi in tr]
        br0=[(yi-1,xi+1) for yi,xi in br]
        tl0=[(yi+1,xi-1) for yi,xi in tl]
        bl0=[(yi-1,xi-1) for yi,xi in bl]
        '''

        # the shifting approach
        tr=np.roll(np.roll(mask, -1, axis=0), -1, axis=1)*mask
        br=np.roll(np.roll(mask, 1, axis=0), -1, axis=1)*mask
        tl=np.roll(np.roll(mask, -1, axis=0), 1, axis=1)*mask
        bl=np.roll(np.roll(mask, 1, axis=0), 1, axis=1)*mask

        # add 2 connectivity edges
        #addWeightedEdges(tr,tr0,quslab*costhetas+qvslab*sinthetas)
        #addWeightedEdges(br,br0,quslab*costhetas-qvslab*sinthetas)
        #addWeightedEdges(tl,tl0,-quslab*costhetas+qvslab*sinthetas)
        #addWeightedEdges(bl,bl0,-quslab*costhetas-qvslab*sinthetas)
        addWeightedEdges2(tr,quslab*costhetas+qvslab*sinthetas,'tr')
        addWeightedEdges2(br,quslab*costhetas-qvslab*sinthetas,'br')
        addWeightedEdges2(tl,-quslab*costhetas+qvslab*sinthetas,'tl')
        addWeightedEdges2(bl,-quslab*costhetas-qvslab*sinthetas,'bl')

    return g

def getARAxis(g, quslab, qvslab, mask):
    '''Find AR axis from AR region mask

    Args:
        g (networkx.DiGraph): directed planar graph constructed from AR mask
                              and flows. See maskToGraph().
        quslab (cdms.TransientVariable): 2D map of u-flux.
        qvslab (cdms.TransientVariable): 2D map of v-flux.
        mask (ndarray): 2D binary map showing the location of an AR with 1s.

    Returns:
        path (ndarray): Nx2 array storing the AR axis coordinate indices in
                        (y, x) format.
        axismask (ndarray): 2D binary map with same shape as <mask>, with
                            grid cells corresponding to coordinates in <path>
                            set to 1s.
    '''

    nodes=list(g.nodes())

    #---------------Find boundary nodes---------------
    edge=mask-morphology.binary_erosion(mask)

    gy,gx=np.gradient(np.array(mask))
    inedge=(gx*quslab+gy*qvslab)*edge
    inedgecoor=np.where(inedge>0)
    inedgecoor=zip(inedgecoor[0],inedgecoor[1])
    inedgecoor=list(set(inedgecoor).intersection(nodes))

    outedgecoor=np.where(inedge<0)
    outedgecoor=zip(outedgecoor[0],outedgecoor[1])
    outedgecoor=list(set(outedgecoor).intersection(nodes))

    n1=len(inedgecoor)
    n2=len(outedgecoor)

    # when mask is at edge of the map. Rarely happens.
    if n1==0:
        inedgecoor=nodes
        n1=len(inedgecoor)
    if n2==0:
        outedgecoor=nodes
        n2=len(outedgecoor)

    dists=np.zeros((n1,n2))

    def sumDists(path,attr,g):
        '''Sum edge distances along a path'''
        s=0
        for ii in range(len(path)-1):
            if NX_VERSION=='2':
                sii=g[path[ii]][path[ii+1]][attr]
            else:
                sii=g.edge[path[ii]][path[ii+1]][attr]

            # penalize sharp turns. Doesn't make big difference but notably
            # slower
            '''
            if ii+2<len(path):
                pii1=(lats[path[ii][0]], lons[path[ii][1]])
                pii2=(lats[path[ii+1][0]], lons[path[ii+1][1]])
                pii3=(lats[path[ii+2][0]], lons[path[ii+2][1]])
                theta1=computeTheta(pii1,pii2)
                theta2=computeTheta(pii2,pii3)
                dtheta=theta1.dot(theta2)
                dtheta=abs(dtheta)**1

                #if ii==0:
                    #dtheta_old=1.

                #dtheta=np.mean([dtheta,dtheta_old])
                #sii=sii*dtheta
                #dtheta_old=dtheta
            '''
            s+=sii

        return s

    #---------------Find "longest" path---------------
    for ii in range(n1):
        eii=inedgecoor[ii]
        pathsii=nx.single_source_dijkstra_path(g,eii,weight='weight')
        pathsii=dict([(kk,vv) for kk,vv in pathsii.items() if kk in outedgecoor])
        if len(pathsii)>0:
            distdict=dict([(kk, sumDists(vv,'ivt',g)) for kk,vv in pathsii.items()])
            nodeii=sorted(distdict,key=distdict.get)[-1]
            distii=distdict[nodeii]
            dists[ii,outedgecoor.index(nodeii)]=distii

    if np.max(dists)==0:
        # this may happen when a mask is touching the map edges, and inedgecoor
        # outedgecoor can't be linked by a path. Very rarely happen, but damn
        # annoying. A fallback solution is to use an undirected graph linking
        # the most inward and most outward pixels.
        mostin=np.unravel_index(np.argmax(inedge), mask.shape)
        mostout=np.unravel_index(np.argmin(inedge), mask.shape)
        g_und=g.to_undirected()
        try:
            path=nx.dijkstra_path(g_und,mostin,mostout,weight='weight')
        except:
            # if it still can't find a path, make a full connected network
            g_full=maskToGraph(mask, quslab, qvslab, np.ones(mask.shape),
                    np.ones(mask.shape), -np.inf)
            path=nx.dijkstra_path(g_full,mostin,mostout,weight='weight')
    else:
        maxidx=np.argmax(dists)
        yidx,xidx=np.unravel_index(maxidx,(n1,n2))
        path=nx.dijkstra_path(g,inedgecoor[yidx],outedgecoor[xidx],weight='weight')

    # get a mask for axis
    axismask=np.zeros(mask.shape)
    for (y,x) in path:
        axismask[y,x]=1

    path=np.array(path)

    return path, axismask

def cropMask(mask, edge=4):
    '''Cut out a bounding box around mask==1 areas

    Args:
        mask (ndarray): 2D binary map showing the location of an AR with 1s.
        edge (int): number of pixels as edge at 4 sides.

    Returns:
        mask[y1:y2, x1:x2] (ndarray): a sub region cut from <mask> surrouding
                                      regions with value=1.
        (yy,xx): y-, x- indices of the box of the cut region. Can later by
                 used in applyCropIdx(new_slab, (yy,xx)) to crop out the same
                 region from a new array <new_slab>.
    '''

    yidx,xidx=np.where(mask==1)
    if len(yidx)==0:
        raise Exception("mask empty")

    y1=np.min(yidx)
    y2=np.max(yidx)
    x1=np.min(xidx)
    x2=np.max(xidx)

    y1=max(0,y1-edge)
    y2=min(mask.shape[0],y2+edge)
    x1=max(0,x1-edge)
    x2=min(mask.shape[1],x2+edge)
    xx=np.arange(x1,x2)
    yy=np.arange(y1,y2)

    return mask[y1:y2,x1:x2], (yy,xx)

def applyCropIdx(slab, cropidx):
    '''Cut out a bounding box from given 2d slab given corner indices

    Args:
        slab (ndarray): 2D array to cut a box from.
        cropidx (tuple): (y, x) coordinate indices, output from cropMask().

    Returns:
        cropslab (ndarray): 2D sub array cut from <slab> using <cropidx> as
                            boundary indices.
    '''

    cropslab=np.array(slab)[np.ix_(*cropidx)]
    try:
        croplat=slab.getLatitude()[:][cropidx[0]]
        croplon=slab.getLongitude()[:][cropidx[1]]

        croplat=cdms.createAxis(croplat)
        croplat.designateLatitude()
        croplat.id='y'
        croplat.units='degree'
        croplat.name='latitude'

        croplon=cdms.createAxis(croplon)
        croplon.designateLongitude()
        croplon.id='x'
        croplon.units='degree'
        croplon.name='longitude'

        cropslab=MV.array(cropslab)
        cropslab.setAxis(0,croplat)
        cropslab.setAxis(1,croplon)
    except:
        pass

    return cropslab

def insertCropSlab(shape, cropslab, cropidx, axislist=None):
    '''Insert the cropped sub-array back to a larger empty slab

    Args:
        shape (tuple): (n, m) size of the larger slab.
        cropslab (ndarray): 2D array to insert.
        cropidx (tuple): (y, x) coordinate indices, output from cropMask(),
                         defines where <cropslab> will be inserted into.
    Kwargs:
        axislist (list or None): if list, a list of cdms.TransientAxis objs.

    Returns:
        result (ndarray): 2D slab with shape (n, m), an empty array with a
                          box at <cropidx> replaced with data from <cropslab>.
                          Optionally, axes information is added if <axistlist>
                          is not None, making it an TransientVariable.
    '''

    result=np.zeros(shape)
    result[np.ix_(*cropidx)]=cropslab

    if axislist is not None:
        result=MV.array(result)
        result.setAxisList(axislist)

    return result

def getMaskEdge(mask):
    '''Get the ordered boundary cell indices around non-zeros values in a
    binary mask

    Args:
        mask (ndarray): 2D binary mask.

    Returns:
        edge (ndarray): Nx2 array storing (x, y) coordinate indices of the
                        grid cells in <mask> that form the boundary of
                        objects defined as non-zero values.
    '''
    edge=mask-morphology.binary_erosion(mask)
    edge=np.where(edge>0)
    edge=zip(edge[1],edge[0])
    edge=np.array(edge)
    edge=funcs.getLineFromPoints(edge)

    return edge

def partPeaks(cropmask, cropidx, orislab, max_ph_ratio):
    '''Separate local maxima by topographical prominence

    Args:
        cropmask (ndarray): 2D binary array, defines regions of local maxima.
        cropidx (tuple): (y, x) coordinate indices, output from cropMask().
        orislab (ndarray): 2D array, giving magnitude/height/intensity values
                           defining the topography.
        max_ph_ratio (float): maximum peak/height ratio. Local peaks with
                              a peak/height ratio larger than this value is
                              treated as an independent peak.

    Returns:
        result (ndarray): 2D binary array, similar as the input <cropmask>
                          but with connected peaks (if any) separated so that
                          each connected region (with 1s) denotes an
                          independent local maximum.
    '''

    cropslab=applyCropIdx(orislab,cropidx)
    if 0 in cropidx[0] or 0 in cropidx[1] or orislab.shape[0]-1 in\
            cropidx[0] or orislab.shape[1]-1 in cropidx[1]:
        include_edge=True
    else:
        include_edge=False

    # compute prominences
    peaks,peakid,peakpro,peakparents=pp2d.getProminence(cropslab*cropmask,
            10.,include_edge=include_edge,centroid_num_to_center=1,verbose=False)

    peakheights=(peakpro>0)*cropslab*cropmask
    ratios=cropmask*peakpro/peakheights

    # take maxima whose prominence/height ratio> max_ph_ratio
    localpeaks=np.where(ratios>max_ph_ratio)
    localpeaks=zip(localpeaks[0],localpeaks[1])

    mask1=np.zeros(cropmask.shape)    # modified mask
    # residual mask, the union of complimentary masks. A complimentary mask
    # is the sea level mask that separates a peak from its parent. Note that
    # peaks' sea levels are not necessarily at the same height.
    resmask=np.zeros(cropmask.shape)

    def breakPeaks(yidx,xidx,localpeaks,col):
        '''Separate the contour of a peak from its parent by iteratively
        rising the sea level
        '''
        labels=morphology.label(cropslab*cropmask>col)
        dropthis=False
        while True:
            #plabels=[labels[yjj,xjj] for yjj,xjj in localpeaks]
            plabels=[labels[yjj,xjj] for yjj,xjj in localpeaks if labels[yjj, xjj]==labels[yidx, xidx]]
            #if len(set(plabels))==len(localpeaks):
            if len(plabels)==1:
                break
            col+=5.
            if col>cropslab[yidx,xidx]:
                dropthis=True
                col-=5.
                break
            labels=morphology.label(cropslab*cropmask>col)

        tmpmask=np.zeros(cropmask.shape)
        if not dropthis:
            tmpmask[yidx,xidx]=1
            tmpmask=morphology.reconstruction(tmpmask,cropslab>col,'dilation')

        return tmpmask,col

    if len(localpeaks)==1:
        mask1=cropmask
    else:
        # sort by prominence/height ratios
        ratios=[ratios[int(yjj), int(xjj)] for yjj,xjj in localpeaks]
        heights=[peakheights[int(yjj), int(xjj)] for yjj, xjj in localpeaks]
        sortidx=np.argsort(ratios)
        ratios.sort()
        localpeaks=[localpeaks[idjj] for idjj in sortidx]
        '''
        localpeakids=[peakid[yjj,xjj] for yjj,xjj in localpeaks]
        localpeakcols=[peaks[idjj]['col_level'] for idjj in localpeakids]
        c_col=np.min(localpeakcols)-10

        labs=morphology.label(cropslab*cropmask>c_col)
        while True:
            if c_col>np.max(cropslab*cropmask):
                break
            plabels=[labs[yjj,xjj] for yjj,xjj in localpeaks]
            if len(set(plabels))==len(localpeaks):
                break
            c_col+=5.
            labs=morphology.label(cropslab*cropmask>c_col)
        mask1=morphology.reconstruction(localpeaks_map, cropslab>c_col, 'dilation')
        '''

        for yjj,xjj in localpeaks:
            yjj=int(yjj)
            xjj=int(xjj)
            idjj=peakid[yjj,xjj]
            coljj=peaks[idjj]['col_level']
            if peaks[idjj]['parent']==0 and peakheights[yjj,xjj]==np.max(heights):
                # the heighest peak
                tmpmask=np.zeros(cropslab.shape)
                tmpmask[yjj,xjj]=1
                tmpmask=morphology.reconstruction(tmpmask,
                        (cropmask-resmask)>0,'dilation')
                mask1=mask1+tmpmask
            else:
                # separate local peaks
                tmpmask,coljj2=breakPeaks(yjj,xjj,localpeaks,coljj)
                # if childrens overlap, may not need this anymore
                if (tmpmask+mask1).max()>1:
                    tmpmask2=np.zeros(cropmask.shape)
                    tmpmask2[yjj,xjj]=1
                    tmpmask=morphology.reconstruction(tmpmask2,tmpmask-tmpmask*resmask,'dilation')
                mask1=mask1+tmpmask
                resmask=np.where((resmask==1) | (cropslab<coljj2),1,0)

    result=insertCropSlab(orislab.shape,mask1,cropidx)

    return result

def getARData(slab, quslab, qvslab, anoslab, quano, qvano, areas,
        mask_list, axis_list, timestr, param_dict, shift_lon, isplot,
        outputdir):
    '''Fetch AR related data

    Args:
        slab (cdms.TransientVariable): (n * m) 2D array of IVT, in kg/m/s.
        quslab (cdms.TransientVariable): (n * m) 2D array of u-flux, in kg/m/s.
        qvslab (cdms.TransientVariable): (n * m) 2D array of v-flux, in kg/m/s.
        anoslab (cdms.TransientVariable): (n * m) 2D array of IVT anomalies,
                                        in kg/m/s.
        quano (cdms.TransientVariable): (n * m) 2D array of u-flux anomalies,
                                        in kg/m/s.
        qvano (cdms.TransientVariable): (n * m) 2D array of v-flux anomalies,
                                        in kg/m/s.
        areas (cdms.TransientVariable): (n * m) 2D grid cell area slab, in km^2.
        mask_list (list): list of 2D binary masks, each with the same shape as
                      <anoslab> etc., and with 1s denoting the location of a
                      found AR.
        axis_list (list): list of AR axis coordinates. Each coordinate is defined
                     as a Nx2 ndarray storing (y, x) indices of the axis
                     (indices defined in the matrix of corresponding mask
                     in <masks>.)
        timestr (str): string of time snap.
        param_dict (dict): parameter dict defined in Global preamble.
        shift_lon (float): starting longitude of data domain, defined in Global
                           preamble.
        isplot (bool): if True, create plot of AR axis, flux orientation and
                       cross-sectional flux for each AR.
        outputdir (str): folder to save plots. If None, don't save. If <isplot>
                         is False, not relevant.

    Returns:
        labels (cdms.TransientVariable): (n * m) 2D int map showing all ARs
                                       at current time. Each AR is labeled by
                                       an int label, starting from 1. Background
                                       is filled with 0s.
        angles (cdms.TransientVariable): (n * m) 2D map showing orientation
                                       differences between AR axes and fluxes,
                                       for all ARs. In degrees.
        crossfluxes (cdms.TransientVariable): (n * m) 2D map showing cross-
                                           sectional fluxes in all ARs.
                                           In kg/m/s.
        anocrossflux (cdms.TransientVariable): similar as <crossfluxes> but for
                                             anomalous fluxes (corresponding
                                             to <anoslab>).
        df (pandas.DataFrame): AR record table. Each row is an AR, see code
                               below for columns.
    '''

    max_isoq=param_dict['max_isoq']
    min_length=param_dict['min_length']
    min_length_hard=param_dict['min_length_hard']
    rdp_thres=param_dict['rdp_thres']
    min_area=param_dict['min_area']

    lonax=slab.getLongitude()   # NOTE: max > 360
    latax=slab.getLatitude()

    # prepare outputs
    labels=MV.zeros(slab.shape)
    angles=MV.zeros(slab.shape)
    crossfluxes=MV.zeros(slab.shape)
    results={}

    #-----------------Loop through ARs-----------------
    for ii in range(len(mask_list)):

        maskii=mask_list[ii]

        # region properties, in pixel units
        rpii=measure.regionprops(maskii, intensity_image=np.array(slab))[0]

        # get centroid
        centroidy,centroidx=rpii.weighted_centroid
        centroidy=latax[int(centroidy)]
        centroidx=lonax[int(centroidx)]

        # get axis coordinate array
        skelii=axis_list[ii]
        latsii=latax[skelii[:,0]]
        lonsii=lonax[skelii[:,1]]
        axisii=np.c_[latsii,lonsii]

        # segment axis using rdp
        axis_rdpii=np.array(rdp.rdpGC(axisii.tolist(),rdp_thres))  # lat,lon

        # area
        areaii=(maskii*areas).sum()  # km^2

        # compute length
        lens=funcs.greatCircle(axis_rdpii[:-1,0], axis_rdpii[:-1,1],
                axis_rdpii[1:,0], axis_rdpii[1:,1])/1e3
        lenii=lens.sum() #km

        # skip if too small and too short
        if areaii<min_area or lenii<min_length_hard:
            continue

        # mean width
        widthii=areaii/lenii # km

        # mask contour
        contii=funcs.getBinContour(maskii,lonax,latax)

        # isoperimetric quotient
        isoquoii=4*np.pi*rpii.area/rpii.perimeter**2

        # length/width ratio
        ratioii=lenii/widthii

        # mean strength
        slabii=MV.masked_where(maskii==0,slab)
        strengthii=cdutil.averager(slabii,axis='xy',
                weights=['generate','generate'])

        # strength std
        strengthstdii=float(stats.std(slabii,axis='xy'))

        # anomaly strength
        anoslabii=MV.masked_where(maskii==0,anoslab)
        anostrengthii=cdutil.averager(anoslabii,axis='xy',
                weights=['generate','generate'])

        # max strength
        max_strengthii=float(MV.max(slabii))

        # compute angles and cross-section flux of total flux
        cropmask,cropidx=cropMask(maskii)
        cropskelii=skelii-np.array([cropidx[0].min(), cropidx[1].min()])

        cropu=applyCropIdx(quslab,cropidx)
        cropv=applyCropIdx(qvslab,cropidx)

        anglesii,anglesmeanii,crossfluxii,seg_thetasii=crossSectionFlux(
                cropmask, cropu, cropv, axis_rdpii)

        # create plots
        if isplot:
            pass
            #plotARCrosssectionFlux(cropmask, cropu, cropv, cropskelii, axis_rdpii,
                    #'%s AR-%d' %(timestr, ii+1), shift_lon, anglesii, anglesmeanii,
                    #crossfluxii, seg_thetasii, outputdir)

        # insert crop back to the big map
        anglesii=insertCropSlab(maskii.shape,anglesii,cropidx,
                slab.getAxisList())
        anglesii=MV.where(maskii==1,anglesii,0)

        crossfluxii=insertCropSlab(maskii.shape,crossfluxii,cropidx,
                slab.getAxisList())
        crossfluxii=MV.where(maskii==1,crossfluxii,0)

        # mean meridional flux
        cropv=applyCropIdx(qvslab,cropidx)
        cropv=MV.masked_where(cropmask==0,cropv)
        qvmeanii=cdutil.averager(cropv,axis='xy',weights=['generate',\
                'generate'])

        # is candidate a strict AR
        is_relaxedii=False
        if isoquoii>max_isoq or ratioii<2:
            is_relaxedii=True
        if lenii<min_length:
            is_relaxedii=True
        if qvmeanii<=0:
            is_relaxedii=True

        labels=labels+maskii*(ii+1)
        angles=angles+anglesii
        crossfluxes=crossfluxes+crossfluxii

        results[ii+1]={
                'id': ii+1,
                'time':timestr,
                'contour_y': contii.vertices[:,1],
                'contour_x': contii.vertices[:,0],
                'centroid_y': centroidy,
                'centroid_x': centroidx,
                'axis_y':axisii[:,0],
                'axis_x':axisii[:,1],
                'axis_rdp_y':axis_rdpii[:,0],
                'axis_rdp_x':axis_rdpii[:,1],
                'area': areaii,
                'length': lenii,
                'width': widthii,
                'iso_quotient':isoquoii,
                'LW_ratio':ratioii,
                'strength':strengthii,
                'strength_ano':anostrengthii,
                'strength_std':strengthstdii,
                'max_strength':max_strengthii,
                'mean_angle': float(anglesmeanii),
                'is_relaxed':is_relaxedii,
                'qv_mean':qvmeanii
                }

    labels.setAxisList(slab.getAxisList())
    angles.setAxisList(slab.getAxisList())
    crossfluxes.setAxisList(slab.getAxisList())

    labels.id='labels'
    labels.long_name='AR labels'
    labels.standard_name=labels.long_name
    labels.title=labels.long_name
    labels.units=''

    angles.id='angles'
    angles.long_name='AR moisture flux orientation difference'
    angles.standard_name=angles.long_name
    angles.title=angles.long_name
    angles.units='degree'

    crossfluxes.id='ivt_cross'
    crossfluxes.long_name='AR total cross sectional moisture flux'
    crossfluxes.standard_name=crossfluxes.long_name
    crossfluxes.title=crossfluxes.long_name
    crossfluxes.units=getattr(slab, 'units', '')

    keys=['id', 'time', 'contour_y', 'contour_x', 'centroid_y', 'centroid_x',
            'axis_y', 'axis_x', 'axis_rdp_y', 'axis_rdp_x',
            'area', 'length', 'width', 'iso_quotient', 'LW_ratio',
            'strength', 'strength_ano', 'strength_std', 'max_strength',
            'mean_angle', 'is_relaxed', 'qv_mean']

    df=pd.DataFrame(results).T
    if len(df)>0:
        df=df[keys]

    return labels,angles,crossfluxes,df

def uvDecomp(u0, v0, i1, i2):
    '''Decompose background-transient components of u-, v- fluxes

    Args:
        u0 (cdms.TransientVariable): nd array of total u-flux.
        v0 (cdms.TransientVariable): nd array of total v-flux.
        i1 (cdms.TransientVariable): nd array of the reconstruction component
                                   of IVT.
        i2 (cdms.TransientVariable): nd array of the anomalous component
                                   of IVT (i2 = IVT - i1).
    Returns:
        u1 (cdms.TransientVariable): nd array of the u-flux component
                                   corresponding to <i1>, i.e. the background
                                   component.
        v1 (cdms.TransientVariable): nd array of the v-flux component
                                   corresponding to <i1>, i.e. the background
                                   component.
        u2 (cdms.TransientVariable): nd array of the u-flux component
                                   corresponding to <i2>, i.e. the transient
                                   component.
        v2 (cdms.TransientVariable): nd array of the v-flux component
                                   corresponding to <i2>, i.e. the transient
                                   component.
    '''

    i0=i1+i2
    v1=v0*i1/i0
    v2=v0*i2/i0
    u1=u0*i1/i0
    u2=u0*i2/i0

    return u1,u2,v1,v2

def save2DF(result_dict):
    '''Save AR records to a pandas DataFrame

    Args:
        result_dict (dict): key: time str in 'yyyy-mm-dd hh:00'
                            value: pandas dataframe. See getARData().

    Returns:
        result_df (pandas.DataFrame): AR record table containing records
                                      from multiple time steps sorted by
                                      time.
    '''

    for ii,kk in enumerate(result_dict.keys()):
        vv=result_dict[kk]
        if ii==0:
            result_df=vv
        else:
            result_df=pd.concat([result_df,vv],axis=0,ignore_index=True)

    result_df['time']=pd.to_datetime(result_df.time)
    result_df=result_df.sort_values(by='time')

    return result_df

def plotAR(ardf, ax, bmap):
    '''Helper function to plot the regions and axes of ARs

    Args:
        ardf (pandas.DataFrame): table containing AR records.
        ax (matplotlib axis): axis to plot onto.
        bmap (Basemap obj): defining the geo map.
    '''

    for ii in range(len(ardf)):

        vv=ardf.iloc[ii]
        isrelaxkk=vv['is_relaxed']

        # plot contour
        px=vv['contour_x']
        py=vv['contour_y']

        px,py=bmap(px,py)
        linewidth=1 if isrelaxkk else 1
        linestyle=':' if isrelaxkk else '-'
        ax.plot(px,py,color='k',linestyle=linestyle,linewidth=linewidth)

        # plot axis
        px=vv['axis_x']
        py=vv['axis_y']

        px,py=bmap(px,py)
        ax.plot(px,py,'g:',linewidth=1.5)

        # plot cross flux text
        '''
        lenkk=vv['length']
        areakk=vv['area']
        widthkk=vv['width']
        cx=float(vv['centroid_x'])%360
        cy=float(vv['centroid_y'])
        cx,cy=bmap(cx,cy)

        strkk=r'ID=%d, $R=%.0f$' %(ii+1,np.sqrt(areakk/3.14)) + '\n'+\
                r'$L = %d km$' %lenkk +'\n'+\
                r'$W = %d km$' %widthkk

        ax.annotate(strkk,xy=(cx,cy),
                horizontalalignment='center',
                verticalalignment='center',
                fontsize=8,
                bbox=dict(facecolor='white',alpha=0.5))
        '''

    return

def getNormalVectors(point_list, idx):
    '''Get the normal vector and the tagent vector to the plane dividing
    2 sections along the AR axis.

    Args:
        point_list (list): list of (lat, lon) coordinates.
        idx (int): index of the point in <point_list>, denoting the point
                   in question.

    Returns:
        normi (tuple): the (x, y, z) Cartesian coordinate of the unit
                       normal vector, at the point denoted by <idx>,
                       on the Earth surface. This is the normal vector to
                       the plane spanned by the vector Theta and P.
                       Where P is the vector pointing to the point in
                       question (point_list[idx]), and Theta is the
                       tangent vector evenly dividing the angle formed by
                       <P,P1>, and <P,P2>. Where P1, P2 are 2 points on
                       both side of P.
        thetai (tuple): the (x, y, z) Cartesian coordinate of the tangent
                        vector Theta above.
    '''
    pi=point_list[idx]
    pic=spherical2Cart(*pi)
    theta1=computeTheta(pi,point_list[idx-1])
    theta2=computeTheta(pi,point_list[idx+1])
    thetai=(theta1+theta2)/2.

    normi=np.cross(pic,thetai)
    normi=normi/np.linalg.norm(normi)
    thetai=thetai/np.linalg.norm(thetai)

    return normi,thetai

def crossSectionFlux(mask, quslab, qvslab, axis_rdp):
    '''Compute setion-wise orientation differences and cross-section fluxes
    in an AR

    Args:
        mask (ndarray): CROPPED (see cropMask and applyCropIdx) 2D binary map
                        showing the location of an AR with 1s.
        quslab (cdms.TransientVariable): CROPPED (n * m) 2D array of u-flux,
                                       in kg/m/s.
        qvslab (cdms.TransientVariable): CROPPED (n * m) 2D array of v-flux,
                                       in kg/m/s.
        axis_rdp (ndarray): Nx2 array storing the (lat, lon) coordinates of
                           rdp-simplified AR axis.

    Returns:
        angles (TransientVariable): 2D map with the same shape as <mask>,
                                    showing section-wise orientation
                                    differences between horizontal flux (as
                                    in <quslab>, <qvslab>) and the AR axis of
                                    that section. In degrees. Regions outside
                                    of AR (0s in <mask>) are masked.
        anglesmean (float): area-weighted averaged of <angles> inside <mask>.
        crossflux (TransientVariable): 2D map with the same shape as <mask>,
                                       the section-wise cross-section fluxes
                                       in the AR, defined as the projection
                                       of fluxes onto the AR axis, i.e. flux
                                       multiplied by the cos of <angles>.
        seg_thetas (list): list of (x, y, z) Cartesian coordinates of the
                           tangent vectors along section boundaries.
    '''
    # get coordinates
    axislist=quslab.getAxisList()
    lats=quslab.getLatitude()[:]
    lons=quslab.getLongitude()[:]
    lonss,latss=np.meshgrid(lons,lats)

    # convert to cartesian coordinates
    carts=spherical2Cart(latss,lonss)
    vs=wind2Cart(quslab,qvslab,latss,lonss)
    vsnorm=np.linalg.norm(vs,axis=0)
    vsnorm=vs/vsnorm[None,:,:]

    # loop through segments to get orientation differences
    nsegs=len(axis_rdp)-1
    seg_thetas=[]
    angles=np.zeros(mask.shape)

    for ii in range(nsegs):

        pic=spherical2Cart(*axis_rdp[ii])
        pi1c=spherical2Cart(*axis_rdp[ii+1])

        if ii==0:
            setL=1.
            thetai=0 # dummy place holder
        else:
            # get evenly dividing angle theta and normal vector to theta
            normi,thetai=getNormalVectors(axis_rdp, ii)
            # dot products between normal vector and grid coordinates
            dotsi=(normi[:,None,None]*carts).sum(axis=0)
            setL=np.where(dotsi*(normi.dot(pi1c))>=0,1,0)

        if ii==nsegs-1:
            setR=1.
            thetai=0 # dummy place holder
        else:
            normi1,thetai=getNormalVectors(axis_rdp, ii+1)
            dotsi1=(normi1[:,None,None]*carts).sum(axis=0)
            setR=np.where(dotsi1*(normi1.dot(pic))>0,1,0)

        segii=setL*setR*mask
        seg_thetas.append(thetai)

        # sel the correct region if shape too curvy
        segregii=measure.label(segii)
        if segregii.max()>1:
            piidx=[funcs.findIndex(axis_rdp[ii][0],lats),\
                    funcs.findIndex(axis_rdp[ii][1],lons)]
            for jj in range(segregii.max()):
                segjj=np.where(segregii==jj+1,1,0)
                if segjj[piidx[0],piidx[1]]==1:
                    segii=segjj
                    break

        # mean orientation of AR axis segment
        meanori=np.cross(pic,pi1c)
        meanori=meanori/np.linalg.norm(meanori)

        # orientation of flux vectors
        '''
        fluxori=np.cross(carts,vs,axisa=0,axisb=0) # output: ny,nx,3
        norms=np.linalg.norm(fluxori,axis=2)
        fluxori=fluxori/(1e-6+norms[:,:,None])

        # get angles as arccos of the dot product of meanori and fluxori
        anglesii=(meanori[None,None,:]*fluxori).sum(axis=-1)
        anglesii=anglesii*segii
        '''

        # get sin(angle of flux vector and axis segment plane)
        # sign of sin() is: >0: flux vector aligns with meanori, and it
        # is pointing towards the cold side (according to thermal wind).
        # sin() <0: flux vector aligns against meanori, and it is pointing
        # towards the warm side.
        anglesii=(meanori[:,None,None]*vsnorm).sum(axis=0)
        #anglesii=np.sqrt(1-cos_alphaii**2)*np.where(cos_alphaii<0,-1,1)
        anglesii=anglesii*segii

        angles=angles+anglesii

    # compute cross section flux
    angles=np.array(angles)
    cos_angles=np.sqrt(1-angles**2)
    crossflux_c=cos_angles[None,:,:]*vs
    # convert to local tangent winds
    crossflux_u,crossflux_v=cart2Wind(crossflux_c,latss,lonss)
    crossflux=np.sqrt(crossflux_u**2+crossflux_v**2)
    crossflux=MV.masked_where(mask==0,crossflux)
    crossflux.setAxisList(axislist)

    # convert cos to angle in degrees
    angles=np.arcsin(angles)/np.pi*180
    angles=MV.masked_where(mask==0,angles)
    angles.setAxisList(axislist)

    anglesmean=cdutil.averager(angles,axis='xy',weights=['generate','generate'])

    return angles,anglesmean,crossflux,seg_thetas


# -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2021-01-17 19:53
import os
import tempfile
from typing import Iterable

import torch

from elit.utils.io_util import merge_files
from elit.utils.time_util import CountdownTimer

class FileCache(object):
    def __init__(self, filename=None, delete=True) -> None:
        self.delete = delete
        if not filename:
            filename = tempfile.NamedTemporaryFile(prefix='elit-cache-', suffix='.pkl', delete=delete).name
        self._filename = filename

    def close(self):
        if self.delete:
            if os.path.isfile(self._filename):
                os.remove(self._filename)

    def __del__(self):
        self.close()

class RandomAccessFileCache(FileCache):
    def __init__(self, filename=None) -> None:
        super().__init__(filename)
        self.fp = open(filename, 'wb+')
        self.offsets = dict()

    def __setitem__(self, key, value):
        # Always write to the end of file
        self.fp.seek(0, 2)
        start = self.fp.tell()
        torch.save(value, self.fp, _use_new_zipfile_serialization=False)
        self.offsets[key] = start

    def __getitem__(self, key):
        offset = self.offsets.get(key, None)
        assert offset is not None, f'{key} does not exist in the cache'
        self.fp.seek(offset)
        return torch.load(self.fp)

    def __contains__(self, key):
        return key in self.offsets

    def close(self):
        if self.fp:
            self.fp.close()
            self.fp = None
        super().close()

    def __len__(self):
        return len(self.offsets)

class SequentialFileCache(FileCache):
    def __init__(self, iterator: Iterable = None, size=None, filename=None, delete=True, device=None) -> None:
        super().__init__(filename, delete)
        if isinstance(device, int):
            device = torch.device(f'cuda:{device}' if device >= 0 else torch.device('cpu'))
        self.device = device
        self.size = size
        # If the cache is already there then load the size and return
        if not delete and filename and os.path.isfile(filename):
            if not size:
                with open(self._filename, "rb") as f:
                    self.size = torch.load(f)
            return
        os.makedirs(os.path.dirname(self._filename), exist_ok=True)
        # Otherwise generate the cache
        timer = CountdownTimer(size) if size else None
        with open(self._filename, "wb") as f:
            if size:
                torch.save(size, f, _use_new_zipfile_serialization=False)
            for i, batch in enumerate(iterator):
                torch.save(batch, f, _use_new_zipfile_serialization=False)
                if timer:
                    timer.log(f'Caching {self._filename} [blink][yellow]...[/yellow][/blink]', erase=True)
                self.size = i + 1
        if not size:
            _content = self._filename + '.content'
            os.rename(self._filename, _content)
            _index = self._filename + '.index'
            with open(_index, "wb") as f:
                torch.save(self.size, f, _use_new_zipfile_serialization=False)
            merge_files([_index, _content], self._filename)
            os.remove(_content)
            os.remove(_index)

    def __iter__(self):
        with open(self._filename, "rb") as f:
            self.size = torch.load(f)
            for i in range(self.size):
                batch = torch.load(f, map_location=self.device)
                yield batch

    def __len__(self):
        return self.size


import torch
from src.test_statistics import *
from src.utils import get_W_matrix,KMM_weights_for_W_matrix
from sklearn.preprocessing import KBinsDiscretizer
from sklearn.base import clone
import numpy as np
import random
import math

def get_binned_weights(weights, n_bins):
    binner = KBinsDiscretizer(n_bins=n_bins,encode = "ordinal")
    binner.fit(weights.reshape(-1,1))
    binned_weights = binner.transform(weights.reshape(-1,1))
    return binned_weights

def binned_permutation(binned_weights):
    perm = np.array((range(len(binned_weights))))
    for i in range(int(binned_weights.max().item())):
        bin_i = ((binned_weights == i).squeeze(1))
        perm[ bin_i ] = np.random.choice(perm[ bin_i ], len(perm[ bin_i ]),replace = False)
    return perm

def invert_permutation(permutation):
    inv = np.empty_like(permutation)
    inv[permutation] = np.arange(len(inv), dtype=inv.dtype)
    return inv

def weights_tol(weights,weights_minmax):
    weights[weights < weights_minmax] = weights_minmax
    weights[weights > 1-weights_minmax] = 1-weights_minmax
    return weights

def kernel_permutation_test(data_train,data_test,X_ker,Y_ker,weights_model,test_stat="DATE",n_bins=10,n_permutations=200,num_train_permutations=1,reg=[1,1],permute_weights=False,func="cme", KMM_weights = False,weights_minmax=10**(-5)):

    weights_model_train = clone(weights_model)
    weights_model_test = clone(weights_model)

    weights_model_train.fit(data_train.X,data_train.T)
    weights_model_test.fit(data_test.X,data_test.T)

    weights_train = weights_tol(torch.tensor(weights_model_test.predict_proba(data_train.X)[:,1]).float(),weights_minmax)
    weights_test = weights_tol(torch.tensor(weights_model_train.predict_proba(data_test.X)[:,1]).float(),weights_minmax)
    data_full = data_test.join(data_train)
    weights = torch.concat([weights_train,weights_test])

    shuffle = torch.randperm(len(data_full.T))
    weights = weights[shuffle]
    data_full = data_full.return_shuffled_data(shuffle)

    data_train,data_test = data_full.split()
    weights_train_perm,weights_test_perm = weights[:math.floor(len(weights)/2)],weights[math.floor(len(weights)/2):]

    weights_model_train.fit(data_train.X,data_train.T)

    weights_test = weights_tol(torch.tensor(weights_model_train.predict_proba(data_test.X)[:,1]).float(),weights_minmax)

    if test_stat == "DATE":

        W0_weights = 1/KMM_weights_for_W_matrix(X_ker,data_train.X0,data_train.X,KMM_weights)
        W1_weights = 1/KMM_weights_for_W_matrix(X_ker,data_train.X1,data_train.X,KMM_weights)

        W0 = get_W_matrix(X_ker(data_train.X0).evaluate(),reg[0],func,weights=W0_weights)
        W1 = get_W_matrix(X_ker(data_train.X1).evaluate(),reg[1],func,weights=W1_weights)

        base_stat = DATE_test_stat(data_train,data_test,X_ker,Y_ker,weights_test,W0,W1)

    elif test_stat == "DETT":

        W1_weights = 1/KMM_weights_for_W_matrix(X_ker,data_train.X1,data_train.X0,KMM_weights)
        W1 = get_W_matrix(X_ker(data_train.X1).evaluate(),reg[1],func,weights=W1_weights)
        base_stat = DETT_test_stat(data_train,data_test,X_ker,Y_ker,weights_test,W1)

    elif test_stat == "Diff":
        base_stat = diff_test_stat(data_train,data_test,X_ker,Y_ker)

    binned_weights_train = get_binned_weights(weights_train_perm, n_bins)
    binned_weights_test = get_binned_weights(weights_test_perm, n_bins)

    if test_stat == "DATE":
        permuted_model_list = [(data_train,weights_model_train,W0,W1)]

    elif test_stat == "DETT":
        permuted_model_list = [(data_train,weights_model_train,W1)]

    for i in range(num_train_permutations):
        train_permutation = binned_permutation(binned_weights_train)
        permuted_train_data = data_train.return_permuted_data(train_permutation)

        if test_stat == "DATE":

            W0_weights_perm = 1/KMM_weights_for_W_matrix(X_ker,permuted_train_data.X0,permuted_train_data.X,KMM_weights)
            W1_weights_perm = 1/KMM_weights_for_W_matrix(X_ker,permuted_train_data.X1,permuted_train_data.X,KMM_weights)

            W0_permuted = get_W_matrix(X_ker(permuted_train_data.X0).evaluate(),reg[0],func,W0_weights_perm)
            W1_permuted = get_W_matrix(X_ker(permuted_train_data.X1).evaluate(),reg[1],func,W1_weights_perm)

            permuted_weights_model = clone(weights_model)
            permuted_weights_model.fit(permuted_train_data.X,permuted_train_data.T)
            permuted_model_list.append((permuted_train_data,permuted_weights_model,W0_permuted,W1_permuted))

        elif test_stat == "DETT":

            W1_weights_perm = 1/KMM_weights_for_W_matrix(X_ker,permuted_train_data.X1,permuted_train_data.X,KMM_weights)

            W1_permuted = get_W_matrix(X_ker(permuted_train_data.X1).evaluate(),reg[1],func,W1_weights_perm)
            permuted_weights_model = clone(weights_model)
            permuted_weights_model.fit(permuted_train_data.X,permuted_train_data.T)
            permuted_model_list.append((permuted_train_data,permuted_weights_model,W1_permuted))

    permuted_stats = [base_stat]

    for i in range(n_permutations):
        test_permutation = binned_permutation(binned_weights_test)
        permuted_test_data = data_test.return_permuted_data(test_permutation)

        sample_index = random.randrange(len(permuted_model_list)-1)

        permuted_models = permuted_model_list[sample_index]
        if test_stat == "DATE":
            test_stat_weight_permuted = weights_tol(torch.tensor(permuted_models[1].predict_proba(permuted_test_data.X)[:,1]).float(),weights_minmax)
            permuted_stats.append(DATE_test_stat(permuted_models[0],permuted_test_data,X_ker,Y_ker,test_stat_weight_permuted,permuted_models[2],permuted_models[3]))

        elif test_stat == "DETT":
            test_stat_weight_permuted = weights_tol(torch.tensor(permuted_models[1].predict_proba(permuted_test_data.X)[:,1]).float(),weights_minmax)
            permuted_stats.append(DETT_test_stat(permuted_models[0],permuted_test_data,X_ker,Y_ker,test_stat_weight_permuted,permuted_models[2]))

        elif test_stat == "Diff":
            train_permutation = binned_permutation(binned_weights_train)
            permuted_train_data_diff = data_test.return_permuted_data(train_permutation)
            permuted_stats.append(diff_test_stat(permuted_train_data_diff,permuted_test_data,X_ker,Y_ker))

    p_val = np.mean(np.array(permuted_stats) >= base_stat)

    return {"p_val": p_val,"stat": base_stat ,"permuted_stats": permuted_stats}

def goodness_of_fit_test(fit_samples,data_train,data_test,X_ker,Y_ker,weights_model,t=1,test_stat="DATE",reg=1,func="cme", KMM_weights = False):

    weights = torch.tensor(weights_model.predict_proba(data_test.X)[:,1]).float()

    if test_stat == "DATE":

        if t==1:
            W1_weights = 1/KMM_weights_for_W_matrix(X_ker,data_train.X1,data_train.X,KMM_weights)
            W1 = get_W_matrix(X_ker(data_train.X1).evaluate(),reg[1],func,weights=W1_weights)
            fit_stat = DATE_goodness_of_fit(fit_samples,data_train,data_test,X_ker,Y_ker,weights,W1,t=1)
        else:
            W0_weights = 1/KMM_weights_for_W_matrix(X_ker,data_train.X0,data_train.X,KMM_weights)
            W0 = get_W_matrix(X_ker(data_train.X0).evaluate(),reg[0],func,weights=W0_weights)
            fit_stat = DATE_goodness_of_fit(fit_samples,data_train,data_test,X_ker,Y_ker,1-weights,W0,t=0)

    elif test_stat == "DETT":
        if t==1:
            W1_weights = 1/KMM_weights_for_W_matrix(X_ker,data_train.X1,data_train.X,KMM_weights)
            W1 = get_W_matrix(X_ker(data_train.X1).evaluate(),reg[1],func,weights=W1_weights)
            fit_stat = DETT_goodness_of_fit(fit_samples,data_train,data_test,X_ker,Y_ker,weights,W1,t=1)
        else:
            W0_weights = 1/KMM_weights_for_W_matrix(X_ker,data_train.X0,data_train.X,KMM_weights)
            W0 = get_W_matrix(X_ker(data_train.X0).evaluate(),reg[0],func,weights=W0_weights)
            fit_stat = DETT_goodness_of_fit(fit_samples,data_train,data_test,X_ker,Y_ker,1-weights,W0,t=0)

    return fit_stat


'''module for building state machines to search through TokenSets'''
import re
from functools import reduce
from typing import List

import networkx as nx
from networkx.drawing.nx_agraph import to_agraph

import auto_types as T
import random

class Spec:

    def __init__(self, name: str):
        self.name = name
        self.requirements = set()

    def add_req(self, req):
        self.requirements.add(req)

    def match(self, tokens: T.TokenSetFast) -> T.TokenSetFast:
        return reduce(T.TokenSetFast.intersect, [tokens.filter(lambda x: f(x, tokens)) for f in self.requirements]).map(lambda x: x.update_type(self.name))

class Ruleset:

    def __init__(self, name):
        self.graph = nx.MultiDiGraph()
        self.name = name

    def add_instance(self, _id, _type) -> int:
        # _id = len(self.graph.nodes)
        self.graph.add_node(_id, type=_type)
        return _id

    def instance_by_id(self, id: str):
        return self.graph.nodes[id]

    def add_relation(self, tail: int, head: int):
        self.graph.add_edge(tail, head)

    def draw(self, filename='rules.png', label_key='type') -> None:
        G = self.graph.copy()
        labels = {}
        for node in G.nodes:
            labels[node] = {'label': G.nodes[node][label_key]}
        nx.set_node_attributes(G, labels)
        A = to_agraph(G)
        A.layout('dot')
        A.draw(filename)

    def start_types(self) -> List[str]:
        results = []
        for n in self.graph.nodes:
            if self.graph.in_degree(n) == 0:
                results.append(n)
        return results

    def match(self, tokens: T.TokenSetFast, instance_id: int = None) -> T.TokenSetFast:
        '''Returns the first subset of a Tokenset satisifies the ruleset.'''
        if instance_id is None:
            instance_id = self.start_types()[0]
        start_index = tokens.min_index
        first_tokens = tokens.starts_at(start_index)
        target_type = self.instance_by_id(instance_id)['type']

        accepted_tokens = first_tokens.filter(lambda x: x.type == target_type)
        if len(accepted_tokens) == 0:
            return T.TokenSetFast([])

        next_node_ids = list(self.graph.neighbors(instance_id))
        if len(next_node_ids) == 0:
            return tokens.subset(T.Range(start_index, accepted_tokens.max_index))

        # results: List[T.TokenSet] = []
        for node_id in next_node_ids:
            for accepted_tkn in accepted_tokens.content:
                next_range = T.Range(accepted_tkn.match.end, tokens.max_index)
                match = self.match(tokens.subset(next_range), node_id)
                if len(match) > 0:
                    return tokens.subset(T.Range(start_index, match.max_index))

        # if len(results) > 0:
        #     min_result_index = min([x.max_index for x in results])
        #     return tokens.subset(T.Range(start_index, min_result_index))
        return T.TokenSetFast([])

    def find(self, tokens: T.TokenSetFast, count: int = 100, start: str='cw') -> List[T.TokenSetFast]:
        start_types = self.start_types()

        window_size = 300  # in number of characters
        i = 2768
        num_found = 0
        results = set()
        while num_found < count and i < tokens.max_index:
            x = tokens.subset(T.Range(i, i + window_size))
            m = self.match(x, start)
            if len(m) == 0:
                i = tokens.next_at(i + 1).min_index
            else:
                results.add(T.Token(self.name, m.compile_to_str(),
                                    T.Range(m.min_index, m.max_index)))
                num_found += 1
                i = tokens.next_at(m.max_index).min_index
        return T.TokenSetFast(results)


from django.shortcuts import render,redirect
from .forms import UserCreateForm,SignUpForm
from django.contrib.auth.forms import UserCreationForm,AuthenticationForm
from django.contrib.auth import login,logout
# Create your views here.

def signup_view(request):
    #get 요청 시 HTML 응답
    if request.method=='GET':
        form=SignUpForm
        context={'form':form}
        return render(request,'accounts/signup.html',context)
    else:
        #post 요청 시 데이터 확인 후 회원 생성
        form=SignUpForm(request.POST)

        if form.is_valid():
            # #회원가입처리
            # form.cleaned_data['username']
            # form.cleaned_data['email']
            # form.cleaned_data['password2']
            instance=form.save()
            return redirect('index')
        else:
            return redirect('accounts:signup')

def login_view(request):
    #get.post divide
    if request.method=='GET':
        return render(request,'accounts/login.html',{'form':AuthenticationForm()})
    else:
        #데이터 유효성 검사
        form=AuthenticationForm(request,data=request.POST)

        if form.is_valid():
            # 비즈니스 로직 처리
            login(request,form.user_cache)
            #응답
            return redirect('index')
            pass
        else:

            return redirect(request,'accounts/login.html',{'form':form})
    #data velidation
    #take business logic
    #response

def logout_view(request):
    if request.user.is_authenticated:
        logout(request)
    return redirect('index')


""""
Wrapper class for the FPDF.
Enables use of HTML.
@author Chase Fleming
4/23/17
"""

from fpdf import FPDF
from operator import itemgetter

title = "Participant Schedule"

def converter(interval):
    times = {110.0: "10:00 am", 110.5: "10:30 am", 111.0: "11:00 am", 111.5: "11:30 am", 112.0: "12:00 pm",
             112.5: "12:30 pm", 113.0: "1:00 pm", 113.5: "1:30 pm", 114.0: "2:00 pm", 114.5: "2:30 pm",
             115.0: "3:00 pm", 115.5: "3:30 pm", 116.0: "4:00 pm", 116.5: "4:30 pm", 117.0: "5:00 pm",
             117.5: "5:30 pm", 118.0: "6:00 pm", 118.5: "6:30 pm", 119.0: "7:00 pm", 119.5: "7:30 pm",
             120.0: "8:00 pm", 120.5: "8:30 pm", 121.0: "9:00 pm", 121.5: "9:30 pm", 122.0: "10:00 pm",
             210.0: "10:00 am", 210.5: "10:30 am", 211.0: "11:00 am", 211.5: "11:30 am", 212.0: "12:00 pm",
             212.5: "12:30 pm", 213.0: "1:00 pm", 213.5: "1:30 pm", 214.0: "2:00 pm", 214.5: "2:30 pm",
             215.0: "3:00 pm", 215.5: "3:30 pm", 216.0: "4:00 pm", 216.5: "4:30 pm", 217.0: "5:00 pm",
             217.5: "5:30 pm", 218.0: "6:00 pm", 218.5: "6:30 pm", 219.0: "7:00 pm", 219.5: "7:30 pm",
             220.0: "8:00 pm", 220.5: "8:30 pm", 221.0: "9:00 pm", 221.5: "9:30 pm", 222.0: "10:00 pm",
             109.0: "9:00 am", 109.5: "9:30 am", 209: "9:00 am", 209.5: "9:30 am"}
    datestring = ""
    day = str(interval[0][0])[0]
    if day == "1":
        datestring += "Friday"
    else:
        datestring += "Saturday"
    time1 = times[interval[0][0]]
    time2 = times[interval[0][1]]
    time3 = time1 + " to " + time2
    datestring = datestring + " " + time3
    return datestring

class PDF(FPDF):
    def header(self):
        # logos
        self.image("/Users/chasefleming/PycharmProjects/NHFSchedule/docs/images/ACE_logo_Bee.jpg", 10, 10, 50, 50)
        self.image("/Users/chasefleming/PycharmProjects/NHFSchedule/docs/images/generic.jpg", 160, 10, 50, 50)
        self.set_font('Helvetica', '', 28)
        # calculate width of title and position
        w = self.get_string_width(title) + 10
        self.set_x((210 - w) / 2)
        # title
        self.cell(w, 39, title, 0, 1, 'C', 0)

    def footer(self):
        # position at 1.5 cm from bottom
        self.set_y(-15)
        # helvetica 10 font
        self.set_font('Helvetica', '', 10)
        # bottom information
        self.cell(0, 10, "Hilton: Rooms 1-10, Marriott: Rooms 22-30, Hyatt: Rooms 33-42 and Exam Room", 0, 0, 'C')

    def schedule_name(self, name, uid, seed):
        self.set_font('Helvetica', 'B', 22)
        # Title
        w = self.get_string_width(name) + 10
        self.set_x((210 - w) / 2)
        self.set_line_width(1.25)
        self.cell(w, 15, name, 1, 1, 'C', 0)
        # Fill id
        code = "ID: " + seed.upper() + "-" + str(uid).upper()
        w = self.get_string_width(code) + 10
        self.set_x((210 - w) / 2)
        self.set_line_width(1.25)
        self.cell(w, 15, code, 1, 1, 'C', 0)
        # Line break
        self.ln(5)

    def schedule_body(self, schedule):
        # add schedule to pdf
        self.set_margins(10, 10)
        self.set_line_width(.75)
        self.set_font('Helvetica', '', 12)
        for spot in sorted(schedule, key=itemgetter(1)):
            eventtitle = spot[0]
            time = converter(spot[1])
            place = spot[2]
            self.cell(80, 15, eventtitle, align="C", border=1)
            self.cell(80, 15, time, align="C", border=1)
            self.cell(30, 15, place, align="C", border=1)
            self.ln(15)

    def print_schedule(self, name, schedule, uid, seed):
        self.add_page()
        self.schedule_name(name, uid, seed)
        self.schedule_body(schedule)


from textblob import TextBlob
import nltk

#nltk.download("stopwords") ##3 downloadsa file of common stop words
# only do this once

from nltk.corpus import stopwords
from pathlib import Path

import pandas as pd

stops = stopwords.words("english")

blob= TextBlob("Today is a beautiful day") ### we are going to use a list comprehension to generate words that are not stop words

#print([word for word in blob.words if word not in stops])
### it is going to cycle in our blob each word, and if it is not one of the stop words, it will print it out.

#we are about to create a word cloud of top words frequently used in the text of "romeo and juliet"
# we will get a list of tuples

blob= TextBlob(Path('RomeoAndJuliet.txt').read_text())

items = blob.word_counts.items()
#this above will create every word with every count
#print(items)

#### we want to get rid of all the tuples that have item 0 is the actual word not the number, if it is not in the stop list
### then it goes into the new list we are creating. It is iterating through each words and it searches the stop list then doing what
### i just described above.
items_no_stops= [item for item in items if item [0] not in stops]
#print(items_no_stops)

###3 here we want to grab the top 20 words, you don't have to install the library

from operator import itemgetter
### helps us sort this list we made above

sorted_items = sorted (items_no_stops, key=itemgetter(1),reverse=True) ### reverse is getting us descending order, true is case sensitive
### the itemgetter is getting the second object in the list, the numbers pretty much and giving us the words with the highest number
top20= sorted_items[:21] ### this is slicing because we only want the top 20
print(top20)

##3after this we activated the virtual environment nlp_venv/scripts/activate
#### then installed pandas pip install pandas
### then install imageio pip install imageio

df= pd.DataFrame(top20, columns=['word', 'count'])

print(df)

import matplotlib.pyplot as plt
from wordcloud import WordCloud
import imageio

###pip install wordcloud

df.plot.bar(x='word', y= 'count', rot=0, legend= False,
color = ["y","c","m","b", "g", "r"])

plt.gcf().tight_layout()

plt.show

###it has to be a white background, so we use imageio because it allows us to specify

text= Path("RomeoAndJuliet.txt"). read_text()

mask_image = imageio.imread('mask_heart.png')

wordcloud = WordCloud(colormap= 'prism', mask=mask_image, background_color='w')

wordcloud = wordcloud.generate(text)

wordcloud= wordcloud.to_file("RomeoandJulietHeart.png")

plt.imshow(wordcloud)

print("done")


import os
import glob
import cv2 as cv

X_path = glob.glob(os.path.join('DiretorioEscolhido/NomePasta', '*'))

X = []

for f in X_path:
    try:
        cv.imwrite(f,cv.resize(cv.imread(f), (224, 224), interpolation=cv.INTER_AREA))

    except:
        print(f)


import numpy as np
import plotly.offline as pyo
import plotly.graph_objs as go
np.random.seed(56)

# data
x_values = np.linspace(0,1,100)
y_values = np.random.randn(100)

# edited this to make a line chart
trace0 = go.Scatter(x=x_values, y=y_values+5,
                   mode='markers+lines',
                   name='markers')

trace1 = go.Scatter(x=x_values, y=y_values,
                   mode='lines',
                   name='lines')
# data list
data=[trace0, trace1] #needs to be a list in plotly

# Layout is optional to edit titles, axis, etc
layout = go.Layout(title='Line Chart')

fig = go.Figure(data=data, layout=layout)
pyo.plot(fig)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""BMP 室温気圧センサー."""

import logging
from datetime import datetime

import Adafruit_BMP.BMP085 as BMP085

from db import MongoDB

class Bmp(MongoDB):
    """BMP180 IO."""

    def __init__(self):
        """イニシャライザ."""
        super().__init__()

        # センサー
        self.sensor = BMP085.BMP085()

        # MongoDB
        db_name = 'bmp'
        self.db = self.client[db_name]

    def __str__(self):
        return "bmp"

    def get_sensor(self):
        """Get センサー値.

        Returns
        -------
        dict
            各センサー値

        """
        try:
            return {
                'btemp': float(self.sensor.read_temperature()),
                'press': round(float(self.sensor.read_pressure()) / 100, 1),
                'alti': round(float(self.sensor.read_altitude()) / 100, 1),
                'sealev': float(self.sensor.read_sealevel_pressure()),
                'timestamp': datetime.now()
            }

        except Exception as e:
            logging.error('SensorError: {}'.format(e))
            return None


from flask import current_app
from marshmallow import Schema, ValidationError, fields, validates_schema
from marshmallow.validate import Length

class CreateTaskSchema(Schema):
    title = fields.Str(
        required=True,
        validate=Length(min=1),
    )
    description = fields.Str(required=True, validate=Length(max=128))
    user_id = fields.Str(required=True, validate=Length(equal=14))

    @validates_schema
    def validate_user_id(self, data, **kwargs):
        board = current_app.config["DB_CONN"]["factwise"]["board"].find_one({"name": data.get("title", "")})
        if data.get("user_id") not in current_app.config["DB_CONN"]["factwise"]["team"].distinct(
            "users.user_id", {"team_id": board.get("team_id")}
        ):
            raise ValidationError("user_id not belong to the team to whom the board is assigned")
        if data.get("title") not in current_app.config["DB_CONN"]["factwise"]["board"].distinct("name"):
            raise ValidationError("Board not available for the requested title")


import unittest
import itertools
from bisect import bisect
import operator
import sys
if sys .version_info >=(3 ,):
  xrange =range
class LebesgueSet (object ):
  _inf =float ('infinity')# can be tested with math.isinf()
  _minf =-_inf
  UNION ,INTER ,XOR =range (3 )
  def __init__ (self ,points ,left_infinite =False ,fast =False ):
    """Create a new LebesgueSet object.
    @ points: a sequence of real numbers
    @ left_infinite : a boolean, True if the first interval is
                      semi-infinite ]-infinity, .[
    @ fast : can be set to True if the sequence of points is
             already sorted and has no duplicates.
    """
    if not fast :
      points =sorted (set (points ))
    self .points =list (points )
    self .left_infinite =bool (left_infinite )

  def intervals (self ):
    start =0
    p =self .points
    n =len (p )
    if self .left_infinite :
      if n :
        yield (self ._minf ,p [0 ])
        start =1
      else :
        yield (self ._minf ,self ._inf )
        return
    while start +1 <n :
      yield (p [start ],p [start +1 ])
      start +=2
    if start <n :
      yield (p [start ],self ._inf )

  def __str__ (self ):
    return str (list (self .intervals ()))

  def __nonzero__ (self ):
    return self .left_infinite or self .points

  def __invert__ (self ):
    """Compute the set's complement"""
    return LebesgueSet (self .points ,not self .left_infinite ,fast =True )

  def is_bounded (self ):
    return not (self .left_infinite or (len (self .points )&1 ))

  def lower_bound (self ):
    if self .left_infinite :
      return self ._minf
    elif not self .points :
      return self ._inf
    else :
      return self .points [0 ]

  def upper_bound (self ):
    if self .left_infinite ^(len (self .points )&1 ):
      return self ._inf
    elif not self .points :
      return self ._minf
    else :
      return self .points [-1 ]

  def zoom (self ,center =0.0 ,factor =1.0 ):
    if factor ==0.0 :
      points =()
      left_infinite =False
    else :
      points =(center +factor *(x -center )for x in self .points )
      if factor >0.0 :
        left_infinite =self .left_infinite
      else :
        left_infinite =(self .upper_bound ()==self ._inf )
        points =reversed (points )
    return LebesgueSet (points ,left_infinite ,fast =True )

  def measure (self ):
    """self.measure() -> a finite or infinite float.
    Compute the Lebesgue measure of the set self. If this
    measure is infinite, return float('infinity')."""
    if self .is_bounded ():
      p = self.points
      return sum ( [float(p[i+1]) - p[i] for i in xrange(0,len(p),2)] )
    else :
      return self ._inf

  def status (self ,x_real ):
    """self.status(x) returns -1, 0, 1 depending on x being outside,
    on the boundary, or inside the set self (x is a real number)"""
    i =bisect (self .points ,x_real )
    if (i >0 )and x_real ==self .points [i -1 ]:
      return 0
    return 1 if self .left_infinite ^(i &1 )else -1

  def is_interior (self ,x ):
    """True if x is in the topological interior of the set self."""
    return self .status (x )==1

  def is_exterior (self ,x ):
    """True if x is in the topological interior of the complement."""
    return self .status (x )==-1

  def is_boundary (self ,x ):
    """True if x is one end of one of the intervals"""
    return self .status (x )==0

  def is_adherent (self ,x ):
    """True if x is in one of the closed intervals defining self."""
    return self .status (x )>=0

  def deltas (self ,negated =False ):
    infinite =self .left_infinite ^(1 if negated else 0 )
    assert infinite in (0 ,1 )
    for i ,x in enumerate (self .points ):
      yield (x ,-1 if (infinite ^(i &1 ))else 1 )

  @classmethod
  def operate (cls ,op ,family ):
    """Compute the union, intersection or xor of a family of sets.
    @ op : one of LebesgueSet.UNION or .INTER or .XOR
    @ family : a non empty sequence of LebesgueSet's.
    A real number x belongs to the xor of the family if it belongs
    to an odd number of members of the family.
    """
    family =tuple (family )
    value =sum (u .left_infinite for u in family )
    P =[]
    if op ==cls .XOR :
      left_infinite =bool (value &1 )
      L =sorted (itertools .chain (*(u .points for u in family )))
      for k ,group in itertools .groupby (L ):
        if len (tuple (group ))&1 :
          P .append (k )
    else :
      inter =(op ==cls .INTER )
      if inter :
        value =len (family )-value
      left_infinite =not (value )if inter else bool (value )
      L =sorted (itertools .chain (*(u .deltas (inter )for u in family )))
      for k ,group in itertools .groupby (L ,key =operator .itemgetter (0 )):
        group =tuple (group )
        new_value =value +sum (delta [1 ]for delta in group )
        if not (value and new_value ):
          P .append (k )
        value =new_value
    return LebesgueSet (P ,left_infinite ,fast =True )

  def __and__ (self ,other ):
    """self & other -> intersection of self and other."""
    return self .operate (self .INTER ,(self ,other ))

  def __or__ (self ,other ):
    """self | other, self + other -> union of self and other."""
    return self .operate (self .UNION ,(self ,other ))
  __add__ =__or__

  def __xor__ (self ,other ):
    """self ^ other -> symetric difference of self and other."""
    return self .operate (self .XOR ,(self ,other ))

  def __sub__ (self ,other ):
    """self - other -> set difference."""
    return self &~other

  @classmethod
  def union (cls ,family ):
    """union of a family of Lebesgue sets."""
    return cls .operate (cls .UNION ,family )

  @classmethod
  def inter (cls ,family ):
    """intersection of a family of Lebesgue sets"""
    return cls .operate (cls .INTER ,family )

  @classmethod
  def xor (cls ,family ):
    """xor of a family of Lebesgue sets."""
    return cls .operate (cls .XOR ,family )

class LebesgueTestCase (unittest.TestCase ):
  """Test class for LebesgueSets."""
  def setUp (self ):
    import random
    self .gauss =random .gauss
    self .randint =random .randint
    self .nbr_families =100
    self .nbr_test_points =1000

  def random_point (self ):
    return self .gauss (0 ,10 )

  def random_union (self ):
      points =(self .random_point ()for i in range (20 ))
      left_infinite =self .randint (0 ,1 )
      return LebesgueSet (points ,left_infinite )
  def random_family (self ):
    return [self .random_union ()for i in range (3 )]

  def runTest (self ):
    itt =itertools
    for i in xrange (self .nbr_families ):
      family =self .random_family ()
      points =set (itt .chain (*(u .points for u in family )))
      union_res =LebesgueSet .union (family )
      inter_res =LebesgueSet .inter (family )
      xor_res =LebesgueSet .xor (family )
      for j in xrange (self .nbr_test_points ):
        while True :
          x =self .random_point ()
          if not x in points :
            break
        status_list =list (u .status (x )for u in family )
        us_ ,is_ ,xs_ =(u .status (x )
        for u in (union_res ,inter_res ,xor_res ))
        u_expected =1 if any (s ==1 for s in status_list )else -1
        i_expected =1 if all (s ==1 for s in status_list )else -1
        x_expected =1 if sum (s ==1 for s in status_list )&1 else -1
        self .assertEqual (us_ ,u_expected ,"Union failed")
        self .assertEqual (is_ ,i_expected ,"Intersection failed")
        self .assertEqual (xs_ ,x_expected ,"Xor failed")

if __name__ =="__main__":
  unittest .main ()


import json
import torch
from torch.utils.data import Dataset
from transformers import GPT2LMHeadModel, GPT2Tokenizer, DataCollatorForLanguageModeling, Trainer, TrainingArguments
from data_preprocess import *

# Load the JSON file with prompt-completion pairs
with open("prompt_completion_pairs.json", "r") as json_file:
    prompt_completion_pairs = json.load(json_file)

# Load the GPT-2 model and tokenizer
model_name = "gpt2"
tokenizer = GPT2Tokenizer.from_pretrained(model_name)

# Add a pad token to the tokenizer's vocabulary
pad_token = "<PAD>"
tokenizer.add_special_tokens({"pad_token": pad_token})

# Load the GPT-2 model
model = GPT2LMHeadModel.from_pretrained(model_name)

# Define a custom dataset class
class CustomDataset(torch.utils.data.Dataset):
    def __init__(self, tokenizer, pairs, block_size):
        self.examples = []
        for pair in pairs:
            prompt = pair["prompt"]
            completion = pair["completion"]
            self.examples.append((prompt, completion))
        self.tokenizer = tokenizer
        self.block_size = block_size

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, index):
        prompt, completion = self.examples[index]
        input_text = f"{prompt}\n{completion}"
        input_ids = self.tokenizer.encode(input_text, add_special_tokens=True)
        return torch.tensor(input_ids)

# Prepare data for fine-tuning
train_dataset = CustomDataset(tokenizer=tokenizer, pairs=prompt_completion_pairs, block_size=32)
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)  # mlm=False disables masking

# # Set up training arguments
training_args = TrainingArguments(
    output_dir="./output",
    overwrite_output_dir=True,
    num_train_epochs=5,
    per_device_train_batch_size=4,
    save_steps=500,
    save_total_limit=2,
    logging_dir="./logs",
    logging_steps=100,
    do_train=True,
)

# Initialize Trainer and start fine-tuning
trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=train_dataset,
)

trainer.train()


import cv2
import numpy as np
import os
import uuid
import copy

def rotateImage(image, angle):
    l = len(image.shape)
    image_center = tuple(np.array(image.shape[:2]) / 2)
    rot_mat = cv2.getRotationMatrix2D(image_center, angle, 1.0)
    result = cv2.warpAffine(image, rot_mat, image.shape[:2], flags=cv2.INTER_LINEAR)
    if len(result.shape) < l:
        y, x = result.shape
        result = result.reshape((y, x, 1))
    return result

def zoom(image, zoom_scale):
    size = image.shape
    l = len(size)
    image = cv2.resize(image, None, fx=zoom_scale, fy=zoom_scale)
    if len(image.shape) < l:
        y, x = image.shape
        image = image.reshape((y, x, 1))
    new_size = image.shape

    if len(size) == 3:
        if zoom_scale > 1:
            return image[int((new_size[0] - size[0]) / 2): int((new_size[0] - size[0]) / 2 + size[0]),
                   int((new_size[1] - size[1]) / 2): int((new_size[1] - size[1]) / 2 + size[1]), :]
        elif zoom_scale == 1:
            return image
        else:
            new_image = np.zeros(size).astype('uint8')
            new_image[int((size[0] - new_size[0]) / 2): int((size[0] - new_size[0]) / 2 + new_size[0]),
            int((size[1] - new_size[1]) / 2): int((size[1] - new_size[1]) / 2 + new_size[1]), :] = image
            return new_image

def sample(image, rotation_min, rotation_max, fliplr, flipud, zoom_min, zoom_max):
    angle = np.random.uniform(rotation_min, rotation_max)
    image = rotateImage(image, angle)

    if fliplr:
        if np.random.random() < 0.5:
            image = np.fliplr(image)

    if flipud:
        if np.random.random() < 0.5:
            image = np.flipud(image)

    zoom_scale = np.random.uniform(zoom_min, zoom_max)
    image = zoom(image, zoom_scale)

    return image

def augmentor(npy_path, save_path, batch_number=1, rotation_min=0, rotation_max=0, fliplr=False, flipud=False, zoom_min=1, zoom_max=1, input_c=6):
    c = 0
    npy_files = [os.path.join(npy_path, f) for f in os.listdir(npy_path) if f.endswith('.npy')]
    while c < batch_number:
        for npy_file in npy_files:
            data = np.load(npy_file)
            data = sample(data, rotation_min, rotation_max, fliplr, flipud, zoom_min, zoom_max)
            unid = uuid.uuid4().hex
            image = data[:, :, :input_c]
            masks = data[:, :, input_c:]
            num_objs = masks.shape[2]
            print(masks.shape[2])
            for i in reversed(range(num_objs)):
                mask = masks[:, :, i]
                if mask.max() < 250:
                    masks = np.delete(masks, i, axis=2)
                    continue
                mask = mask >= 250
                pos = np.where(mask)
                xmin = np.min(pos[1])
                xmax = np.max(pos[1])
                ymin = np.min(pos[0])
                ymax = np.max(pos[0])
                if xmin >= xmax:
                    masks = np.delete(masks, i, axis=2)
                    continue
                if ymin >= ymax:
                    masks = np.delete(masks, i, axis=2)
                    continue
            print(masks.shape[2])
            print('\n')
            data = np.append(image, masks, axis=2)
            save_file = npy_file.split('.npy')[0].split('/')[-1] + "_" + unid + ".npy"
            save_file = os.path.join(save_path, save_file)
            np.save(save_file, data)

        c += 1

def balanced_augmentor(image_path, label_path, aug_path, augmentation_batch=1, augmentation_ratio=[],
                       rotation_min=0, rotation_max=0, fliplr=False, flipud=False, zoom_min=1, zoom_max=1):

    image_files = [os.path.join(image_path, f) for f in os.listdir(image_path) if f.endswith('.jpg')]
    while augmentation_batch:
        augmentation_batch -= 1
        for image_file in image_files:
            f_name = image_file.split('/')[-1][:-4]
            mask_file = os.path.join(label_path, f_name + "_nd.npy")
            cls_file = os.path.join(label_path, f_name + "_cls.npy")

            if not os.path.isfile(mask_file):
                continue
            if not os.path.isfile(cls_file):
                continue

            image = cv2.imread(image_file)
            masks = np.load(mask_file)
            clses = np.load(cls_file)  # cls should start with 0, e.g. [0, 1, 2, 3, ...]
            image_mask = np.concatenate((image, masks), axis=2)

            n = augmentation_ratio[int(clses.max())]
            for _ in range(n):
                data = copy.deepcopy(image_mask)
                data = sample(data, rotation_min, rotation_max, fliplr, flipud, zoom_min, zoom_max)
                image = data[:, :, :3]
                masks = data[:, :, 3:]
                cls = copy.deepcopy(clses)

                if masks.max() < 0.1:
                    continue

                print(masks.shape[2])
                num_objs = masks.shape[2]
                for i in reversed(range(num_objs)):
                    mask = masks[:, :, i]
                    if mask.max() < 250:
                        masks = np.delete(masks, i, axis=2)
                        cls = np.delete(cls, i)
                        continue

                    mask = mask >= 250
                    pos = np.where(mask)
                    xmin = np.min(pos[1])
                    xmax = np.max(pos[1])
                    ymin = np.min(pos[0])
                    ymax = np.max(pos[0])

                    if (xmin >= xmax) | (ymin >= ymax):
                        masks = np.delete(masks, i, axis=2)
                        cls = np.delete(cls, i)
                        continue

                if masks.shape[2] == 0:
                    continue

                if masks.max() == 0:
                    continue

                print(masks.shape[2])
                print('\n')
                unid = uuid.uuid4().hex

                new_mask_file = os.path.join(aug_path, f_name + "_" + unid + "_nd.npy")
                new_cls_file = os.path.join(aug_path, f_name + "_" + unid + "_cls.npy")
                new_image_file = os.path.join(aug_path, f_name + "_" + unid + ".jpg")

                np.save(new_mask_file, masks)
                np.save(new_cls_file, cls)
                cv2.imwrite(new_image_file, image)

if __name__ == '__main__':
    config = dict(
        batch_number=6,
        rotation_min=0,#-90,
        rotation_max=0,#90,
        fliplr=True,
        flipud=True,
        zoom_min=1,#0.8,
        zoom_max=1,#1.2,
        input_c=6)

    config_ = dict(
        augmentation_batch=5,
        augmentation_ratio=[1, 3, 15, 50, 100],
        rotation_min=-90,
        rotation_max=90,
        fliplr=True,
        flipud=True,
        zoom_min=0.8,
        zoom_max=1.2)
    #image_path = './datasets/Eureka/images/'
    #label_path = './datasets/Eureka/labels/'
    #aug_path = './datasets/Eureka/aug/'
    #balanced_augmentor(image_path, label_path, aug_path, **config_)

    npy_path = './datasets/hypolith/rgb_masks/'
    aug_path = './datasets/hypolith/aug/'
    augmentor(npy_path, aug_path, **config)


import copy
from collections import defaultdict

from game_board import GameBoard

########################################################################################################################
# PUBLIC INTERFACE
########################################################################################################################
def solve(input_board, max_iterations):
    """ Entry point to the solvers. Just calling this function, by default, should yield "good" results.

    The idea is that it'll call whatever solver happens to be the best at some point, depending on what's implemented.
    """
    # Define a pipeline of solvers to use. Since the intermediate solutions of each solver should be valid, the
    # intermediate results can be fed into one another.
    solvers = [
        _deductive,
        _backtrack
    ]

    intermediate_board = input_board
    for solver in solvers:
        intermediate_board = solver(intermediate_board, max_iterations)
        if intermediate_board.is_solved():
            break

    return intermediate_board

########################################################################################################################
# INTERNALS (here be dragons)
########################################################################################################################
def report_start(solver_name, occupancy):
    print(f"[{solver_name}] starting. Starting board occupancy at {occupancy:.2f}.")

def report_progress(solver_name, iterations, max_iterations):
    if iterations % 100 == 0:
        print(f"[{solver_name}]: {(iterations/max_iterations)*100:.2f}% of max_iterations.", end="\r")

def report_result(solver_name, solved, occupancy, reached_max_iterations):
    print(f"[{solver_name}] finished. Solved? {solved}. Final occupancy: {occupancy:.2f}. Reached maximum iterations? {reached_max_iterations}")

def _backtrack(input_board, max_iterations):
    """ Use the backtrack method to solve a board.

    The backtrack method is usually described/implemented as a recursive call, but I'm allergic to recursive programs.

    Since this was always going to be a lengthy function, I decided to break up the main loop into sub-functions that
    operate on the function's shared state and describe the atomic functions of the algorithm. This allowed me to define
    the main loop in almost 1-to-1 correspondence with the algorithm. This allows me to both understand the algorithm as
    a composition of operations, and perhaps move out functionality in the future.

    I'm not sure about this structure, but I'll leave it as an example. Perhaps a class would've been a better
    representation.

    Reference: https://en.wikipedia.org/wiki/Sudoku_solving_algorithms#Backtracking
    """
    # Duplicate the input board so we don't change it in place:
    board = copy.deepcopy(input_board)

    # Initilise state
    report_start("Backtrack", board.occupancy())
    cells = list(board.board_iterator())
    current_cell_idx = 0
    iterations = 0

    # Basic functions on state
    def get_current_value():
        i, j = cells[current_cell_idx]
        val = board.value(i, j)

        return val

    def initialise():
        i, j = cells[current_cell_idx]
        board.value(i, j, 1)

    def increment():
        i, j = cells[current_cell_idx]
        board.increment(i, j)

    def backtrack():
        nonlocal current_cell_idx
        # Clear cell and backtrack
        i, j = cells[current_cell_idx]
        board.erase(i, j)
        current_cell_idx -= 1

        # Skip over fixed cells
        while current_cell_idx > 0 and board.is_fixed(*cells[current_cell_idx]):
            current_cell_idx -= 1

    def advance():
        nonlocal current_cell_idx
        current_cell_idx += 1

        # Skip over fixed cells
        while current_cell_idx < len(cells) and board.is_fixed(*cells[current_cell_idx]):
            current_cell_idx += 1

    # The main loop is split into three essential clauses:
    # * If the value is empty, it's initialised as 1.
    # * If the value is out of bounds, we backtrack to the previous element we were looking at.
    # * If the board is not valid, we increment the current value.
    # If none of these clauses get triggered, we simply move on to the next cell.
    while not board.is_solved() and iterations < max_iterations and current_cell_idx < 81:
        iterations += 1
        curr_value = get_current_value()
        report_progress("Backtrack", iterations, max_iterations)

        # If the cell is empty, we set it to 1 and move on
        if curr_value == None:
            initialise()
            continue

        # If the cell or board is not a valid value anymore, we erase it and move back
        if curr_value not in board._valid_values:
            backtrack()
            increment()
            continue

        # If the current value is a valid number, we increment it
        if not board.is_valid():
            increment()
            continue

        # And if the board remains valid, we move on
        advance()

    report_result("Backtrack", board.is_solved(), board.occupancy(), iterations == max_iterations)

    return board

def _deductive(input_board, max_iterations):
    """ Implements a deductive method for solving the given board.

    This is based on the common heuristic everyone uses (I assume) where we pencil in the possible values for each cell
    and then unravel the puzzle from there.
    """
    # Duplicate the input board so we don't change it in place:
    board = copy.deepcopy(input_board)
    report_start("Deductive", board.occupancy())

    # We'll keep the pencilled-in numbers here
    pencil_board = defaultdict(set)
    valid_values = board._valid_values

    # Sub-functions
    def generate_pencil_board():
        print("[Deductive] solver generating pencil board.")
        for i, j in board.board_iterator():
            # Only work on empty cells
            if board.value(i, j) == None:
                # Iterate over all valid values
                for val in valid_values:
                    # Try adding, if it works it's a possible value
                    if try_adding_value(i, j, val):
                        key = (i, j)
                        pencil_board[key].add(val)
                        board.erase(i, j)

    def try_adding_value(i, j, value):
        """ Adds a value to the board only if the resulting board is valid.

        Returns whether the value stuck.
        """
        board.value(i, j, value)
        if not board.is_valid():
            board.erase(i, j)
            return False

        return True

    # Start by pencilling in all possibilities
    generate_pencil_board()

    iterations = 0
    while iterations < max_iterations and not board.is_solved() and len(pencil_board) > 0:
        iterations += 1
        report_progress("Deductive", iterations, max_iterations)

        # Iterate over the pencilled board
        for (i, j), values in pencil_board.items():
            # If there's only one possible value for a given cell, we write it in
            if len(values) == 1:
                # TODO: why would this not be valid?
                # TODO: surely we can get stuff from the set without making it into a list
                try_adding_value(i, j, list(values)[0])
                pencil_board[(i, j)] = []
                continue

            # If the current value is the only possible position for this value in this sub-board, we write it in
            sub_board_positions = board.sub_board_positions(*board.sub_board_from_indices(i, j))
            for value in values:
                # Evaluate whether the value is the only possibility for this value in this sub-board
                only_possibility_in_sub_board = True
                for sub_i, sub_j in sub_board_positions:
                    # Only check the pencil board if the key exists, since accessing the defaultdict on a non-existing
                    # key will add it to the dict and, thus, crash us as we're changing the dict mid-iteration.
                    if (sub_i, sub_j) in pencil_board.keys():
                        only_possibility_in_sub_board = not (value in pencil_board[(sub_i, sub_j)])

                # Write the value in and break early since we won't have another value for the current cell
                if only_possibility_in_sub_board:
                    board.value(i, j, value)
                    pencil_board[(i, j)].remove(value)
                    break

    report_result("Deductive", board.is_solved(), board.occupancy(), iterations == max_iterations)

    return board


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('contacts', '0004_auto_20151210_1426'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='region',
            options={'ordering': ['country', 'name']},
        ),
        migrations.AddField(
            model_name='contact',
            name='tax_code',
            field=models.CharField(help_text='Tax Code for Argentina, Brazil, Paraguay, Peru (CUIT/CPF/RUC)', max_length=20, verbose_name='Tax Code', blank=True),
        ),
    ]


import sys
import numpy as np
from fractions import Fraction

if __name__ == "__main__":
    readFile = open(sys.argv[1], "r")
    writeFile = open("out.txt", "w")

    numeroCasos = int(readFile.readline())
    casoMinimo = int(sys.argv[2])
    casoMaximo = int(sys.argv[3])

    for _ in range(casoMinimo-1):
        readFile.readline()
        readFile.readline()

    for caseNumber in range(casoMinimo, casoMaximo+1):
        numeroEntradas = int(readFile.readline())
        valores = readFile.readline().split()
        valores = list(map(int,valores))
        lcmVal = np.lcm.reduce(valores)

        newValores = [i * lcmVal for i in valores]
        sumaCaramelos = sum(newValores)
        numer = Fraction(sumaCaramelos,len(newValores)).numerator
        denom = Fraction(sumaCaramelos,len(newValores)).denominator
        print("Case #%i: %i/%i\n" % (caseNumber, numer, denom))
        writeFile.write("Case #%i: %i/%i\n" % (caseNumber, numer, denom))


import numpy as np
import scipy.spatial
import itertools

from sklearn import metrics

def rand_score( X, labels_true, labels_pred ):
     correct = 0
     total = 0

     arr2 = []
     opop = len(X)
     for i in range(opop):
         arr2.append( i )

     for index_combo in itertools.combinations(arr2, 2):
          index1 = index_combo[0]
          index2 = index_combo[1]

          same_class = (labels_true[index1] == labels_true[index2])
          same_cluster = (labels_pred[index1] == labels_pred[index2])

          if same_class and same_cluster:
              correct += 1
          elif not same_class and not same_cluster:
              correct += 1

          total += 1

     return float(correct) / total

def cluster_centroids(data, clusters, k):
    return np.array([data[ clusters==i ].mean( axis = 0 ) for i in range(k)]) # Compute the mean of data[ i ] where i is set column wise

def kmeans(X, initial_centroids, max_iters):
     eval_mat = []
     clusters = []

     k = len(initial_centroids)

     opop = X.shape[1]-1
     label_true = np.copy(X[:, opop])
     Y =  np.copy(X[:, range(0, opop)])

     try:
         label_true = label_true.astype(np.int)
         label_true = label_true - 1
     except ValueError:
         qq = set(label_true)
         len1 = len(qq)
         for i in range(len1):
            ele1 = qq.pop()
            for j in range( len(X) ):
                if( label_true[j]==ele1 ):
                    label_true[j] = str(i)

     label_true = label_true.astype(np.int)
     Y = Y.astype(np.float)

     centroids = initial_centroids
     for i in range(max_iters):
         # Squared distances between each point and each centroid
        sqdists = scipy.spatial.distance.cdist(Y, centroids, 'sqeuclidean')  # n*k matrix of squared distance, in each row distance of point from each of the centroid is stored

        # Index of the closest centroid to each data point
        clusters = np.argmin(sqdists, axis = 1)                                 # For example np.argmin(a, axis=0) returns the index of the minimum value in each of the columns

        # Finding new centroids
        new_centroids = cluster_centroids(Y, clusters, k)

        # break early if new_centroids = centroids
        if np.array_equal(new_centroids, centroids):
            break

        centroids = new_centroids

     # Normalized Mutual Information(MI)
     nmi = metrics.normalized_mutual_info_score(label_true, clusters)

     # Adjusted Mutual Information(AMI)
     ami = metrics.adjusted_mutual_info_score(label_true, clusters)

     # Rand index(RI)
     ri = rand_score(Y, label_true, clusters)

     # Adjusted Rand index(ARI)
     ari = metrics.adjusted_rand_score(label_true, clusters)

     eval_mat.append( nmi )
     eval_mat.append( ami )
     eval_mat.append( ri )
     eval_mat.append( ari )

     return new_centroids, eval_mat