import numpy as np
from scipy.optimize import minimize
from pySurf.points import *
import mpl_toolkits.mplot3d as m3d
from dataIO.fn_add_subfix import fn_add_subfix

def closest_point_on_line(points,lVersor,lPoint):
    """From a list of points in 3D space as Nx3 array, returns a Nx3 array with the corresponding closest points on the line."""
    #vd=lVersor/((np.array(lVersor)**2).sum())  #normalize vector, not sure it is necessary.
    vd=lVersor/np.sqrt(np.sum(np.array(lVersor)**2))  #normalize vector, not sure it is necessary.
    return lPoint+(points-lPoint).dot(vd)[:,np.newaxis]*(vd)

'''
def closest_point(points,reference):
    """From a list of points in 3D space as Nx3 array, returns the closest one to reference point."""

    dist=np.abs(np.array(points)-reference))

    return points[nanargmin([:,2]
'''

def cylinder_error(odr=(0,0,0,0,0,0),points=None,retall=False,extra=False):  #origin=(0,0,0),direction=(0,1.,0),radius is calculated from best fit
    """Given a set of N points in format Nx3, returns the error on the cylinder defined by origin and direction as a 6(3+3) dim vector.
    If retall is set, additional values are returned :
        radius: best fit radius for the cylinder.
        deltaR[N,3]: deviation from radius for each point.
    extra is equivalent to retall, renamed 2018/02/15 kept for backward compatibility"""

    #ca 400 ms per loop
    origin=odr[0:3]
    direction=odr[3:]
    #vd=direction/((np.array(direction)**2).sum())  #normalize vector, not sure it is necessary.
    x,y,z=np.hsplit(points,3)
    Paxis=closest_point_on_line(points,direction,origin) #points on the cylinder axis closer to points to fit
    deltaR2=((points-Paxis)**2).sum(axis=1)-radius**2  #square difference from expected radius for each point
    fom=np.sqrt(deltaR2.sum()/len(deltaR2))
    retall=retall | extra
    if retall:
        residuals=np.hstack([x,y,deltaR[:,None]])
        return fom,residuals,radius
    else: return fom  #,deltaR,radius

def cylinder_error3(odr:float=(0,0,0,0),points:float=None,radius:float=None,retall:bool=False,extra:bool=False,xy:bool=False)->float:
#from v1
#origin=(0,0,0),direction=(0,1.,0),radius is calculated from best fit

    """
    Calculate rms surface error for points wrt a cylinder defined by axis vector.

    Given a set of N points in format Nx3, returns the error on the cylinder defined by origin and direction of axis. Best fit radius is calculated as best fit if not provided. Otherwise the provided value is used with no fit.
    nan are not allowed and must be filtered before calling the function (this is more efficient if the function is called many times, e.g. in optimization).
    Args:
        points : points in format Nx3
        odr : 4-element axis vector in format (origin_y,origin_z,direction_x,direction_z)
        radius (optional) : If provided, radius is used, otherwise best fit radius for the given axis is calculated.
        retall (optionall): if set, additional values are returned :
            radius: best fit radius for the cylinder.
            deltaR[N,3]: deviation from radius for each point.
    extra (optional) : equivalent to retall, renamed 2018/02/15 kept for backward compatibility.

    Returns:
        rms of radial distance of points from ideal cylindrical surface.
    """

    #2015/09/05 added functions for nan from np
    #2015/08/26 changed sign of deltaR to follow bump up convention (and cone_error3)
    #ca 400 ms per loop
	#get a point on plane y=0 unless xy is set, then plan is x=0.
	#same for origin is vector out of origin plane.
    if xy:
        origin=(0,odr[0],odr[1])
        direction=(1.,odr[2],odr[3])
    else:
        origin=(odr[0],0,odr[1])
        direction=(odr[2],1.,odr[3])

    vd=direction/np.sqrt(1+(np.array(direction)**2).sum())  #normalize vector
    x,y,z=np.hsplit(points,3)
    Paxis=closest_point_on_line(points,vd,origin)
    #origin+(points-origin).dot(vd)[:,np.newaxis]*(vd) #points on the cylinder axis closer to points to fit
    R=np.sqrt(np.nansum(((points-Paxis)**2),axis=1)) #distance of each point from axis
    #R=np.sqrt(((points-Paxis)**2).sum(axis=1)) #distance of each point from axis
    if radius is None:
        radius=np.nanmean(R)  #
        #radius=R.mean()  #
    #plt.plot(R-radius)
    deltaR=(radius-R)  #difference from expected radius for each point
    fom=np.sqrt((np.nansum((deltaR)**2))/len(deltaR))
    #fom=np.sqrt(((deltaR)**2).sum()/len(deltaR))
    residuals=np.hstack([x,y,deltaR[:,None]])
    retall=retall | extra
    if retall: return fom,residuals,radius
    else: return fom  #,deltaR,radius

def cone_error(odr=(0,0,0,0,0,0),points=None,retall=False,extra=False):  #origin=(0,0,0),direction=(0,1.,0),radius is calculated from best fit
    """Given a set of N points in format Nx3, returns the rms surface error on the cone defined by origin (intercept of the axis with x=0) and direction,
    passed as 4-vector odr(origin_y,origin_z,direction_x,direction_z).
    Best fit cone for odr is calculated from linear fit of data.
    If retall is set, additional values are returned :
    coeff: best fit radius for the cone as [m,q] for x' distance from x=0 plan on cone axis R(x')=m x' + q. Half cone angle is atan(m).
    deltaR[N,3]: deviation from radius for each point.
    extra is equivalent to retall, renamed 2018/02/15 kept for backward compatibility
    """
    #ca 400 ms per loop
    origin=odr[0:3]
    direction=odr[3:]
    vd=direction/np.sqrt((1+np.array(direction)**2).sum())  #normalize vector, not sure it is necessary.
    x,y,z=np.hsplit(points,3)
    Paxis=closest_point_on_line(points,direction,origin)
    Paxdist=np.sqrt(((Paxis-origin)**2).sum(axis=1)) #distance of each point from
    R=np.sqrt(((points-Paxis)**2).sum(axis=1)) #distance of each point from axis
    coeff=np.polyfit(Paxdist,R,1) #best fit cone
    deltaR=R-coeff[0]*Paxdist-coeff[1] #residuals
    fom=np.sqrt(((deltaR)**2).sum()/len(deltaR))
    residuals=np.hstack([x,y,deltaR[:,None]])
    retall=retall | extra
    if retall: return fom,residuals,coeff
    else: return fom  #,deltaR,radius

def cone_error3(odr=(0,220.,0,0),points=None,coeff=None,retall=False,extra=False):
#2015/09/05 added functions for nan from np
#2015/01/13 changed sign of deltaR to set bump-positive (it was opposite)
#from v1
#origin=(0,0,0),direction=(0,1.,0),radius is calculated from best fit
    """Given a set of N points in format Nx3, returns the rms surface error on the cone defined by
    its axis (radius and apex are determined by best fit).
    Axis is defined as a 4 elements vector odr=(x,z,cx,cz), not in xz plane.
    origin (intercept of the axis with y=0) and director cosines.
    If coeff is passed as input, the fit for cone surface is not performed and the coeff values are used.
    If retall is set, additional values are returned :
    coeff: best fit radius for the cone as [m,q] for x' distance from x=0 plan on cone axis R(x')=m x' + q. Half cone angle is atan(m).
    deltaR[N,3]: deviation from radius for each point. Bump positive convention (smaller radius is positive).
    extra is equivalent to retall, renamed 2018/02/15 kept for backward compatibility
    """

    import pdb
    #pdb.set_trace()
    points=points[~np.isnan(points[:,2]),:]

    #ca 400 ms per loop
    origin=np.array((odr[0],0,odr[1])) #point of axis at y=0
    direction=np.array((odr[2],np.sqrt(1-odr[2]**2-odr[3]**2),odr[3])) # director cosines, cy is assumed positive
    #direction=direction/np.sqrt((direction**2).sum())
    direction=direction/(np.sqrt(np.nansum(direction**2))) #normalize
    x,y,z=np.hsplit(points,3)
    Paxis=closest_point_on_line(points,direction,origin)
    Paxdist=np.sqrt(np.nansum((Paxis-origin)**2,axis=1))*np.sign(Paxis[:,1]-origin[1])
    #Paxdist=np.sqrt(((Paxis-origin)**2).sum(axis=1))*np.sign(Paxis[:,1]-origin[1])

    #d: on axis coordinate of points
    R=np.sqrt(np.nansum((points-Paxis)**2,axis=1)) #r: distance of each point from axis
    #R=np.sqrt(((points-Paxis)**2).sum(axis=1)) #r: distance of each point from axis
    if coeff is None:
        coeff=np.polyfit(Paxdist,R,1) #best fit cone
    deltaR=coeff[0]*Paxdist+coeff[1]-R #residuals
    fom=np.sqrt(np.nansum((deltaR)**2)/len(deltaR))
    #fom=np.sqrt(((deltaR)**2).sum()/len(deltaR))
    residuals=np.hstack([x,y,deltaR[:,None]])
    retall=retall | extra
    if retall: return fom,residuals,coeff
    else: return fom  #,deltaR,radius

def subtract_cylinder(pp,odr,sampleName=''):
#not used in v1
    """
    odr: 6-vector (origin_y,origin_y,origin_z,direction_x,direction_y,direction_z),
        note that  this is redundant, since only two components are enough for direction
        (magnitude is irrelevant).
    pp: complete set of points Npx3
    """
    fom,deltaR,radius=cylinder_error(odr,pp,retall=False,extra=True)
    xymin=np.nanmin(pp,axis=0)[0:2]
    xymax=np.nanmax(pp,axis=0)[0:2]
    rp=plot_points(np.hstack([pp[:,0:2],deltaR[:,None]*1000]),shape=(281,3001)) #this is done to easily replot and change scale

    #surface map and data
    plt.clf()
    plt.imshow(rp,aspect='equal',interpolation='none',vmin=-5,vmax=10,
        extent=[xymin[1],xymax[1],xymin[0],xymax[0]])
    plt.colorbar()
    plt.title(sampleName+(sampleName if sampleName else '')+'best-fit-cylinder removed.')
    plt.xlabel('Y(mm)')
    plt.ylabel('X(mm)')
    plt.savefig(fn_add_subfix(datafile,'_cylinder','png'))
    save_points(np.hstack([pp[:,0:2],deltaR[:,None]*1000]),fn_add_subfix(datafile,'_cylinder'))

    #cylinder output
    print ('Best fit radius %s'%radius)
    misal=np.arccos(1/np.sqrt(1+((odr[2:]**2).sum())))
    print ('Misalignment of optical axis: %s rad (%s deg)'%(misal,misal*np.pi/180))

    #rms output
    print ('rms entire surface %s'%(np.nanstd(rp)))
    plt.figure()
    plt.plot(np.nanstd(rp))
    plt.plot(np.nanstd(rp,axis=0))
    plt.plot(np.nanstd(rp,axis=1))

    #mask outliers on a copy
    rp2=rp[:]
    np.where(np.isnan(rp2))

def subtract_cone(pp,odr,sampleName='',outfile=None,vmin=None,vmax=None):
#not used in v1
    """
    odr: 6-vector (origin_y,origin_y,origin_z,direction_x,direction_y,direction_z),
        note that  this is redundant, since only two components are enough for direction
        (magnitude is irrelevant).
    pp: complete set of points Npx3
    """
    fom,deltaR,coeff=cone_error(odr,pp,retall=True)
    xymin=np.nanmin(pp,axis=0)[0:2]
    xymax=np.nanmax(pp,axis=0)[0:2]
    rp=plot_points(np.hstack([pp[:,0:2],deltaR[:,None]*1000]),shape=(281,3001)) #this is done to easily replot and change scale

    #surface map and data
    plt.clf()
    plt.imshow(rp,aspect='equal',interpolation='none',vmin=vmin,vmax=vmax,
        extent=[xymin[1],xymax[1],xymin[0],xymax[0]])
    plt.colorbar()
    plt.title(((sampleName+' - ') if sampleName else (''))+'best-fit-cone removed.')
    plt.xlabel('Y(mm)')
    plt.ylabel('X(mm)')
    if outfile:
        plt.savefig(fn_add_subfix(outfile,'_cone','png'))
        save_points(np.hstack([pp[:,0:2],deltaR[:,None]*1000]),fn_add_subfix(outfile,'_cone'))

    #cone output
    m=coeff[0]
    print ('Cone angle:%s+/-%s rad(%s+/-%s deg) '%(np.arctan(m),0,np.arctan(m)*180/np.pi,0))
    print ('Axis intercept at x=0: %smm '%(coeff[1]))
    misal=(np.arccos(1/np.sqrt(1+((odr[2:]**2).sum()))))
    print ('Misalignment of optical axis: %s rad (%s deg)'%(misal,misal*180./pi))

    #rms output
    print ('rms entire surface %s'%(np.nanstd))
    plt.figure()
    plt.plot(np.nanstd(rp))
    plt.plot(np.nanstd(rp,axis=0))
    plt.plot(np.nanstd(rp,axis=1))

    #mask outliers on a copy
    rp2=rp[:]
    np.where(np.isnan(rp2))
    return fom,deltaR,coeff

"""
def fit_cylinder(points,guessValue=None,callback=None):
#not used in v1
    if guessValue!=None:
        odr=guessValue
    else:
        odr=(0,220.,0,0)
    result=minimize(cylinder_error,x0=(odr,),args=(points,),options={'maxiter':1000},callback=callback,method='Nelder-Mead')
    d=result.x[[2,3]]
    angle=np.arccos()

def fit_cone(points,guessValue=None):
#not used in v1
    if guessValue!=None:
        odr=guessValue
    else:
        odr=(0,220.,0,0)
    result=minimize(cone_error,x0=(odr,),args=(points,),options={'maxiter':1000},callback=p,method='Nelder-Mead')
    d=result.x[[2,3]]
    angle=np.arccos()
"""

def fit_cone(pts,odr2,zscale=1.,keepnan=False,**kwargs):
    """   fit pts and return residuals. Info are printed. odr2 is starting guess.

    fom,deltaR,pars=fit_cone(pts,odr2,fom_func)
    zscale is the factor to multiply data to obtain same units as x and y,
    e.g. 1000. for x and y in mm and z in um. Output is in same unit as input.
    pts and deltaR enter and exit with z in microns
    parameters for optimization can be passed as keyword arguments, e.g.:

    #evaluate fom without optimization fit_cone/fit_cylinder
    deltaR,o = fom_function(pts,odr,zscale=1000.,options={'radius':radius})

    # fit from starting guess value odr0
    deltaR,odr=fit_cylinder(c,odr0,zscale=1000.,options={'maxiter':500},
        method='Nelder-Mead')

    typically:
    odr2=(span(xg).sum()/2.,220.,0,0.) #use nominal value for guess direction.

    See developer notes in code.
    """

    fom_func=cone_error3
    pts=pts/[1,1,zscale] #data are um, convert to mm before fit

    #filter for nans, note that this is done here rather than in fom_func so it
    #is performed only once.
    mask=~np.isnan(pts[:,2])

    #find odr from optimization if maxiter is set otherwise use
    # input parameters `odr`. Needs an absurde dictionary with key `options`
    #  use `fit_cone(...,options={'maxiter':500})` Will run optimization
    #  only if this is set to a number, otherwise will just evaluate
    #  the fom. It's an ugly workaround mishandling **kwargs.
    # done for not knowing it better.

    mi=None  #mi=maxiter if called with fit_cone(..,maxiter=int)
    if 'options' in kwargs:
        mi=kwargs['options'].pop('maxiter',None)
    if mi:
        result=minimize(fom_func,x0=(odr2),
                args=(pts[mask,:],),
                **kwargs)   #options={'maxiter':500},
                            #method='Nelder-Mead')#,callback=p)
        odr=result.x
    else:
        print('Dictionary `options` ')
        #if set to 0 r unset do not perform optimization
        odr=odr2

    s=span(pts,axis=0)
    fom,deltaR,coeff=fom_func(odr,pts[mask,:],retall=True)
    deltaR[:,2]=deltaR[:,2]*zscale
    if keepnan:
        pts[mask,2]=deltaR[:,2]
        deltaR=pts

    origin=(odr[0],0,odr[1]) #point of axis at y=0
    direction=np.array((odr[2],np.sqrt(1-odr[2]**2-odr[3]**2),odr[3])) # director cosines, cy is assumed positive
    direction=direction/np.sqrt((direction**2).sum())

    print ('-----------------------------------')
    print ('Results of fit with function: "%s" on region:'%fom_func)
    print ('X: [%6.3f,%6.3f]'%tuple(s[0]))
    print ('Y: [%6.3f,%6.3f]'%tuple(s[1]))
    print ('data range: [%6.3f,%6.3f]'%tuple(s[2]))
    print ('---')
    if mi:
        print (result )

    print ('-----------------------------------')
    print ('Axis direction, components (director cosines): ')
    print (direction)
    print ('Angle of cone axis with x,y,z axis (deg):')
    print (np.arccos(np.abs(direction))*180/np.pi)
    print ('Axis intercept with y=0: ')
    print (origin)
    print ('Cone parameters (angle(deg), R@y=0) [A,B for R=A]:')
    print (np.arctan(coeff[0])*180/np.pi,coeff[1])
    print ('---')
    print ('rms residuals=%s [units of Z]'%(fom*zscale))
    #coordinates of cone apex

    return deltaR,odr

def fit_cylinder(pts,odr2,zscale=1.,keepnan=False,align=False,**kwargs):
    """   fit pts and return residuals. Info are printed.
    odr2: starting guess for fom func parameters.
        for cylinder_error3, it is (origin_y,origin_z,direction_x,direction_z),
        meaning for axis nearly parallel to y: odr2=(x_center,R,0.,0.)
        fom,deltaR,pars=fit_cylinder(pts,odr2,fom_func)
    zscale: factor to divide z data to obtain same units as x and y,
        e.g. 1000. for x and y in mm and z in um. Output is in same unit as input.
    parameters for optimization with scipy.optimize.minimize can be passed as keyword arguments,
        e.g.: deltaR,odr=fit_cylinder(c,odr2,options={'maxiter':500},method='Nelder-Mead')
    if keepnan is set True nans are reinserted after the fit keeping same size for array,
        otherwise nans are removed.
    if align is set True, rotate data to align axis and return residuals on rotated grid (can be resampled on data or viceversa
    with resample_points)-- NOT WORKING
    """
    fom_func=cylinder_error3
    pts=pts/[1,1,zscale] #data are um, convert to mm before fit

    #filter for nans, note that this is done here rather than in fom_func so it
    #is performed only once.
    mask=~np.isnan(pts[:,2])

    if not isinstance(odr2,np.ndarray): odr2=np.array(odr2)

    #find odr from optimization if maxiter is set otherwise use
    mi=None
    radius=None
    if 'options' in kwargs:
        mi=kwargs['options'].pop('maxiter',None)
        radius=kwargs['options'].pop('radius',None)

    if mi is not None:
        if mi != 0:
            result=minimize(fom_func,x0=(odr2),
                    args=(pts[mask,:]),
                    **kwargs)   #options={'maxiter':500},
                                #method='Nelder-Mead')#,callback=p)
            odr=result.x
        else:
            odr=odr2
    else:
        #if set to 0 or unset do not perform optimization
        odr=odr2

    s=span(pts,axis=0)
    fom,deltaR,coeff=fom_func(odr,pts[mask,:],radius=radius,retall=True)  #calculate for best fit parameters
    deltaR[:,2]=deltaR[:,2]*zscale    #output is in same units as input
    if keepnan:
        pts[mask,2]=deltaR[:,2]
        deltaR=pts

    #fom_func, s, zscale, odr, fom, coeff

    print ('-----------------------------------')
    print ('Results of fit with function: "%s" on region:'%fom_func)
    print ('X: [%6.3f,%6.3f]'%tuple(s[0]))
    print ('Y: [%6.3f,%6.3f]'%tuple(s[1]))
    print ('data range: [%6.3f,%6.3f]'%tuple(s[2]))
    print ("---")
    if mi:
        print (result )
    print ('(Angle of cyl axis with y axis (deg):')
    vy=np.sqrt(1-(odr[-2:]**2).sum())
    print (np.arccos(vy)*180/np.pi)
    print ('Rotation about z axis from y axis(deg):')
    az=np.arctan2(odr[2],vy)
    print (az*180/np.pi)
    print ('Radius:')
    print (coeff)
    print ('Axis Parameters:')
    print (odr)
    print ('---')
    print ('rms residuals=%s um'%(fom*zscale))
    if align:
        c=span(deltaR,axis=0)[:2].sum(axis=1)/2 #center of all points
        deltaR=rotate_points(deltaR,az,center=c)
    return deltaR,odr

if __name__=='__main__':
    fom_func=cone_error    #this is the function giving the FOM to be minimized
    def p(x): print (x) #,cylinder_error(x,points)
    outSubfix='_cone' #the name of output file is the datafile with this subfix added
    datafile=r'test\test_fitCylinder\OP2S04OP2S04b\04_OP2S04_xyscan_Height_transformed.dat'
    #datafile='OP2S03c/22_OP2S03_yxsurf_Height_transformed.dat'
    #datafile='OP2S04/04_OP2S04_xyscan_Height_transformed.dat'
    #datafile='OP2S05/05_OP2S05_xysurf_Height_transformed.dat'

    #create points to be fit from a subset of points.
    pts=get_points(datafile,delimiter=' ')
    #pts=rotate_points(pts,-np.pi/2)
    pts=crop_points(pts,(-28,33),(-75,65))
    pts[:,2]=pts[:,2]/1000.
    c=crop_points(pts,(-28,33),(-50,50))    #[0:-1:1000,:]
    odr2=(33,220.,0,0,220)
    result=minimize(fom_func,x0=(odr2[0:-1],),args=(c,),options={'maxiter':1000},callback=p,method='Nelder-Mead')
    print ('-----------------------------------')
    print ('Results of fit on subset of points:')
    print (result)

    #create output results applying the value from fit to all points
    odr=result.x
    fom,deltaR,coeff=fom_func(odr,pts,retall=True)
    origin=(odr[0],0,odr[1])
    direction=(odr[2],1.,odr[3])
    deltaR[:,2]=deltaR[:,2]*1000
    print ('-----------------------------------')
    print ('Results of fit applied to complete set of points:')
    print ('F.O.M.=%s'%(fom))

    plot_points(deltaR,vmin=-5,vmax=10,scatter=1,aspect='equal')
    save_points(deltaR,filename=fn_add_subfix(datafile,outSubfix))
    plt.savefig(fn_add_subfix(datafile,outSubfix,'.png'))


import os
import shutil
import multiprocessing as mp

inp_dir = 'kitti/sequences'
out_dir = 'video_sequence'

def process(seq_id):
    split = 'val' if seq_id == 8 else 'train'
    seq_dir = os.path.join(inp_dir, '{:02d}'.format(seq_id), 'image_2')
    filenames = os.scandir(seq_dir)
    filenames = [name.name for name in filenames]
    for name in filenames:
        inp_path = os.path.join(seq_dir, name)
        new_name = '{:06d}_{}_leftImg8bit.png'.format(seq_id, name[:6])
        out_path = os.path.join(out_dir, split, new_name)
        shutil.copyfile(inp_path, out_path)

seq_ids = range(11)
with mp.Pool(processes=11) as p:
    p.map(process, seq_ids)


from django.core.paginator import Paginator
from django.shortcuts import render

# Create your views here.
from news.Counter import Counter
from news.models import Submission, SubmissionType

def gen_paginator(request, query, number=30):
    subm_paginator = Paginator(query, number)
    page_num = request.GET.get('pages')

    if page_num == None:
        pages = subm_paginator.page(1)
    else:
        pages = subm_paginator.page(page_num)

    page_index = Counter()
    page_index.count = pages.start_index()

    return pages

def news(request):
    pages = gen_paginator(request, Submission.objects.order_by('-points'), 30)
    page_index = Counter()
    page_index.count = pages.start_index()
    return render(request, "news.html", {'pages': pages, 'index': page_index})

def newest(request):
    pages = gen_paginator(request, Submission.objects.order_by('-created_at'), 30)
    page_index = Counter()
    page_index.count = pages.start_index()
    return render(request, "newsest.html", {"pages": pages, 'index': page_index})

def ask(request):
    type = SubmissionType.objects.get(name="ask")
    pages = gen_paginator(request, Submission.objects.filter(type=type).order_by('-points'), 30)
    page_index = Counter()
    page_index.count = pages.start_index()
    return render(request, "ask.html", {"pages": pages, 'index': page_index})


#sklearn 필기숫자 데이터 셋 MLP
from sklearn import datasets
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
import numpy as np

digit = datasets.load_digits()
x_train, x_test, y_train, y_test = train_test_split(digit.data, digit.target, train_size= 0.6)

#mlp 모델 ,옵션
mlp = MLPClassifier(hidden_layer_sizes=(100), #히든 레이어 은닉층 노드 갯수
                    learning_rate_init=0.001, #첫 러닝넷 설정 따로 설정 안하면 이걸로 계속 유지
                    batch_size=32, #전체 훈련 데이터 미니배치 사이즈
                    solver='sgd', #문제를 풀때 미니배치 sgd를 사용한다
                    verbose=True) #훈련을 시킬때 학습과정을 보여준다.
mlp.fit(x_train, y_train)

res = mlp.predict(x_test)

conf = np.zeros((10,10))
for i in range(len(res)):
    conf[res[i]][y_test[i]] += 1
print(conf)

correct = 0
for i in range(10):
    correct += conf[i][i]
accuracy = correct / len(res)
print("테스트 집합에 대한 정확률: ", accuracy*100, "%")


import requests
from datetime import datetime
import urllib.request
import urllib
from lxml import etree
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from time import sleep
from selenium.webdriver.common.by import By
from selenium.common.exceptions import WebDriverException
from threading import Thread
import threading
import socket
import csv
import json
import time
import os
from tqdm import tqdm
from requests.adapters import HTTPAdapter
'''
情报源
情报源注册时长
情报源关注着数目
文章标题
文章作者
文章发布时间
爬取时间
文章正文
阅读数
转发数
链接
'''

def connectchrome():
    options = Options()
    options.add_argument('--no-sandbox')  # 禁用沙箱【不加在liunx下会报错】
    # options.add_argument('headless')  # 设置option
    options.add_argument('log-level=3')
    options.add_argument("--incognito")
    options.add_argument("--no-sandbox");
    options.add_argument("--disable-dev-shm-usage");
    options.add_experimental_option('useAutomationExtension', False)
    options.add_experimental_option('excludeSwitches', ['enable-automation'])
    driver = webdriver.Chrome(options=options)
    driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
        "source": """
            Object.defineProperty(navigator, 'webdriver', {
              get: () => undefined
            })
          """
    })
    driver.maximize_window()
    sleep(2)
    return driver

class ZGJSProcessor:
    # website:http://www.js7tv.cn/
    def __init__(self):
        self.url = 'https://eng.mil.ru/en/news_page/country.htm'

        now_time = int(time.time())
        # 为了方便调试把时间固定
        #self.day_time = time.mktime(time.strptime("2022-03-17", "%Y-%m-%d"))
        self.day_time = time.mktime(time.strptime(time.strftime("%Y-%m-%d", time.localtime(now_time)), "%Y-%m-%d"))
        self.news_url_list = []
        self.head = {'User-Agent': 'Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 101.0.4951.67 Safari / 537.36'}
        # Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 101.0.4951.67 Safari / 537.36
        self.path = 'D:\data\spider\mil'
        basename = 'mil' + time.strftime("%Y%m%d", time.localtime(now_time))
        self.path = os.path.join(self.path, basename)
        self.image_name = 0
        self.video_name = 0
        self.file_download_timeout = [20, 50, 100]

    def get_driver(self):
        driver = connectchrome()
        print("打开浏览器")
        return driver

    # 获取网页url
    def get_news_url_list(self, driver, url):
        try:
            driver.get(url)
        except WebDriverException:
            print("网页打开错误，重新尝试")
            # sleep(20)
            # self.get_news_url_list(driver,url)
        flag = 1
        #try:
        url_list = driver.find_elements(By.XPATH, "//div[@class='newsitem']")
        for element in url_list:
            # print("element.is_displayed: ", element.is_displayed())
            child_element = element.find_elements_by_css_selector("*")
            url = child_element[1].get_attribute("href")
            # print(url)
            t = child_element[0].get_attribute('textContent')[0:10]
            timeArray = time.mktime(time.strptime(t, "%d.%m.%Y"))
            # print(timeArray)
            if int(self.day_time) > int(timeArray):
                flag = 0
                break
            self.news_url_list.append(url)
            #print(t)
        if flag == 1:
            pages_num = driver.find_element(By.XPATH, "//a[@class='pagenava']").get_attribute('textContent')
            # print("pages_num: ", pages_num)
            url_pages = driver.find_elements(By.XPATH, "//a[@class='pagenav']")
            # print("url_pages: ", url_pages)
            url_page = url_pages[int(pages_num)].get_attribute("href")
            # print("下一页网址: ", url_page)
            #driver.quit()
            self.get_news_url_list(driver, url_page)
        sleep(3)
        #except:
            #print("网址获取出错")
        return self.news_url_list

    # 获取文本、图片地址、视频地址
    def get_data_list(self, news_url_list):
        data_list = []
        img_list = []
        video_list = []
        for news_url in news_url_list:
            try:
                dic = {}
                response = requests.get(news_url, headers=self.head).text
                tree = etree.HTML(response)
                # 情报源
                dic["情报源"] = "俄罗斯国防部"
                # 情报源注册时长（空）
                dic["情报源注册时长"] = ""
                # 情报源关注者数目(空)
                dic["情报源关注者数目"] = ""
                # 文章标题
                dic["标题"] = self.is_content(tree.xpath("//div[@id='center']/h1/text()"))
                print("文章标题: ", dic["标题"])
                # 文章作者
                dic["作者"] = self.is_content(tree.xpath("//a[@class='date']/text()"))
                print("文章作者: ", dic["作者"])
                # 文章发布时间
                publish_time = tree.xpath("//*[@id='center']/span/text()")
                dic["发布时间"] = datetime.strptime(self.is_content(publish_time), "%d.%m.%Y (%H:%M)")
                print("文章发布时间: ", dic["发布时间"])
                # 爬取时间
                now = datetime.now()
                dic["爬取时间"] = now.strftime('%Y-%m-%d %H:%M:%S')
                # 文章正文
                content = ""
                temp = tree.xpath('//*[@id="center"]/p')
                for t in temp:
                    content += t.xpath('string(.)')
                dic["正文"] = content
                print("正文: ", dic["正文"])
                # 阅读数
                dic["阅读数"] = ""
                # 转发数
                dic["转发数"] = ""
                # 链接
                dic["url"] = news_url
                data_list.append(dic)
                # 获取滚动图片
                parent_img = tree.xpath("//*[@class='ad-thumb-list']/li")
                for i in parent_img:
                    url = "https://eng.mil.ru/" + self.is_content(i.xpath("./a/@href"))
                    img_list.append([url, dic["标题"]+ "_" + str(self.image_name)])
                    self.image_name = self.image_name + 1
                # 获取固定图片
                if len(parent_img) == 0:
                    img = tree.xpath("//img[@class='image0']/@src")
                    if len(img)!=0:
                        url = "https://eng.mil.ru/" + self.is_content(img)
                        img_list.append([url, dic["标题"]+ "_" +str(self.image_name)])
                        self.image_name = self.image_name + 1
                # 获取视频url
                url = tree.xpath("//video/*/@src")
                for u in url:
                    video_list.append([u, dic["标题"] + "_" + str(self.video_name) + '.mp4'])
                    self.video_name = self.video_name + 1
            except:
                print("当前文章爬取出错，跳过")
        return data_list, img_list, video_list

    def save_data(self, data_list):
        timestamp = datetime.now().strftime('%Y%m%d%H%M%S%f')[:8]
        if not os.path.exists(self.path):
            os.makedirs(self.path)
        text_name = "俄罗斯国防部_" + str(timestamp) + ".json"
        paths = os.path.join(self.path, text_name)
        with open(paths, 'a', encoding='utf-8') as f:
            f.write(json.dumps(data_list, indent=2,ensure_ascii=False,default=str))

    # 多线程下载图片
    def download_img(self, img_url, title, semlock):
        # try:
        semlock.acquire()
        paths = os.path.join(self.path, 'images/')
        if not os.path.exists(paths):
            os.makedirs(paths)
        opener = urllib.request.build_opener()  # 实例化一个OpenerDirector
        opener.addheaders = [('User-Agent','Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 101.0.4951.67 Safari / 537.36')]  # 添加header,注意格式
        urllib.request.install_opener(opener)  # 将OpenerDirector装进opener
        urllib.request.urlretrieve(img_url, '{0}{1}.jpg'.format(paths, str(title)))
        print(img_url, " 下载完成")
        semlock.release()
        # except:
        #     print(img_url, " 爬取失败")

    def download_video(self, video_url, title,semlock):
        # print("爬取视频")
        # response = requests.get('https://eng.mil.ru/en/news_page/country/more.htm?id=12413390@egNews', headers=self.head).text
        # tree = etree.HTML(response)
        # url = tree.xpath("//video/*/@src")

        #semlock.acquire()
        #try:
        paths = os.path.join(self.path, 'video/')
        if not os.path.exists(paths):
            os.makedirs(paths)
        paths = paths + title
        s = requests.Session()
        # 如果失败重试5次
        s.mount(video_url,
                HTTPAdapter(max_retries=self.file_download_timeout[0]))
        downloaded = requests.get(video_url, stream=True,headers=self.head,timeout = (self.file_download_timeout[1],self.file_download_timeout[2]))
        # 获取视频总长度
        length = float(downloaded.headers['content-length'])
        with open(paths, 'wb') as f:
            # f.write(downloaded.content)
            '''
            使用手动设置更新
            total 设置总大小
            initial 当前操作文件的大小
            desc 进度条前的描述
            ncols=120设置进度条显示长度
            nit_scale 如果设置，迭代的次数会自动按照十、百、千来添加前缀，默认为false
            '''
            pbar = tqdm(total=length, initial=os.path.getsize(paths), unit_scale=True, desc=paths, ncols=120)
            for chuck in downloaded.iter_content(chunk_size=512):
                f.write(chuck)
                # 手动更新的大小
                pbar.update(512)
            #semlock.release()
        # except:
        #     print(video_url," 爬取失败")

    def is_content(self, list):
        if len(list) == 0:
            return ""
        else:
            return str(list[0])

    def crawler(self):
        driver = self.get_driver()
        news_url_list = self.get_news_url_list(driver, self.url)
        print("网址个数：", len(news_url_list))
        driver.quit()
        data_list, img_list, video_list= self.get_data_list(news_url_list)
        self.save_data(data_list)
        # 最大线程数
        max_connections = 3
        semlock = threading.BoundedSemaphore(max_connections)
        for img in img_list:
            print("图片url : ", img[0], img[1])
            t = threading.Thread(target=self.download_img, args=(img[0], img[1], semlock))
            t.start()
            time.sleep(1)
        for v in video_list:
            print("视频url : ", v[0], v[1])
            # t = threading.Thread(target=self.download_video, args=(v[0], v[1], semlock))
            # t.start()
            # time.sleep(1)
            self.download_video(v[0],v[1],semlock)

    def crawler_keyword(self,keyword = "china"):
        driver = self.get_driver()
        url = "https://eng.mil.ru/en/news_page/country.htm"
        try:
            driver.get(url)
        except WebDriverException:
            print("网页打开错误，重新尝试")
        driver.find_element(By.XPATH, "//input[@class='searchfield']").send_keys(keyword)
        time.sleep(3)
        driver.find_element(By.XPATH, "//input[@class='searchbutton']").click()
        time.sleep(3)
        list = driver.find_elements(By.XPATH, "//*[@id='center']/table/tbody/tr")
        url_list = []
        for i in list:
            #print(i.get_attribute('textContent'))
            url = i.find_element(By.TAG_NAME,'a').get_attribute("href")
            print("url: ",url)
            url_list.append(url)
        data_list, img_list, video_list = self.get_data_list(url_list)
        self.save_data(data_list)
        # 最大线程数
        max_connections = 3
        semlock = threading.BoundedSemaphore(max_connections)
        for img in img_list:
            print("图片url : ", img[0], img[1])
            t = threading.Thread(target=self.download_img, args=(img[0], img[1], semlock))
            t.start()
            time.sleep(1)
        for v in video_list:
            print("视频url : ", v[0], v[1])
            # t = threading.Thread(target=self.download_video, args=(v[0], v[1], semlock))
            # t.start()
            # time.sleep(1)
            self.download_video(v[0], v[1], semlock)

if __name__ == '__main__':
    myZGJScrawler = ZGJSProcessor()
    myZGJScrawler.crawler_keyword()


#!/usr/bin/python3

import curses
from random import randrange

def neigh_num (state, i, j, w, h):
    cnt = 0
    pos = [(-1, -1), (0, -1), (1, -1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)]
    for (x, y) in pos:
        if i + x >= 0 and i + x < w and j + y >= 0 and j + y < h:
            if state[i + x][j + y]:
                cnt += 1
    return cnt

def evolve (state, w, h):
    new_state = [[False for _ in range(h)] for _ in range(w)]
    for i in range(w):
        for j in range(h):
            cnt = neigh_num(state, i, j, w, h)
            if (state[i][j] and (cnt == 2 or cnt == 3)) or (not state[i][j] and cnt == 3):
                new_state[i][j] = True
    return new_state

def random_state (state, n, w, h):
    for _ in range(n):
        (i, j) = (randrange(w), randrange(h))
        while state[i][j] == True:
            (i, j) = (randrange(w), randrange(h))
        state[i][j] = True

def print_state (stdscr, state, w, h):
    for i in range(w):
        for j in range(h):
            if state[i][j]:
                stdscr.addstr(j, i, 'O') # curses display is reversed : (y, x)

def main (stdscr):
    curses.curs_set(False) # Do not display blinking cursor
    (w, h) = (curses.COLS, curses.LINES)
    state = [[False for _ in range(h)] for _ in range(w)]
    random_state(state, int((w * h) / 10), w, h)

    k = 'a'
    while k != 'q':
        state = evolve(state, w, h)
        stdscr.clear()
        print_state(stdscr, state, w, h)
        stdscr.refresh()
        k = stdscr.getkey()

curses.wrapper(main)


import os
import neat

import backprop_neat
from anti_spoofing.eval_functions import evaluate_eer_acc
import torch
import numpy as np
import multiprocessing

import neat_local.visualization.visualize as visualize
from anti_spoofing.utils_ASV import make_visualize, show_stats
from anti_spoofing.data_loader import load_data
from anti_spoofing.eval_function_eoc import eval_genome_eoc, quantified_eval_genome_eoc, eval_eer_gc, \
    ProcessedASVEvaluatorEocGc

"""
NEAT APPLIED TO ASVspoof 2019
"""

def run(config_file, n_gen):
    """
    Launches a run until convergence or max number of generation reached
    :param config_file: path to the config file
    :param n_gen: max number of generation
    :return: the best genontype (winner), the configs, the stats of the run and the accuracy on the testing set
    """
    # Load configuration.
    config_ = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
                          neat.DefaultSpeciesSet, neat.DefaultStagnation,
                          config_file)

    # Create the population, which is the top-level object for a NEAT run.
    p = neat.Population(config_)

    # Add a stdout reporter to show progress in the terminal.
    p.add_reporter(neat.StdOutReporter(True))
    stats_ = backprop_neat.StatisticsReporter()
    p.add_reporter(stats_)
    p.add_reporter(backprop_neat.Checkpointer(generation_interval=1000, time_interval_seconds=None))

    # Run for up to n_gen generations.
    if USE_DATASET:
        multi_evaluator = ProcessedASVEvaluatorEocGc(multiprocessing.cpu_count(), quantified_eval_genome_eoc,
                                                     data=trainloader, pop=config_.pop_size, config=config_,
                                                     gc_eval=eval_eer_gc, validation_data=devloader,
                                                     batch_increment=200, initial_batch_size=100, batch_generations=50)

    else:
        multi_evaluator = ProcessedASVEvaluatorEocGc(multiprocessing.cpu_count(), eval_genome_eoc, data=trainloader,
                                                     pop=config_.pop_size, config=config_,
                                                     gc_eval=eval_eer_gc, validation_data=devloader)

    winner_ = p.run(multi_evaluator.evaluate, n_gen)

    gc = multi_evaluator.gc
    generations_gc = multi_evaluator.app_gc
    print("Grand champion appeared at generation ", generations_gc)

    # Display the winning genome.
    # print('\nBest genome:\n{!s}'.format(winner_))

    return gc, winner_, config_, stats_, generations_gc

if __name__ == '__main__':
    # Determine path to configuration file. This path manipulation is
    # here so that the script will run successfully regardless of the
    # current working directory.
    local_dir = os.path.dirname(__file__)
    config_path = os.path.join(local_dir, 'ASV_neat_preprocessed.cfg')

    USE_DATASET = True

    trainloader, devloader, evalloader = load_data(batch_size=100, length=3 * 16000, num_train=40000,
                                                   num_test=40000, batch_size_test=5000, option="lfcc",
                                                   multi_proc=True, include_eval=True, return_dataset=USE_DATASET)

    eer_list_gc = []
    accuracy_list_gc = []
    eer_list = []
    accuracy_list = []

    eer_list_gc_eval = []
    accuracy_list_gc_eval = []
    eer_list_eval = []
    accuracy_list_eval = []

    winner_list = []
    gc_list = []
    generation_gc_list = []

    for iterations in range(20):
        print("iterations number =", iterations)
        gc, winner, config, stats, gen_gc = run(config_path, 200)
        visualize.plot_stats(stats, ylog=False, view=True)

        with torch.no_grad():
            eer, accuracy = evaluate_eer_acc(winner, config, devloader)
            eer_eval, accuracy_eval = evaluate_eer_acc(winner, config, evalloader)
            eer_gc, accuracy_gc = evaluate_eer_acc(gc, config, devloader)
            eer_gc_eval, accuracy_gc_eval = evaluate_eer_acc(gc, config, evalloader)
        eer_list.append(eer)
        accuracy_list.append(accuracy)
        eer_list_gc.append(eer_gc)
        accuracy_list_gc.append(accuracy_gc)
        eer_list_eval.append(eer_eval)
        accuracy_list_eval.append(accuracy_eval)
        eer_list_gc_eval.append(eer_gc_eval)
        accuracy_list_gc_eval.append(accuracy_gc_eval)
        generation_gc_list.append(gen_gc)

    print("\n")
    print("equal error rate", eer_list)
    show_stats(np.array(eer_list))

    print("\n")
    print("equal error rate gc", eer_list_gc)
    show_stats(np.array(eer_list_gc))

    print("\n")
    print("equal error rate eval", eer_list_eval)
    show_stats(np.array(eer_list_eval))

    print("\n")
    print("equal error rate gc eval", eer_list_gc_eval)
    show_stats(np.array(eer_list_gc_eval))

    print("\n")
    print("Grand champions appeared at generations", generation_gc_list)
    show_stats(np.array(generation_gc_list))

    # make_visualize(winner, config, stats)
    # make_visualize(gc, config, stats)


import os
import time
import logging
from tempfile import NamedTemporaryFile
try:
    from utils.utils import Utils
    from utils.ufm_rest_client import UfmRestClient, HTTPMethods
    from utils.args_parser import ArgsParser
    from utils.config_parser import ConfigParser
    from utils.logger import Logger
except ModuleNotFoundError as e:
    print("Error occurred while importing python modules, "
          "Please make sure that you exported your repository to PYTHONPATH by running: "
          f'export PYTHONPATH="${{PYTHONPATH}}:{os.path.dirname(os.getcwd())}"')

class UfmTopologyConfigParser(ConfigParser):

    config_file = "ufm_topology.cfg"

    UFM_TOPOLOGY_SECTION = "ufm-topology"
    UFM_TOPOLOGY_SECTION_STORED_DATA_PATH = "path_to_export"
    UFM_TOPOLOGY_SECTION_GEPHI_FILE_NAME = "gephi_file_name"
    UFM_TOPOLOGY_SECTION_TOPO_FILE_NAME = "topo_file_name"
    UFM_TOPOLOGY_SECTION_EXPORT_TO_GEXF = "export_to_gephi"
    UFM_TOPOLOGY_SECTION_EXPORT_AS_TOPO = "export_as_topo"

    UFM_TOPOLOGY_COMPARE_SECTION = 'ufm-topology-compare'
    UFM_TOPOLOGY_COMPARE_SECTION_COMPARE_TOPOLOGY_WITH = "compare_topology_with"
    UFM_TOPOLOGY_COMPARE_SECTION_EXPORT_COMPARE_TO_GEPHI= "export_compare_topology_to_gephi"

    def __init__(self, args):
        super().__init__(args)
        self.sdk_config.read(self.config_file)

    def get_path_to_export(self):
        return self.get_config_value(self.args.path_to_export,
                                     self.UFM_TOPOLOGY_SECTION,
                                     self.UFM_TOPOLOGY_SECTION_STORED_DATA_PATH,
                                     'api_results')

    def get_gephi_file_name(self):
        return self.get_config_value(self.args.gephi_file_name,
                                     self.UFM_TOPOLOGY_SECTION,
                                     self.UFM_TOPOLOGY_SECTION_GEPHI_FILE_NAME,
                                     f'{Utils.get_timebased_filename()}_topology.gexf')

    def get_topo_file_name(self):
        return self.get_config_value(self.args.topo_file_name,
                                     self.UFM_TOPOLOGY_SECTION,
                                     self.UFM_TOPOLOGY_SECTION_TOPO_FILE_NAME,
                                     f'{Utils.get_timebased_filename()}.topo')

    def get_export_to_gephi(self):
        return self.safe_get_bool(self.args.export_to_gephi,
                                  self.UFM_TOPOLOGY_SECTION,
                                  self.UFM_TOPOLOGY_SECTION_EXPORT_TO_GEXF,
                                  False)

    def get_export_as_topo(self):
        return self.safe_get_bool(self.args.export_as_topo,
                                  self.UFM_TOPOLOGY_SECTION,
                                  self.UFM_TOPOLOGY_SECTION_EXPORT_AS_TOPO,
                                  False)

    def get_compare_topo_with(self):
        return self.get_config_value(self.args.compare_topology_with,
                                     self.UFM_TOPOLOGY_COMPARE_SECTION,
                                     self.UFM_TOPOLOGY_COMPARE_SECTION_COMPARE_TOPOLOGY_WITH,
                                     "")

    def get_export_topology_compare_to_gephi(self):
        return self.safe_get_bool(self.args.export_compare_topology_to_gephi,
                                  self.UFM_TOPOLOGY_COMPARE_SECTION,
                                  self.UFM_TOPOLOGY_COMPARE_SECTION_EXPORT_COMPARE_TO_GEPHI,
                                  False)

class UfmTopologyConstants(object):
    NODE_GUID = "guid"
    NODE_NAME = "system_name"
    NODE_IP = "ip"
    NODE_SEVERITY = "severity"
    NODE_TYPE = "type"

    NODE_ATTR_ID = "id"
    NODE_ATTR_TITLE = "title"
    NODE_ATTR_TYPE = "type"

    NODES_ATTRIBUTES = [
        {
            NODE_ATTR_ID: NODE_NAME,
            NODE_ATTR_TITLE: "Name",
            NODE_ATTR_TYPE: "string"
        },
        {
            NODE_ATTR_ID: NODE_SEVERITY,
            NODE_ATTR_TITLE: "Severity",
            NODE_ATTR_TYPE: "string"
        },
        {
            NODE_ATTR_ID: NODE_GUID,
            NODE_ATTR_TITLE: "GUID",
            NODE_ATTR_TYPE: "string"
        },
        {
            NODE_ATTR_ID: NODE_IP,
            NODE_ATTR_TITLE: "IP",
            NODE_ATTR_TYPE: "string"
        },
        {
            NODE_ATTR_ID: NODE_TYPE,
            NODE_ATTR_TITLE: "Type",
            NODE_ATTR_TYPE: "string"
        }
    ]

    LINK_SOURCE_GUID = "source_guid"
    LINK_DEST_GUID = "destination_guid"
    LINK_NAME = "name"

    UNAFFECTED_NODE_COLOR = {
        "R": 118,
        "G": 185,
        "B": 0,
        "A": 1
    }

    ADDED_NODE_COLOR = {
        "R": 77,
        "G": 137,
        "B": 189,
        "A": 1
    }

    DELETED_NODE_COLOR = {
        "R": 150,
        "G": 150,
        "B": 150,
        "A": 1
    }

    args_list = [
        {
            "name": "--export_to_gephi",
            "help": "Option to export the topology as gexf file",
            "no_value": True
        }, {
            "name": "--export_as_topo",
            "help": "Option to export the topology as topo file",
            "no_value": True
        }, {
            "name": "--path_to_export",
            "help": "Option to specify where the exported files will be stored [Default = 'api_results']"
        }, {
            "name": "--gephi_file_name",
            "help": "Option to specify the name of the exported Gephi file [Default = 'gephi.gexf']"
        }, {
            "name": "--topo_file_name",
            "help": "Option to specify the name of the exported topo file [Default = 'topology.gexf']"
        }, {
            "name": "--compare_topology_with",
            "help": "The path of the external topo file to compare it with the current topology"
        }, {
            "name": "--export_compare_topology_to_gephi",
            "help": "Option to export the result of the topology compare as gexf file",
            "no_value": True
        }
    ]

class UfmTopologyGephiExporter(object):

    def __init__(self):
        self.fo = None

    def _write_node_attributes(self):
        self.fo.write('''
\t\t<attributes class="node" mode="static">
''')
        for attr in UfmTopologyConstants.NODES_ATTRIBUTES:
            self.fo.write('\t\t\t<attribute id="{0}" title="{1}" type="{2}"/>\n'
                          .format(attr.get(UfmTopologyConstants.NODE_ATTR_ID),
                                  attr.get(UfmTopologyConstants.NODE_ATTR_TITLE),
                                  attr.get(UfmTopologyConstants.NODE_ATTR_TYPE)))
        self.fo.write('''\t\t</attributes>''')

    def _write_node(self, node, added=False, removed=False):
        self.fo.write('\t\t\t<node id="{0}" label="{1}">\n'
                      .format(node.get(UfmTopologyConstants.NODE_GUID),
                              node.get(UfmTopologyConstants.NODE_NAME)))

        if added:
            node_color = UfmTopologyConstants.ADDED_NODE_COLOR
        elif removed:
            node_color = UfmTopologyConstants.DELETED_NODE_COLOR
        else:
            node_color = UfmTopologyConstants.UNAFFECTED_NODE_COLOR
        self.fo.write(f'\t\t\t\t<viz:color r="{node_color.get("R")}" g="{node_color.get("G")}" '
                      f'b="{node_color.get("B")}" a="{node_color.get("A")}"/>\n')

        self.fo.write('\t\t\t\t<attvalues>\n')
        for attr in UfmTopologyConstants.NODES_ATTRIBUTES:
            self.fo.write('\t\t\t\t\t<attvalue for="{0}" value="{1}"/>\n'
                          .format(attr.get(UfmTopologyConstants.NODE_ATTR_ID),
                                  node.get(attr.get(UfmTopologyConstants.NODE_ATTR_ID))))
        self.fo.write('\t\t\t\t</attvalues>\n')
        self.fo.write('\t\t\t</node>\n')

    def _write_nodes(self, nodes, added_nodes=None, removed_nodes=None):
        self.fo.write('''
\t\t<nodes>
''')
        for node in nodes:
            self._write_node(node, added= added_nodes and node.get(UfmTopologyConstants.NODE_GUID) in added_nodes)
        if removed_nodes:
            for key, node in removed_nodes.items():
                self._write_node(node, removed=True)
        self.fo.write('''\t\t</nodes>''')

    def _write_edges(self, links):
        self.fo.write('''
\t\t<edges>
''')
        for link in links:
            self.fo.write('\t\t\t<edge id="{0}" source="{1}" target="{2}"/>\n'
                          .format(link.get(UfmTopologyConstants.LINK_NAME),
                                  link.get(UfmTopologyConstants.LINK_SOURCE_GUID),
                                  link.get(UfmTopologyConstants.LINK_DEST_GUID)))
        self.fo.write('''\t\t</edges>
    ''')

    def close(self, path_to_export):
        """
        Close the temporary file and rename it to "normal"
        """
        if not self.fo is None:
            self.fo.close()
            if os.path.exists(path_to_export):
                os.remove(path_to_export)
            os.rename(self.fo.name, path_to_export)
            self.fo = None

    def get_added_nodes_dict(self, added_nodes_arr):
        added_nodes = {}
        for node in added_nodes_arr:
            added_nodes[node.get(UfmTopologyConstants.NODE_GUID)] = node
        return added_nodes

    def get_removed_nodes_dict(self, removed_nodes_arr):
        removed_nodes = {}
        for node in removed_nodes_arr:
            _node = {}
            # set the node attributes of the removed nodes because it dose not exist in the compare result
            for attr in UfmTopologyConstants.NODES_ATTRIBUTES:
                _id = attr.get(UfmTopologyConstants.NODE_ATTR_ID)
                if _id == UfmTopologyConstants.NODE_NAME:
                    _node[UfmTopologyConstants.NODE_NAME] = node
                else:
                    _node[_id] = "N/A"

            removed_nodes[node] = _node
        return removed_nodes

    def export_topology_as_gexf_file(self, nodes, links, path_to_export, compare_result=None):
        # this function will export the topology as GEXF  (Graph Exchange XML Format)
        # .GEXF file can be used later on in the visualization tools (e.g. Gephi)

        added_nodes = None
        removed_nodes = None
        if compare_result:
            # Convert the result of compare from arrays to dicts
            added_nodes = self.get_added_nodes_dict(compare_result.get("added").get("nodes"))
            removed_nodes = self.get_removed_nodes_dict(compare_result.get("removed").get("nodes"))

        self.fo = NamedTemporaryFile(delete=False, dir=os.path.dirname(path_to_export),
                                     prefix='.tmp',
                                     mode='w', encoding='ascii') # Open temporary file streaming

        logging.info("Generating %s file",path_to_export)

        self.fo.write('''<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft"
    xmlns:viz="http://www.gexf.net/1.2draft/viz"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd"
    version="1.1">
    <graph mode="static" defaultedgetype="undirected">''')
        self._write_node_attributes()
        self._write_nodes(nodes= nodes, added_nodes=added_nodes, removed_nodes=removed_nodes)
        self._write_edges(links= links)

        self.fo.write('''</graph>
</gexf>
''')
        self.close(path_to_export= path_to_export)
        logging.info("The file %s is saved successfully", path_to_export)

class UFMTopologyTopoExporter:

    @staticmethod
    def export_topology_as_topo_file(export_to_path):
        try:
            url = "Topology_Compare/topology_file"
            response = ufm_rest_client.send_request(url, method=HTTPMethods.POST)
            response = response.json()
            file_name = response.get('file_name')
            file_content = ufm_rest_client.send_request(url + "/" + file_name)
            with open(export_to_path, 'wb') as f:
                f.write(file_content.content)
                logging.info("The file %s is saved successfully", export_to_path)
        except Exception as e:
            logging.error(f'Error to export topology: {e}')

class UFMTopologyCompare:

    @staticmethod
    def compare_topology_with_custom_topo(path_custom_topo_file, path_to_export):
        try:
            with open(path_custom_topo_file, 'rb') as f:
                url = "Topology_Compare/networkdiff"
                response = ufm_rest_client.send_request(url, method=HTTPMethods.POST,files={'filename': f})
                job_id = response.json()
                job_is_completed = False
                while not job_is_completed:
                    time.sleep(5)
                    jobs_url = f'jobs/{job_id}'
                    job_response = ufm_rest_client.send_request(jobs_url)
                    if job_response.raise_for_status():
                        break
                    job_response = job_response.json()
                    job_is_completed = job_response.get("Status") == "Completed"
                if job_is_completed:
                    topo_diff_response = ufm_rest_client.send_request(url).json()
                    Utils.write_json_to_file(path_to_export,topo_diff_response)
                    return topo_diff_response

        except Exception as e:
            logging.error(f'Error to compare topology: {e}')

if __name__ == "__main__":

    # init app args
    args = ArgsParser.parse_args("UFM Topology Management", UfmTopologyConstants.args_list)

    # init app config parser & load config files
    config_parser = UfmTopologyConfigParser(args)

    # init logs configs
    logs_file_name = config_parser.get_logs_file_name()
    logs_level = config_parser.get_logs_level()
    Logger.init_logs_config(logs_file_name, logs_level)

    # init ufm rest client
    ufm_rest_client = UfmRestClient(host = config_parser.get_ufm_host(),
                                    client_token=config_parser.get_ufm_access_token())

    if config_parser.get_export_to_gephi():
        # load systems,links
        nodes = ufm_rest_client.get_systems()
        links = ufm_rest_client.get_links()
        gephi_file_name = config_parser.get_gephi_file_name()
        path_to_export = config_parser.get_path_to_export() + '/' + gephi_file_name
        exporter = UfmTopologyGephiExporter()
        exporter.export_topology_as_gexf_file(nodes=nodes, links=links, path_to_export=path_to_export)

    if config_parser.get_export_as_topo():
        topo_file_name = config_parser.get_topo_file_name()
        path_to_export = config_parser.get_path_to_export() + '/' + topo_file_name
        UFMTopologyTopoExporter.export_topology_as_topo_file(path_to_export)

    if len(config_parser.get_compare_topo_with()) > 0:
        file_path = config_parser.get_compare_topo_with()

        result_file_name = Utils.get_timebased_filename()
        export_to_path = f'{config_parser.get_path_to_export()}/{result_file_name}_compare.json'
        compare_result = UFMTopologyCompare.compare_topology_with_custom_topo(file_path,export_to_path)

        if config_parser.get_export_topology_compare_to_gephi():
            nodes = ufm_rest_client.get_systems()
            links = ufm_rest_client.get_links()
            path_to_export = f'{config_parser.get_path_to_export()}/{result_file_name}_compare.gexf'

            exporter = UfmTopologyGephiExporter()
            exporter.export_topology_as_gexf_file(nodes=nodes, links=links,
                                                  path_to_export=path_to_export, compare_result=compare_result)


tokens = ('COUNT','SYMBOL')

# Tokens
t_SYMBOL    = r'[A-Z][a-z]*'

def t_COUNT(t):
    r'\d+'
    t.value = int(t.value)
    return t

# Ignored characters
t_ignore = " \t"

errorLine = None

def t_newline(t):
    r'\n+'
    t.lexer.lineno += t.value.count("\n")

def setErrorLine(eLine):
    global errorLine
    errorLine = eLine

def getErrorLine():
    global errorLine
    return errorLine

def t_error(t):
    if (t.value[0] != '#'):
        t.lexer.skip(1)
        eLine = tok.lineno
        setErrorLine(eLine)
        print("Error in formula")
    else:
        print(count)
        exit()

# Build the lexer
import ply.lex as lex
lex.lex()

if __name__ == "__main__":
    input_data = ""
    new_input_data = ""

    while new_input_data != "#":
        new_input_data = input()

        if(new_input_data != "#"):
            input_data += new_input_data + "\n"

input_data += "#"
lex.input(input_data)

names = {'H', 'He', 'Li', 'Be', 'B', 'C', 'N', 'O', 'F', 'Ne', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'Cl', 'Ar', 'K', 'Ca', 'Sc',
'Ti', 'V', 'Cr', 'Mn', 'Fe', 'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Kr', 'Rb', 'Sr', 'Y', 'Zr', 'Nb', 'Mo', 'Tc', 'Ru',
'Rh', 'Pd', 'Ag', 'Cd', 'In', 'Sn', 'Sb', 'Te', 'I', 'Xe', 'Cs', 'Ba', 'La', 'Ce' , 'Pr', 'Nd', 'Pm', 'Sm', 'Eu', 'Gd', 'Tb', 'Dy', 'Ho', 'Er', 'Tm', 'Yb', 'Lu', 'Hf',
'Ta', 'W', 'Re', 'Os', 'Ir', 'Pt', 'Au', 'Hg', 'Tl', 'Pb', 'Bi', 'Po', 'At', 'Rn', 'Fr', 'Ra', 'Ac', 'Th', 'Pa', 'U', 'Np', 'Pu', 'Am', 'Cm', 'Bk', 'Cf', 'Es', 'Fm',
'Md', 'No', 'Lr', 'Rf', 'Db', 'Sg', 'Bh', 'Hs', 'Mt', 'Ds', 'Rg', 'Cn', 'Uut'}

count = 0
tok = lex.token()
tempSymbol = ""

while True:
    templine = tok.lineno

    while templine == tok.lineno:
        if templine != getErrorLine():
            if (tok is None):
                print('Error in formula')
                setErrorLine(templine)
            elif (tok.type == 'COUNT'):
                count = count + tok.value - 1
            elif (tok.type == 'SYMBOL'):
                if (str(tok.value) in names and str(tok.value) != tempSymbol):
                    count = count+1
                    tempSymbol = str(tok.value)
                else:
                    print('Error in formula')
                    setErrorLine(templine)

        tok = lex.token()
        if (templine != None):
            if (templine != tok.lineno):
                if (templine!=getErrorLine()):
                    print(count)
                count = 0

#code adapted from dabeaz.com


from __future__ import absolute_import, division, print_function
from iotbx.pdb.tst_pdb import dump_pdb
from cctbx import geometry_restraints
from iotbx.pymol import pml_stick, pml_write
from cctbx.array_family import flex
from scitbx.matrix import col, sqr
from scitbx.math import euler_angles_as_matrix
from libtbx.test_utils import approx_equal, not_approx_equal
import random
import sys
from six.moves import range
from six.moves import zip

if (1): # fixed random seed to avoid rare failures
  random.seed(0)
  flex.set_random_seed(0)

class residual_functor(object):

  def __init__(self, restraint_type, **keyword_arguments):
    assert "sites" not in keyword_arguments
    self.restraint_type = restraint_type
    self.keyword_arguments = keyword_arguments

  def __call__(self, sites):
    self.keyword_arguments["sites"] = sites
    return self.restraint_type(**self.keyword_arguments).residual()
    del self.keyword_arguments["sites"]

def finite_differences(sites, residual_obj, eps=1.e-6):
  sites_mod = sites[:]
  gradients = []
  for i_site,site in enumerate(sites):
    grad = []
    for i_x in range(3):
      t = []
      for signed_eps in [eps, -eps]:
        add = [0]*3
        add[i_x] = signed_eps
        sites_mod[i_site] = site + col(add)
        r = residual_obj(sites_mod)
        t.append(r)
        sites_mod[i_site] = site
      grad.append((t[0]-t[1])/(2*eps))
    gradients.append(grad)
  return gradients

def exercise_bond():
  for i_trial in range(5):
    sites = random_sites(n_sites=2)
    b = geometry_restraints.bond(sites=sites, distance_ideal=0, weight=0)
    distance_ideal = b.distance_model
    sigma = distance_ideal * 0.01
    weight = 1 / sigma**2
    residual_obj = residual_functor(
      restraint_type=geometry_restraints.bond,
      distance_ideal=distance_ideal,
      weight=weight)
    for i_pert in range(5):
      if (i_pert == 0):
        sites_mod = sites
      else:
        sites_mod = []
        for site in sites:
          shift = col([random.uniform(-3,3)*sigma for i in range(3)])
          sites_mod.append(site+shift)
      b = geometry_restraints.bond(
        sites=sites_mod, distance_ideal=distance_ideal, weight=weight)
      ag = b.gradients()
      fg = finite_differences(sites_mod, residual_obj)
      for analytical,finite in zip(ag,fg):
        assert approx_equal(analytical, finite)

def exercise_nonbonded(nonbonded_type, repulsion_function):
  for i_trial in range(5):
    sites = random_sites(n_sites=2)
    r = nonbonded_type(
      sites=sites,
      vdw_distance=1,
      function=repulsion_function)
    vdw_distance = r.delta
    residual_obj = residual_functor(
      restraint_type=nonbonded_type,
      vdw_distance=vdw_distance,
      function=repulsion_function)
    for i_pert in range(5):
      if (i_pert == 0):
        sites_mod = sites
      else:
        sites_mod = []
        for site in sites:
          shift = col([random.uniform(-1,1)*vdw_distance*0.05
            for i in range(3)])
          sites_mod.append(site+shift)
      r = nonbonded_type(
        sites=sites_mod,
        vdw_distance=vdw_distance,
        function=repulsion_function)
      if (   not hasattr(repulsion_function, "irexp")
          or repulsion_function.irexp <= 2
          or not_approx_equal(r.residual(), 0)):
        fg = flex.vec3_double(finite_differences(sites_mod, residual_obj)) \
          .as_double()
        ag = flex.vec3_double(r.gradients()).as_double()
        scale = max(1, flex.mean(flex.abs(fg)))
        assert approx_equal(ag/scale, fg/scale, eps=1.e-2)
  sites = [[0,0,0], [0,0,0]]
  vdw_distance = 1.8
  for i_step in range(1,100):
    delta = i_step/50.
    sites[1][0] = delta
    r = nonbonded_type(
      sites=[col(site) for site in sites],
      vdw_distance=vdw_distance,
      function=repulsion_function)
    assert approx_equal(delta, r.delta)

def exercise_angle():
  eps = 1
  n_trials = 20
  for i_trial in range(n_trials):
    if (i_trial < -99):
      if (i_trial == n_trials-1):
        eps = 0
      else:
        eps *= 0.1
      sites = [col(site) for site in [(-1,0,0),(0,0,0),(1,eps,0)]]
      angle_ideal = 170
    else:
      while 1:
        sites = random_sites(n_sites=3)
        angle_ideal = geometry_restraints.angle(
          sites, angle_ideal=0, weight=0).angle_model
        if (angle_ideal > 10):
          break
    sigma = angle_ideal * 0.1
    weight = 1 / sigma**2
    residual_obj = residual_functor(
      restraint_type=geometry_restraints.angle,
      angle_ideal=angle_ideal,
      weight=weight)
    for i_pert in range(5):
      if (i_pert == 0):
        sites_mod = sites
      else:
        sites_mod = []
        for site in sites:
          shift = col([random.uniform(3,3)*sigma for i in range(3)])
          sites_mod.append(site+shift)
      a = geometry_restraints.angle(
        sites=sites_mod, angle_ideal=angle_ideal, weight=weight)
      if (eps != 0):
        ag = a.gradients()
      fg = finite_differences(sites_mod, residual_obj)
      if (eps == 0):
        for finite in fg:
          print(finite)
          print()
      else:
        for analytical,finite in zip(ag,fg):
          if (0):
            print(analytical)
            print(finite)
          if (0):
            for x,y in zip(analytical, finite):
              if (y == 0): print(None, end=' ')
              else: print(x/y, end=' ')
            print()
          if (0):
            print()
          assert approx_equal(analytical, finite,
                              eps=max(1.e-6,max(analytical)*1.e-6))

def random_site(max_coordinate):
  return col([random.uniform(-max_coordinate, max_coordinate)
    for i in range(3)])

def random_sites(n_sites, max_coordinate=10, min_distance=0.5,
                 max_trials=1000):
  sites = []
  run_away_counter = 0
  while 1:
    run_away_counter += 1
    assert run_away_counter < max_trials
    new_site = random_site(max_coordinate)
    ok = True
    for site in sites:
      if ((new_site-site).norm_sq() < min_distance**2):
        ok = False
        break
    if (ok):
      sites.append(new_site)
      run_away_counter = 0
      if (len(sites) == n_sites):
        break
  return sites

def exercise_dihedral_core(sites, angle_ideal, angle_esd, periodicity,
                                  angle_model):
  assert periodicity > 0
  for signed_periodicity in [periodicity, -periodicity]:
    dih = geometry_restraints.dihedral(
      sites=sites,
      angle_ideal=angle_ideal,
      weight=1./angle_esd**2,
      periodicity=signed_periodicity)
    if (angle_model is not None):
      assert approx_equal(angle_model, dih.angle_model)
    ag = flex.vec3_double(dih.gradients())
    residual_obj = residual_functor(
      restraint_type=geometry_restraints.dihedral,
      angle_ideal=dih.angle_ideal,
      weight=dih.weight,
      periodicity=signed_periodicity)
    fg = flex.vec3_double(finite_differences(sites, residual_obj)).as_double()
    ag = ag.as_double()
    scale = max(1, flex.mean(flex.abs(fg)))
    assert approx_equal(ag/scale, fg/scale)

def exercise_dihedral():
  sites = [col(site) for site in [
    (1,0,0), (0,0,0), (0,1,0), (1,0,1)]]
  angle_ideal = -45
  for flip in range(2):
    if (flip != 0):
      sites = [sites[i] for i in [0,2,1,3]]
      angle_ideal *= -1
    dih = geometry_restraints.dihedral(
      sites=sites,
      angle_ideal=angle_ideal,
      weight=1)
    assert approx_equal(dih.angle_ideal, angle_ideal)
    assert approx_equal(dih.angle_model, angle_ideal)
    assert approx_equal(dih.delta, 0)
    for i_trial in range(20):
      sites_mod = []
      for site in sites:
        shift = col([random.uniform(-.1,.1) for i in range(3)])
        sites_mod.append(site+shift)
      exercise_dihedral_core(
        sites_mod, angle_ideal, angle_esd=1, periodicity=1, angle_model=None)
  for sites,angle_ideal,angle_esd,period,angle_model in dihedral_test_data:
    sites = [col(site) for site in sites]
    exercise_dihedral_core(
      sites, angle_ideal, angle_esd, period, angle_model)

def improper_permutation(sites):
  return [sites[0], sites[1], sites[3], sites[2]]

def exercise_chirality(verbose=0):
  # monomer library: CA N CB C
  sites = [col(site) for site in [
    (27.660, 9.903, 2.078),
    (28.049, 9.675, 3.486),
    (28.183, 11.269, 1.625),
    (28.085, 8.759, 1.165)]]
  chir = geometry_restraints.chirality(
    sites=sites,
    volume_ideal=-2.48,
    both_signs=False,
    weight=1)
  assert approx_equal(chir.volume_model, -2.411548478)
  assert approx_equal(chir.residual(), 0.00468561086412)
  dih = geometry_restraints.dihedral(
    sites=improper_permutation(sites),
    angle_ideal=35.26439,
    weight=1)
  assert approx_equal(dih.angle_model, 32.2587249641)
  assert approx_equal(dih.residual(), 9.03402230782)
  if (verbose):
    dump_pdb("sites.pdb", sites)
    print("volume_ideal:", chir.volume_ideal)
    print("volume_model:", chir.volume_model)
    print("angle_ideal:", dih.angle_ideal)
    print("angle model:", dih.angle_model)
  for i_trial in range(50):
    volume_ideal = chir.volume_ideal
    for both_signs in [False, True]:
      sites_mod = []
      for site in sites:
        shift = col([random.uniform(-.5,.5) for i in range(3)])
        sites_mod.append(site+shift)
      if (both_signs): volume_ideal = abs(volume_ideal)
      c = geometry_restraints.chirality(
        sites=sites_mod,
        volume_ideal=volume_ideal,
        both_signs=both_signs,
        weight=500*1/0.2**2)
      residual_obj = residual_functor(
        restraint_type=geometry_restraints.chirality,
        volume_ideal=c.volume_ideal,
        both_signs=c.both_signs,
        weight=c.weight)
      fg = flex.vec3_double(
        finite_differences(sites_mod, residual_obj)).as_double()
      ag = flex.vec3_double(
        c.gradients()).as_double()
      scale = max(1, flex.mean(flex.abs(fg)))
      assert approx_equal(ag/scale, fg/scale)
      d = geometry_restraints.dihedral(
        sites=improper_permutation(sites_mod),
        angle_ideal=dih.angle_ideal,
        weight=750)
      ag_dih = dih.gradients()
      if (verbose and i_trial == 0 and not both_signs):
        dump_pdb("sites_mod.pdb", sites_mod)
        max_g_len = 0
        for g in ag_dih:
          max_g_len = max(max_g_len, abs(col(g)))
        for g in flex.vec3_double(fg):
          max_g_len = max(max_g_len, abs(col(g)))
        sticks = []
        for site,g in zip(improper_permutation(sites_mod),ag_dih):
          sticks.append(
            pml_stick(
              begin=site,
              end=site+col(g)/max_g_len,
              colors=[[1,0,0]]*2,
              width=0.01))
        pml_write(f=open("dih.pml", "w"), label="dih", sticks=sticks)
        sticks = []
        for site,g in zip(sites_mod,flex.vec3_double(fg)):
          sticks.append(
            pml_stick(
              begin=site,
              end=site+col(g)/max_g_len,
              colors=[[0,1,0]]*2,
              width=0.01))
        pml_write(f=open("chir.pml", "w"), label="chir", sticks=sticks)

def exercise_planarity():
  weights = flex.double([1,2,3,4])
  for points,norm in [([(1,1,0), (-1,-1,0), (-1,1,0), (1,-1,0)], (0,0,1)),
                      ([(0,1,1), (0,-1,-1), (0,-1,1), (0,1,-1)], (1,0,0)),
                      ([(1,0,1), (-1,0,-1), (-1,0,1), (1,0,-1)], (0,1,0)),
                      ([(1,1,-1), (-1,-1,1), (-1,1,-1), (1,-1,1)], (0,1,1))]:
    norm = col(norm)
    norm /= abs(norm)
    for i_trial in range(5):
      for i_pert in range(5):
        if (i_trial == 0 and i_pert == 0):
          sites = [col(v) for v in points]
          rot_norm = norm
        else:
          shift = col([random.uniform(-10,10) for i in range(3)])
          rot = sqr(euler_angles_as_matrix(
            [random.uniform(0,360) for i in range(3)]))
          rot_norm = rot * norm
          if (i_pert == 0):
            sites = [rot*col(v)+shift for v in points]
          else:
            sites = []
            for v in points:
              f = 0.1
              pert = col([(random.random()-0.5)*f for i in range(3)])
              sites.append(rot*col(v)+shift+pert)
        pl = geometry_restraints.planarity(sites=sites, weights=weights)
        n = col(pl.normal())
        gradients_analytical = pl.gradients()
        if (i_pert == 0):
          assert abs(abs(n.dot(rot_norm))-1) < 1.e-5
          assert approx_equal(pl.residual(), 0)
          for grad in gradients_analytical:
            assert approx_equal(grad, [0,0,0])
        assert approx_equal(pl.residual(), pl.lambda_min())
        residual_obj = residual_functor(
          restraint_type=geometry_restraints.planarity,
          weights=weights)
        gradients_finite = finite_differences(sites, residual_obj)
        assert approx_equal(gradients_finite, gradients_analytical)

def exercise():
  exercise_bond()
  for irexp in [1,2,3,4,5]:
    for rexp in [3,4]:
      exercise_nonbonded(
        nonbonded_type=geometry_restraints.nonbonded_prolsq,
        repulsion_function=geometry_restraints.prolsq_repulsion_function(
          irexp=irexp,
          rexp=rexp))
  for irexp in [1,2,3,4,5]:
    exercise_nonbonded(
      nonbonded_type=geometry_restraints.nonbonded_inverse_power,
      repulsion_function=geometry_restraints.inverse_power_repulsion_function(
        nonbonded_distance_cutoff=1.e20,
        irexp=irexp))
  for exponent in [1,2,3]:
    exercise_nonbonded(
      nonbonded_type=geometry_restraints.nonbonded_cos,
      repulsion_function=geometry_restraints.cos_repulsion_function(
        max_residual=13,
        exponent=exponent))
  for norm_height_at_vdw_distance in [0.1,0.2,0.3]:
    exercise_nonbonded(
      nonbonded_type=geometry_restraints.nonbonded_gaussian,
      repulsion_function=geometry_restraints.gaussian_repulsion_function(
        max_residual=12,
        norm_height_at_vdw_distance=norm_height_at_vdw_distance))
  exercise_angle()
  exercise_dihedral()
  exercise_chirality(verbose="--verbose" in sys.argv[1:])
  exercise_planarity()
  print("OK")

dihedral_test_data = [
[[ [69.141,9.390,2.567],
[70.651,9.597,2.216],
[70.898,11.039,2.558],
[69.552,11.796,2.401],
], 35,15,3,-27.623156 ],
[[ [69.094,9.368,7.421],
[70.388,10.106,7.552],
[70.526,11.172,6.519],
[71.741,11.347,6.212],
], 0,30,2,-149.24817 ],
[[ [66.200,0.138,5.659],
[67.055,-1.079,5.550],
[68.451,-0.802,4.924],
[68.274,-0.562,3.439],
], 180,15,3,-73.186553 ],
[[ [67.330,-1.591,9.213],
[67.723,-0.520,10.245],
[69.155,0.070,10.088],
[70.080,-1.137,10.323],
], 180,15,3,-63.28745 ],
[[ [64.929,-1.607,9.124],
[63.628,-2.233,9.453],
[62.488,-1.199,9.105],
[61.125,-2.031,9.143],
], 180,15,3,166.85678 ],
[[ [70.128,2.016,15.960],
[70.973,2.866,16.769],
[71.244,4.032,15.846],
[70.108,4.001,14.818],
], -25,15,3,21.478587 ],
[[ [51.224,5.278,19.162],
[49.792,5.189,19.476],
[49.560,5.913,20.813],
[50.073,5.054,21.945],
], 180,15,3,-74.662005 ],
[[ [47.467,9.582,20.922],
[46.748,10.422,21.963],
[47.382,10.282,23.330],
[46.730,9.309,24.166],
], 180,15,3,-96.181112 ],
[[ [46.748,10.422,21.963],
[47.382,10.282,23.330],
[46.730,9.309,24.166],
[45.949,8.736,25.060],
], 0,15,4,-95.23452 ],
[[ [54.391,1.288,14.581],
[55.651,1.273,15.387],
[56.467,-0.004,15.273],
[55.681,-1.123,15.821],
], 180,15,3,-63.913454 ],
[[ [67.779,9.293,11.519],
[68.887,10.169,12.058],
[70.267,9.371,12.083],
[71.437,10.417,12.273],
], 180,15,3,164.95936 ],
[[ [74.272,17.569,20.340],
[74.302,18.830,21.056],
[73.048,19.608,20.974],
[72.642,20.465,21.916],
], 0,15,4,152.18051 ],
[[ [67.020,12.001,18.091],
[66.093,10.976,18.685],
[66.532,9.757,19.391],
[65.579,8.603,19.413],
], 60,15,3,-155.18126 ],
[[ [59.525,6.469,10.069],
[60.307,5.499,9.152],
[60.931,6.381,7.983],
[61.840,5.442,7.183],
], 180,15,3,173.24322 ],
[[ [54.325,-2.812,2.710],
[54.755,-1.497,2.220],
[54.490,-1.281,0.739],
[54.841,-2.246,-0.385],
], 180,15,3,-48.495407 ],
[[ [54.449,2.584,5.084],
[55.475,3.162,5.951],
[56.953,2.788,5.575],
[57.159,1.313,5.585],
], 180,15,3,-58.143716 ],
[[ [55.273,5.394,6.776],
[55.070,6.795,6.699],
[53.864,7.253,7.664],
[52.594,6.446,7.313],
], 180,15,3,-55.174556 ],
[[ [55.070,6.795,6.699],
[53.864,7.253,7.664],
[52.594,6.446,7.313],
[52.327,5.272,8.025],
], 90,20,2,93.499752 ],
[[ [59.901,2.524,-2.158],
[60.037,1.248,-1.326],
[60.057,1.643,0.158],
[60.110,0.416,1.003],
], 180,15,3,176.77171 ],
[[ [34.183,3.032,3.282],
[34.441,2.116,4.414],
[35.660,1.339,3.940],
[35.767,1.558,2.468],
], -25,15,3,-16.279611 ],
[[ [35.660,1.339,3.940],
[35.767,1.558,2.468],
[35.251,2.993,2.262],
[34.183,3.032,3.282],
], -30,15,3,-36.045859 ],
[[ [18.701,2.935,18.700],
[17.376,2.652,19.264],
[17.387,1.554,20.356],
[17.769,0.234,19.673],
], 180,15,3,-66.444499 ],
[[ [17.350,2.078,24.829],
[18.086,1.833,26.121],
[17.850,2.399,27.443],
[17.154,2.723,28.532],
], 0,15,4,53.638935 ],
[[ [15.882,6.548,24.854],
[15.228,7.545,25.766],
[16.213,8.309,26.673],
[17.715,8.339,26.304],
], 180,15,3,18.478984 ],
[[ [14.761,12.218,19.123],
[15.765,11.584,18.129],
[17.272,11.907,18.285],
[17.609,13.327,17.950],
], 180,15,3,-69.045664 ],
[[ [15.999,13.559,4.210],
[17.156,14.185,3.544],
[16.653,14.774,2.121],
[15.890,13.732,1.302],
], 180,15,3,-50.961962 ],
[[ [19.166,15.041,4.775],
[19.811,16.046,5.668],
[19.418,17.518,5.404],
[19.549,18.202,4.065],
], 180,15,3,-54.972852 ],
[[ [19.418,17.518,5.404],
[19.549,18.202,4.065],
[20.987,18.306,3.555],
[21.515,19.426,3.487],
], 0,30,2,-111.91728 ],
[[ [21.126,16.443,21.482],
[21.298,16.312,20.045],
[20.406,15.288,19.376],
[20.706,13.843,19.714],
], 180,15,3,-68.764347 ],
[[ [23.143,21.668,10.296],
[22.933,22.521,9.149],
[22.565,23.890,9.595],
[22.302,24.797,8.392],
], 180,15,3,-178.47172 ],
]

if (__name__ == "__main__"):
  exercise()


from pprint import pprint
import json
from copy import deepcopy
file_open = open('data.csv','r')
file_read = file_open.read()
count = 0
timetable=[]
dictionary={
        'day':'RK',
        'prof':'RK',
        'subject':'RK',
        'subject_number':'RK',
        'time':'RK',
        'room':'RK',
}
counter=0
new_word=''
word_list=[]
for letter in file_read:
	if(letter==',' or letter=="\n"):
		word_list.append(new_word)
		new_word=''

	else:
		new_word=new_word+letter

with open('data.json','w') as file:
		timelist=[]
		for word in word_list:
			if(count==0):
				dictionary['day']=word
			if(count==1):
				dictionary['prof']=word
			if(count==2):
				dictionary['subject']=word
			if(count==3):
				dictionary['subject_number']=word
			if(count==4):
				dictionary['time']=word
			if(count==5):
				dictionary['room']=word
				count=-1
				timetable.append(deepcopy(dictionary))
			count=count+1
		json.dump(timetable, file)


import os,traceback,requests,time,json
from datetime import datetime,timedelta
from tqdm import tqdm
import settings
from saveLastChaeck import saveLastCheck
from .parser import ParseOuestfrance
from HttpRequest.uploader import AsyncKafkaTopicProducer
cpath =os.path.dirname(__file__) or "."
kafkaTopicName = settings.KAFKA_OUESTFRANCE
commanPattern =settings.KAFKA_COMMON_PATTERN
website= "ouestfrance-immo.com"
commonIdUpdate = f"activeid-{website}"
class OuestFranceScraper:
    def __init__(self,paremeter,timeout = 5) -> None:
        self.logfile = open(f"{cpath}/error.log",'a')
        self.timeout = timeout
        self.headers = {
            "Connection": "keep-alive",
            "sec-ch-ua": '"Chromium";v="106", "Google Chrome";v="106", "Not;A=Brand";v="99"',
            "Accept": "application/json, text/plain, */*",
            "sec-ch-ua-mobile": "?0",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
            "sec-ch-ua-platform": "Windows",
            "Origin": "https://www.ouestfrance-immo.com",
            "Sec-Fetch-Site": "same-site",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Dest": "empty",
            "Referer": "https://www.ouestfrance-immo.com/",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
        }
        self.paremeter = paremeter
        self.size=100
        self.producer = AsyncKafkaTopicProducer()
        self.searchurl = "https://www-api.ouestfrance-immo.com/api/annonces/"
        self.session = requests.Session()
    def init_session(self):
        self.session.close()
        self.session = requests.Session()
    def fetch(self,url,method = "get",sid=0,retry=0,**kwargs):
        kwargs['headers'] = self.headers
        try:
            if method=="post":
                r = self.session.post(url,timeout=self.timeout,**kwargs)
            else:
                r = self.session.get(url,timeout=self.timeout,**kwargs)
            # print(f"{r.status_code} : response status")
        except Exception as e:
            traceback.print_exc(file=self.logfile)
            print(e)
            time.sleep(1)
            self.init_session()
            if retry<10:
                retry+=1
                return self.fetch(url,method=method,sid=sid,retry=retry,**kwargs)
            else:return None
        if r.status_code not in [200,404,400]:
            print(url)
            print(method)
            print(kwargs)
            self.init_session(sid=sid)
            if retry<10:
                retry+=1
                return self.fetch(url,method=method,sid=sid,retry=retry,**kwargs)
            else:return None
        return r
    def save(self,adslist,onlyid=False):
        if onlyid:
            # now = datetime.now()
            ads = [ParseOuestfrance(ad) for ad in adslist]
            # ads = [{"id":"quest-"+str(ad.get("id")), "last_checked": now.isoformat(),"available":True} for ad in adslist]
            self.producer.PushDataList_v1(commonIdUpdate,ads)
        else:
            self.producer.PushDataList(kafkaTopicName,adslist)
            adslist = [ParseOuestfrance(ad) for ad in adslist]
            self.producer.PushDataList(commanPattern,adslist)
        # data = ""
        # for ad in adslist:
        #     data += json.dumps(ad)+"\n"
        # with open(f"{cpath}/out.json","a") as file:
        #     file.write(data)
        # parseAdList = [ParseLogicImmo(ad) for ad in adslist]
        # self.producer.PushDataList(commonTopicName,parseAdList)
        # self.producer.PushDataList(nortifyTopic,parseAdList)
    def Crawlparam(self,param,allPage = True,first=False,save=True,onlyid=False):
        # input()
        response = self.fetch(self.searchurl, method = "get", params=param,)
        if not response:
            return 0
        # print(response.status_code,"+++++++++")
        res = response.json()
        pagesize = param["limit"]
        ads = res['data']
        fetchedads = ads
        if save:self.save(fetchedads,onlyid=onlyid)
        if first:
            return fetchedads[0]
        if allPage:
            totalpage = res["count"]
            # input()
            print(len(ads),"_total ads+++++++++++++")
            totalpage = int(totalpage/pagesize)+1
            print(f"total page {totalpage}")
            for i in tqdm(range(2,totalpage)):
                param["page"] = i
                self.Crawlparam(param,allPage=False,onlyid=onlyid)
                # break
        else:
            return fetchedads
    def getTotalResult(self,dic):
        dic = {
            **dic,
            "limit":1
        }
        response = self.fetch(self.searchurl, method = "get", params=dic)
        res = response.json()
        count = res["count"]
        return count
    def getMax(self,dic,max="prix"):
        dic = {
            **dic,
            "tri":f"{max}_desc",
            f"{max}_min":1,
            "limit":1
        }
        response = self.fetch(self.searchurl, method = "get", params=dic)
        res = response.json()
        maxval = res["data"][0][max]
        return maxval
    def genFilter(self,parameter={},onlyid=False,low="prix_min",max="prix_max"):
        if parameter:
            dic = parameter.copy()
        else:dic = self.paremeter.copy()
        # totalresult = getTotalResult(session,dic)
        # acres = totalresult
        # dic['recherche[produit]']=typ
        if low=="prix_min":
            # iniinterval =[98976, 98989]
            iniinterval = [0,1000]
            maxprize = self.getMax(dic.copy())
        else:
            maxprize = self.getMax(dic.copy(),max="surface")
            iniinterval =[0,1]
        # maxprize = getMaxPrize(session,dic)
        maxresult = 4999
        filterurllist = ""
        finalresult = 0
        nooffilter = 0
        while iniinterval[1]<=maxprize:
            dic[low],dic[max] = iniinterval
            totalresult = self.getTotalResult(dic)
            if totalresult <= maxresult and totalresult>0:
                # print("condition is stisfy going to next interval",totalresult)
                # print(iniinterval,">apending")
                filterurllist += json.dumps(iniinterval) + "/n/:"
                # FetchFilter(dic,onlyid,session2)
                self.CrawlOuestfrance(dic.copy(),onlyid)
                print("going to next")
                iniinterval[0] = iniinterval[1]+1
                iniinterval[1] = iniinterval[0]+int(iniinterval[0]/2)
                finalresult +=totalresult
                nooffilter +=1
            # elif acres-finalresult<4800 and acres-finalresult>0:
            #     print(maxresult, acres , maxresult-acres)
            #     # input()
            #     # print("elif 1")
            #     last = 10
            #     iniinterval[1] = maxprize
            elif totalresult == 0:
                # print("elif 1",iniinterval)
                last = 10
                iniinterval[0] = iniinterval[1]
                iniinterval[1] = iniinterval[0] + (int(iniinterval[1]/last) or 1)
                # iniinterval[0] = iniinterval[1]
                # iniinterval[1] +=1
            elif iniinterval[1]-iniinterval[0] <=2 and totalresult>maxresult and low=="prix_min":
                finalresult +=totalresult
                self.genFilter(dic.copy(),onlyid,"surface_min","surface_max")
                iniinterval[0] = iniinterval[1]
                iniinterval[1]+=1
            elif totalresult>maxresult:
                # print("elif 2",iniinterval)
                last = -5
                dif = iniinterval[1]-iniinterval[0]
                iniinterval[1] = iniinterval[1] + int(dif/-2)
                if iniinterval[0]>iniinterval[1]:
                    iniinterval[1] = iniinterval[0]+1
            # retrydic[iniinterval[0]] +=1
            print(f"\r{totalresult}-{maxresult}:::: of  {finalresult}==>{iniinterval} {maxprize} {low} no of filter",end="")
            # print(totalresult,"-",maxresult,"::::",acres ," of ", finalresult,"==>",iniinterval)
        self.CrawlOuestfrance(dic.copy(),onlyid)

    def getLastUpdate(self):
        try:
            with open(f"{cpath}/lastUpdate.json",'r') as file:
                data = json.load(file)
            return data
        except:
            return {}
    def getTimeStamp(self,strtime):
        formate = '%Y-%m-%d %H:%M:%S'
        # 2022-06-19 05:26:55
        try:
            t = datetime.strptime(strtime,formate)
        except:return strtime
        return t.timestamp()
    def getUpdateDicFromAd(self,ad):
        nowtime  = datetime.now()
        upd={
                "timestamp": nowtime.timestamp(),
                "lastupdate": self.getTimeStamp(ad.get('date_deb_aff')),
                "Rlastupdate": ad.get('date_deb_aff'),
                "created": self.getTimeStamp(ad.get('date_creation')),
                "rcreated": ad.get('date_creation'),
            }
        return upd
    def getAdStatus(self,id):
        url = f"{self.searchurl}{id}"
        r = self.fetch(url)
        data = r.json()
        if data["data"]:return True
        else:False
    def getlatestAd(self):
        param = self.paremeter
        param["limit"] = 1
        data = self.Crawlparam(param,allPage=False,first=True)
        param["limit"] = self.size
        return data
    def createNewUpdate(self,latestad=None):
        lastupd = self.getLastUpdate()
        if not latestad:
            latestad = self.getlatestAd()
        lastupd=self.getUpdateDicFromAd(latestad)
        with open(f"{cpath}/lastUpdate.json",'w') as file:
            file.write(json.dumps(lastupd))
    def CrawlOuestfrance(self,param=None,onlyid=False):
        if not param:
            param = self.paremeter.copy()
        if not onlyid:self.createNewUpdate(latestad=None)
        self.Crawlparam(param,onlyid=onlyid)
    def updateLatestAd(self):
        updates = self.getLastUpdate()
        if updates:
            param = self.paremeter
            updated = False
            first = True
            page =1
            adcount = 0
            page = 1
            while not updated:
                param["page"] = page
                # res = self.fetch(searchurl, method = "post", json=param)
                ads = self.Crawlparam(param,False,False,False)
                updatedads = []
                updatetimestamp = updates["lastupdate"]
                for ad in ads:
                    # ad = self.GetAdInfo(ad['id'])
                    adtimestamp = self.getTimeStamp(ad["date_deb_aff"])
                    print(f"   {adtimestamp}> {updatetimestamp}====>",adtimestamp>updatetimestamp)
                    if adtimestamp>updatetimestamp:
                        if first:
                            self.createNewUpdate(ad)
                            first=False
                        updatedads.append(ad)
                        adcount+=1
                    else:
                        print(f"{adcount} new ads scraped ")
                        updated = True
                        break
                self.save(updatedads)
                page+=1
        else:
            print("there is no update available")
def CheckId(id):
    ob = OuestFranceScraper(data)
    found= ob.getAdStatus(id)
    ob.__del__()
    return found
def main_scraper(payload,update=False):
    adtype = payload.get("real_state_type")
    """
    typIds
        201 : for vente
        214 : for location
        216 : for location
    """
    data = {
        "limit":200,
        "tri":"date_desc",
        # "typIds":"201,214,216"
    }
    if adtype == "Updated/Latest Ads" or update:
        ob = OuestFranceScraper(data,timeout=10)
        print("updateing latedst ads")
        ob.updateLatestAd()
    elif payload.get("real_state_type") == "deletedCheck":
        rescrapActiveId()
    else:
        ob = OuestFranceScraper(data,timeout=30)
        ob.genFilter()
def rescrapActiveIdbyType():
    """
    typIds
        201 : for vente
        207 : for vente
        208 : for vente
        209 : for vente
        210 : for vente
        213 : for location
        214 : for location
        216 : for location
    """
    data = {
        "limit":208,
        "tri":"date_desc",
        # "typIds":"201,214,216"
    }
    ob = OuestFranceScraper(data,timeout=30)
    ob.genFilter(data,onlyid=True)
def rescrapActiveId():
    nowtime = datetime.now()
    nowtime = nowtime - timedelta(hours=1)
    rescrapActiveIdbyType()
    # main("buy",True)
    # with concurrent.futures.ThreadPoolExecutor(max_workers=10) as excuter:
    #     futures = [excuter.submit(rescrapActiveIdbyType, i) for i in ["rental","sale"]]
    #     for f in futures:print(f)
    # rescrapActiveIdbyType("rental")
    # print("complited")
    saveLastCheck(website,nowtime.isoformat())
if __name__ == "__main__":
    """
    typIds
        201 : for vente
        214 : for location
        216 : for location
    """
    data = {
        "limit":200,
        "tri":"date_desc",
        # "typIds":"201,214,216"
    }
    ob = OuestFranceScraper(data,timeout=30)
    ob.CrawlOuestfrance()
    # ob.updateLatestAd()

#parms ?limit=2&tri=date_decroissant&balcon=1&criteres_or=1&hasPhoto=1&idslieu=33540&isNeuf=0&limit=10&terrasse=1&tra=V&typIds=201&veranda=1&prix_min=199290&page=1&prix_max=200000


# python3
import socket
import datetime
import re

def request(client):
    buf = client.recv(1024)
    print(buf)
    if buf:
        res = re.findall(r'GET (/.*)HTTP', buf.decode())[0]
    # if res == '/favicon.ico':
    #     return
    else:
        res = 'what?'
    print(res)

    day = datetime.datetime.now()
    day_str = str(day).split('.')[0].encode()
    client.send('HTTP/1.1 200 OK\r\n\r\n'.encode('utf-8') + '
hello world!
'.encode('utf-8') + day_str +
                '
访问链接：'.encode('gbk') + res.encode())
    return buf or 'nothong'

def main(ip='0.0.0.0', port=8000):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind((ip, port))
    sock.listen(5)
    cout = 1
    while cout > 0:
        con, add = sock.accept()
        request(con)
        cout += 1
        con.close()

if __name__ == '__main__':
    main(port=80)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging

import networkx as nx

log = logging.getLogger()

def _find_src_nodes(Gt):
    src_list = []
    for node in Gt.nodes():
        predecessors = list(Gt.predecessors(node))
        if not predecessors:
            src_list.append(node)
    return src_list

def _find_dst_nodes(Gt):
    dst_list = []
    for node in Gt.nodes():
        successors = list(Gt.successors(node))
        if not successors:
            dst_list.append(node)
    return dst_list

def has_simple_path(G, src, dst):
    all_path = list(nx.all_simple_paths(G, src, dst))
    return len(all_path) > 0

def find_all_simple_path(G):
    """
    Generate all simple paths in the graph G from source to target.
    Args:
        G (nx.DiGraph)
    Returns:
        src_list (list of str): source nodes
        dst_list (list of str): dstination nodes
        all_paths (list of list(str)): all simple paths of nodes' names
    """
    src_list = _find_src_nodes(G)
    dst_list = _find_dst_nodes(G)
    log.info('find all path from {} to {}'.format(src_list, dst_list))
    all_paths = []
    for src in src_list:
        for dst in dst_list:
            paths = list(nx.all_simple_paths(G, src, dst))
            log.info('found path: {}'.format(paths))
            all_paths += paths
    return src_list, dst_list, all_paths


import re
from io import BytesIO

import aiohttp
import pytesseract
from PIL import Image, ImageEnhance
from aiohttp import payload
from asgiref.sync import sync_to_async

async def get_banana_num(stories_url: str) -> int:
    stories_data = await get_stories_data(stories_url)
    banana_num = await find_banana_num(stories_data) if stories_data else None
    return banana_num

async def get_stories_data(stories_url: str) -> payload:
    async with aiohttp.ClientSession() as session:
        async with session.get(stories_url) as response:
            return await response.read()

@sync_to_async
def find_banana_num(stories_data: payload):
    image_data = BytesIO(stories_data)
    img: Image.Image = Image.open(image_data)

    enhancer = ImageEnhance.Sharpness(img)
    img = enhancer.enhance(3)

    img_text = pytesseract.image_to_string(img, lang='eng')
    pattern = re.compile(r"#\d+")
    found_strings = re.findall(pattern=pattern, string=img_text)

    return int(found_strings[-1][1:]) if found_strings else None


import os
import asyncio
import logging
from aiogram import Bot
from aiogram.dispatcher import Dispatcher
from aiogram.utils.executor import start_webhook

BOT_TOKEN = os.environ.get('TOKEN')
WEBHOOK_HOST = "https://android-expert-bot.herokuapp.com"
WEBHOOK_PATH = '/webhook'
WEBHOOK_URL = f"{WEBHOOK_HOST}{WEBHOOK_PATH}"

WEBAPP_HOST = '0.0.0.0'
WEBAPP_PORT = os.environ.get('PORT')

bot = Bot(BOT_TOKEN, parse_mode="HTML")
dp = Dispatcher(bot)

async def on_startup(dp):
    await bot.set_webhook(WEBHOOK_URL)

async def on_shutdown(dp):
    logging.warning('Shutting down..')
    await bot.delete_webhook()
    await dp.storage.close()
    await dp.storage.wait_closed()

    logging.warning('Bye!')

if __name__ == '__main__':
    from handlers import dp

    start_webhook(
        dispatcher=dp,
        webhook_path=WEBHOOK_PATH,
        on_startup=on_startup,
        on_shutdown=on_shutdown,
        host=WEBAPP_HOST,
        port=WEBAPP_PORT)


#import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#import dataset
URL="https://raw.githubusercontent.com/AdiPersonalWorks/Random/master/student_scores%20-%20student_scores.csv"
df=pd.read_csv(URL)
print("Data imported successfully")
#inspect the data
print(df.shape)
print(df.head())
print(df.describe())
Scores=df["Scores"]
Hours=df["Hours"]
avg_scores=Scores.mean()
avg_hours=Hours.mean()
print("On Average a student studying:", avg_hours, " will attain an average score of :", avg_scores)
#Correlation between variablesavg_hours
df.corr()
print(" Correlation between hours and scores is 0.976191")

#Graphical Visualization of Data
# Plotting the distribution of scores
df.plot(x='Hours', y='Scores',style='o')
plt.title('Hours vs Percentage Score')
plt.grid(True)
plt.xlabel('Hours Studied')
plt.ylabel('Percentage Score')
plt.show()
#Inference
print(" Positive linear relation between the number of hours studied and percentage of score.")

#Preparing Data for Modelling
X = df.iloc[:, :-1].values
y = df.iloc[:, 1].values
#Training the model
from sklearn.model_selection import train_test_split
##Split the data into train and test model
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.20,random_state=0)

#Fit the model
from sklearn.linear_model import LinearRegression
linreg = LinearRegression()
linreg.fit(X_train, y_train)
print("Training Complete")
print('Intercept: ',linreg.intercept_)

# Plotting the regression line
line = linreg.coef_*X+linreg.intercept_

# Plotting for the test data
plt.scatter(X, y)
plt.plot(X, line,color = 'green')
plt.show()
#Predict the model
print(X_test) # Testing data - In Hours
y_pred = linreg.predict(X_test) # Predicting the scores
# Comparing Actual vs Predicted
df1 = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})
print(df1)
# testing with  own data
hours = 9.25
own_pred = linreg.predict([[hours]])
print("No of Hours of Study = {}".format(hours))
print("Predicted Score = {}".format(own_pred[0]))

#Evaluating the model
from sklearn import metrics
print('Mean Absolute Error:',
      metrics.mean_absolute_error(y_test, y_pred))
print("Residual sum of squares(MSE): %.2f" % np.mean((y_pred - y_test) ** 2))
MSE=np.mean((y_pred - y_test) ** 2)
print("Root mean squares error (RMSE): %.2f" % np.sqrt(MSE))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % linreg.score(X_test, y_test))
print("Mean absolute error: %.2f" % np.mean(np.absolute(y_pred - y_test)))
print("R2-score: %.2f" % metrics.r2_score(y_pred ,y_test))
print(linreg.score(X_test,y_test)*100,'% Prediction Accuracy')


from sqlalchemy import Column, ForeignKey, Integer, String, Text
from sqlalchemy.orm import relationship

from core.db import Base

class FlashCard(Base):
    __tablename__ = "api_flash_cards"

    id = Column(Integer, primary_key=True, index=True, unique=True)
    title = Column(String(255))
    front = Column(Text, nullable=True)
    front_image_url = Column(String(255), nullable=True)
    back = Column(Text, nullable=True)
    back_image_url = Column(String(255), nullable=True)

    theme = Column(Integer, ForeignKey('api_themes.id'))
    theme_id = relationship("Theme")

    user = Column(Integer, ForeignKey('api_users.id'))
    user_id = relationship("User")