from __future__ import print_function
import pysb.bng
import numpy
import sympy
import re
import ctypes
import csv
import scipy.interpolate
import sys
from pysundials import cvode

# Thee set of functions set up the system for annealing runs
# and provide the runner function as input to annealing

def spinner(i):
    spin = ("|", "/","-", "\\")
    print("\r[%s] %d"%(spin[i%4],i), end=' ')
    sys.stdout.flush()

# reltol of 1.0e-3, relative error of ~1%. abstol of 1.0e-3, enough for values that oscillate in the hundreds to thousands
def odeinit(model, reltol=1.0e-3, abstol=1.0e-3, nsteps = 1000, itermaxstep = None):
    '''
    must be run to set up the environment for annealing with pysundials
    '''
    # Generate equations
    pysb.bng.generate_equations(model)
    # Get the size of the ODE array
    odesize = len(model.odes)

    # init the arrays we need
    yzero = numpy.zeros(odesize)  #initial values for yzero

    # assign the initial conditions
    for cplxptrn, ic_param in model.initial_conditions:
        speci = model.get_species_index(cplxptrn)
        yzero[speci] = ic_param.value

    # initialize y with the yzero values
    y = cvode.NVector(yzero)

    # make a dict of ydot functions. notice the functions are in this namespace.
    # replace the kxxxx constants with elements from the params array
    rhs_exprs = []
    for i in range(0,odesize):
        # first get the function string from sympy, replace the the "sN" with y[N]
        tempstring = re.sub(r's(\d+)', lambda m: 'y[%s]'%(int(m.group(1))), str(model.odes[i]))
        # now replace the constants with 'p' array names; cycle through the whole list
        #for j in range(0, numparams):
        #    tempstring = re.sub('(?<![A-Za-z0-9_])%s(?![A-Za-z0-9_])'%(model.parameters[j].name),'p[%d]'%(j), tempstring)
        for j, parameter in enumerate(model.parameters):
            tempstring = re.sub('(?<![A-Za-z0-9_])%s(?![A-Za-z0-9_])' % parameter.name, 'p[%d]' % j, tempstring)
        # make a list of compiled rhs expressions which will be run by the integrator
        # use the ydots to build the function for analysis
        # (second arg is the "filename", useful for exception/debug output)
        rhs_exprs.append(compile(tempstring, '<ydot[%s]>' % i, 'eval'))

    # Create the structure to hold the parameters when calling the function
    # This results in a generic "p" array
    numparams = len(model.parameters)
    class UserData(ctypes.Structure):
        _fields_ = [('p', cvode.realtype*numparams)] # parameters
    PUserData = ctypes.POINTER(UserData)
    data = UserData()

    data.p[:] = [p.value for p in model.parameters]
    paramarray = numpy.array([p.value for p in model.parameters])

    # allocate the "p" array as a pointer array that can be called by sundials "f" as needed
    def f(t, y, ydot, f_data):
        data = ctypes.cast(f_data, PUserData).contents
        rhs_locals = {'y': y, 'p': data.p}
        for i in range(0, len(model.odes)):
            ydot[i] = eval(rhs_exprs[i], rhs_locals)
        return 0

    # initialize the cvode memory object, use BDF and Newton for stiff systems
    cvode_mem = cvode.CVodeCreate(cvode.CV_BDF, cvode.CV_NEWTON)
    # allocate the cvode memory as needed, pass the function and the init ys
    cvode.CVodeMalloc(cvode_mem, f, 0.0, y, cvode.CV_SS, reltol, abstol)
    # point the parameters to the correct array
    # if the params are changed later this does not need to be reassigned (???)
    cvode.CVodeSetFdata(cvode_mem, ctypes.pointer(data))
    # link integrator with linear solver
    cvode.CVDense(cvode_mem, odesize)
    #stepsize
    if itermaxstep != None:
        cvode.CVodeSetMaxStep(cvode_mem, itermaxstep)

    #list of outputs
    xout = numpy.zeros(nsteps)
    yout = numpy.zeros([nsteps, odesize])

    #initialize the arrays
    #print("Initial parameter values:", y)
    xout[0] = 0.0 #CHANGE IF NEEDED
    #first step in yout
    for i in range(0, odesize):
        yout[0][i] = y[i]

    return [f, rhs_exprs, y, odesize, data, xout, yout, nsteps, cvode_mem, yzero, reltol, abstol], paramarray

def odesolve(model, tfinal, envlist, params, useparams=None, tinit = 0.0, ic=True):
    '''
    the ODE equation solver tailored to work with the annealing algorithm
    model: the model object
    envlist: the list returned from annlinit
    params: the list of parameters that are being optimized with annealing
    useparams: the parameter number to which params[i] corresponds
    tinit: initial time
    reltol: relative tolerance
    abstol: absolute tolerance
    ic: reinitialize initial conditions to a value in params or useparams
    '''
    (f, rhs_exprs, y, odesize, data, xout, yout, nsteps, cvode_mem, yzero, reltol, abstol) = envlist

    #set the initial values and params in each run
    #all parameters are used in annealing. initial conditions are not, here
    if useparams is None:
        for i in range(len(params)):
            data.p[i] = params[i]
    else:
        #only a subset of parameters are used for annealing
        for i in range(len(useparams)):
            #print("changing parameter", model.parameters[useparams[i]],"data.p", data.p[useparams[i]],"to", params[i])
            data.p[useparams[i]] = params[i]

    # FIXME:
    # update yzero if initial conditions are being modified as part of the parameters
    # did it this way b/c yzero and data.p may not always be modified at the same time
    # the params list should NOT contain the initial conditions if they are not
    # to be used in the annealing... so this is a hack based on the fact that the
    # initial conditions are contained as part of the model.parameters list.
    #
    if ic is True:
        for cplxptrn, ic_param in model.initial_conditions:
            speci = model.get_species_index(cplxptrn)
            yzero[speci] = ic_param.value

    #reset initial concentrations
    y = cvode.NVector(yzero)

    # Reinitialize the memory allocations, DOES NOT REALLOCATE
    cvode.CVodeReInit(cvode_mem, f, 0.0, y, cvode.CV_SS, reltol, abstol)

    tadd = tfinal/nsteps

    t = cvode.realtype(tinit)
    tout = tinit + tadd

    #print("Beginning integration")
    #print("TINIT:", tinit, "TFINAL:", tfinal, "TADD:", tadd, "ODESIZE:", odesize)
    #print("Integrating Parameters:\n", params)
    #print("y0:", yzero)

    for step in range(1, nsteps):
        ret = cvode.CVode(cvode_mem, tout, y, ctypes.byref(t), cvode.CV_NORMAL)
        if ret !=0:
            print("CVODE ERROR %i"%(ret))
            break

        xout[step]= tout
        for i in range(0, odesize):
            yout[step][i] = y[i]

        # increase the time counter
        tout += tadd
    #print("Integration finished")

    #now deal with observables
    #obs_names = [name for name, rp in model.observable_patterns]
    yobs = numpy.zeros([len(model.observables), nsteps])

    #sum up the correct entities
    for i, obs in enumerate(model.observables):
        coeffs = obs.coefficients
        specs  = obs.species
        yobs[i] = (yout[:, specs] * coeffs).sum(1)

    #merge the x and y arrays for easy analysis
    xyobs = numpy.vstack((xout, yobs))

    return (xyobs,xout,yout, yobs)

def compare_data(xparray, simarray, xspairlist, vardata=False):
    """Compares two arrays of different size and returns the X^2 between them.
    Uses the X axis as the unit to re-grid both arrays.
    xparray: experimental data
    xparrayaxis: which axis of xparray to use for simulation
    simarray: simulation data
    simarrayaxis: which axis of simarray to use for simulation
    """
    # this expects arrays of the form array([time, measurement1, measurement2, ...])
    # the time is assumed to be roughly the same for both and the
    # shortest time will be taken as reference to regrid the data
    # the regridding is done using a b-spline interpolation
    # xparrayvar shuold be the variances at every time point
    #
    # FIXME FIXME FIXME FIXME
    # This prob should figure out the overlap of the two arrays and
    # get a spline of the overlap. For now just assume the simarray domain
    # is bigger than the xparray. FIXME FIXME FIXME
    #rngmax = min(xparray[0].max(), simarray[0].max())
    #rngmin = round(rngmin, -1)
    #rngmax = round(rngmax, -1)
    #print("Time overlap range:", rngmin,"to", rngmax)

    ipsimarray = numpy.zeros(xparray.shape[1])
    objout = []

    for i in range(len(xspairlist)):
        # create a b-spline of the sim data and fit it to desired range
        # import code
        # code.interact(local=locals())

        #some error checking
        #print("xspairlist length:", len(xspairlist[i]))
        #print("xspairlist element type:", type(xspairlist[i]))
        #print("xspairlist[i] elements:", xspairlist[i][0], xspairlist[i][1])
        assert type(xspairlist[i]) is tuple
        assert len(xspairlist[i]) == 2

        xparrayaxis = xspairlist[i][0]
        simarrayaxis = xspairlist[i][1]

        tck = scipy.interpolate.splrep(simarray[0], simarray[simarrayaxis])
        ipsimarray = scipy.interpolate.splev(xparray[0], tck) #xp x-coordinate values to extract from y splines

        # we now have x and y axis for the points in the model array
        # calculate the objective function
        #                        1
        # obj(t, params) = -------------(S_sim(t,params)-S_exp(t))^2
        #                  2*sigma_exp^2

        diffarray = ipsimarray - xparray[xparrayaxis]
        diffsqarray = diffarray * diffarray

        if vardata is True:
            #print("using XP VAR",xparrayaxis+1)
            xparrayvar = xparray[xparrayaxis+1] # variance data provided in xparray in next column
        else:
            # assume a default variance
            xparrayvar = numpy.ones(xparray.shape[1])
            xparrayvar = xparray[xparrayaxis]*.1 # within 10%? FIXME: check w will about this
            xparrayvar = xparrayvar * xparrayvar

        xparrayvar = xparrayvar*2.0
        numpy.seterr(divide='ignore') # FIXME: added to remove the warnings... use caution!!
        objarray = diffsqarray / xparrayvar

        # check for inf in objarray, they creep up when there are near zero or zero values in xparrayvar
        for i in range(len(objarray)):
            if numpy.isinf(objarray[i]) or numpy.isnan(objarray[i]):
                #print("CORRECTING NAN OR INF. IN ARRAY")
                #print(objarray)
                objarray[i] = 1e-100 #zero enough

        #import code
        #code.interact(local=locals())

        objout.append(objarray.sum())
        #print("OBJOUT(%d,%d):%f  OBJOUT(CUM):%f"%(xparrayaxis, simarrayaxis, objarray.sum(), objout))
    #print("OBJOUT(total):", objout)
    return numpy.asarray(objout)

def getlog(sobolarr, params, omag=1, useparams=[], usemag=None):
    # map a set of sobol pseudo-random numbers to a range for parameter evaluation
    # sobol: sobol number array of the appropriate length
    # params: array of parameters
    # omag: order of magnitude over which params should be sampled. this is effectively 3 orders of magnitude when omag=1
    #

    sobprmarr = numpy.zeros_like(sobolarr)
    ub = numpy.zeros(len(params))
    lb = numpy.zeros(len(params))
    # set upper/lower bounds for generic problem
    for i in range(len(params)):
        if i in useparams:
            ub[i] = params[i] * pow(10,usemag)
            lb[i] = params[i] / pow(10,usemag)
        else:
            ub[i] = params[i] * pow(10, omag)
            lb[i] = params[i] / pow(10, omag)

    # see  for more info http://en.wikipedia.org/wiki/Exponential_family
    sobprmarr = lb*(ub/lb)**sobolarr # map the [0..1] sobol array to values sampled over their omags

    # sobprmarr is the N x len(params) array for sobol analysis
    return sobprmarr

def getlin(sobolarr, params, CV =.25, useparams=[], useCV=None):
    """ map a set of sobol pseudo-random numbers to a range for parameter evaluation

    sobol: sobol number array of the appropriate length
    params: array of parameters
    stdev: standard deviation for parameters, this assumes it is unknown for the sampling

    function maps the sobol (or any random) [0:1) array linearly to mean-2sigma < x < mean + 2sigma

    CV is the coefficient of variance, CV = sigma/mean
    """

    sobprmarr = numpy.zeros_like(sobolarr)

    ub = numpy.zeros(len(params))
    lb = numpy.zeros(len(params))
    # set upper/lower bounds for generic problem
    for i in range(len(params)):
        if i in useparams:
            ub[i] = params[i] + params[i]*useCV
            lb[i] = params[i] - params[i]*useCV
        else:
            ub[i] = params[i] + params[i]*CV
            lb[i] = params[i] - params[i]*CV

    # sobprmarr = (sobolarr*(ub-lb)) + lb #map the [0..1] sobol array to the values for integration
    if len(sobprmarr.shape) == 1:
        sobprmarr = (sobolarr*(ub-lb)) + lb
    elif len(sobprmarr.shape) == 2:
        for i in range(sobprmarr.shape[0]):
            sobprmarr[i] = (sobolarr[i]*(ub-lb)) + lb
    else:
        print("array shape not allowed... ")

    # sobprmarr is the N x len(params) array for sobol analysis
    # lb is the lower bound of params
    # ub is the upper bound of params
    return sobprmarr

def genCmtx(sobmtxA, sobmtxB):
    """when passing the quasi-random sobol-treated A and B matrixes, this function iterates over all the possibilities
    and returns the C matrix for simulations.
    See e.g. Saltelli, Ratto, Andres, Campolongo, Cariboni, Gatelli, Saisana, Tarantola Global Sensitivity Analysis"""

    nparams = sobmtxA.shape[1] # shape 1 should be the number of params

    # allocate the space for the C matrix
    sobmtxC = numpy.array([sobmtxB]*nparams)

    # Now we have nparams copies of sobmtxB. replace the i_th column of sobmtxC with the i_th column of sobmtxA
    for i in range(nparams):
        sobmtxC[i,:,i] = sobmtxA[:,i]

    return sobmtxC

def parmeval(model, sobmtxA, sobmtxB, sobmtxC, time, envlist, xpdata, xspairlist, ic=True, norm=True, vardata=False, useparams = None, fileobj=None):
    ''' Function parmeval calculates the yA, yB, and yC_i arrays needed for variance-based global sensitivity analysis
    as prescribed by Saltelli and derived from the work by Sobol.
    '''
    #
    #

    # assign the arrays that will hold yA, yB and yC_n
    yA = numpy.zeros([sobmtxA.shape[0]] + [len(model.observable_patterns)])
    yB = numpy.zeros([sobmtxB.shape[0]] + [len(model.observable_patterns)])
    yC = numpy.zeros(list(sobmtxC.shape[:2]) + [len(model.observable_patterns)]) # matrix is of shape (nparam, nsamples)

    # specify that this is normalized data
    if norm is True:
        # First process the A and B matrices
        print("processing matrix A, %d iterations:", sobmtxA.shape[0])
        for i in range(sobmtxA.shape[0]):
            outlist = odesolve(model, time, envlist, sobmtxA[i], useparams, ic)
            datamax = numpy.max(outlist[0], axis = 1)
            datamin = numpy.min(outlist[0], axis = 1)
            outlistnorm = ((outlist[0].T - datamin)/(datamax-datamin)).T
            outlistnorm[0] = outlist[0][0].copy() # xpdata[0] replace time from original array
            yA[i] = compare_data(xpdata, outlistnorm, xspairlist, vardata)
            spinner(i)

        print("\nprocessing matrix B, %d iterations:", sobmtxB.shape[0])
        for i in range(sobmtxB.shape[0]):
            outlist = odesolve(model, time, envlist, sobmtxB[i], useparams, ic)
            datamax = numpy.max(outlist[0], axis = 1)
            datamin = numpy.min(outlist[0], axis = 1)
            outlistnorm = ((outlist[0].T - datamin)/(datamax-datamin)).T
            # xpdata[0] should be time, get from original array
            outlistnorm[0] = outlist[0][0].copy()
            yB[i] = compare_data(xpdata, outlistnorm, xspairlist, vardata)
            spinner(i)

        # now the C matrix, a bit more complicated b/c it is of size params x samples
        print("\nprocessing matrix C_n, %d parameters:"%(sobmtxC.shape[0]))
        for i in range(sobmtxC.shape[0]):
            print("\nprocessing processing parameter %d, %d iterations"%(i,sobmtxC.shape[1]))
            for j in range(sobmtxC.shape[1]):
                outlist = odesolve(model, time, envlist, sobmtxC[i][j], useparams, ic)
                datamax = numpy.max(outlist[0], axis = 1)
                datamin = numpy.min(outlist[0], axis = 1)
                outlistnorm = ((outlist[0].T - datamin)/(datamax-datamin)).T
                # xpdata[0] should be time, get from original array
                outlistnorm[0] = outlist[0][0].copy()
                yC[i][j] = compare_data(xpdata, outlistnorm, xspairlist, vardata)
                spinner(j)
    else:
        # First process the A and B matrices
        print("processing matrix A:")
        for i in range(sobmtxA.shape[0]):
            outlist = odesolve(model, time, envlist, sobmtxA[i], useparams, ic)
            yA[i] = compare_data(xpdata, outlist[0], xspairlist, vardata)
            spinner(i)

        print("processing matrix B:")
        for i in range(sobmtxB.shape[0]):
            outlist = odesolve(model, time, envlist, sobmtxB[i], useparams, ic)
            yB[i] = compare_data(xpdata, outlistnorm, xspairlist, vardata)
            spinner(i)

        print("processing matrix C_n")
        for i in range(sobmtxC.shape[0]):
            print("processing processing parameter %d"%i)
            for j in range(sobmtxC.shape[1]):
                outlist = odesolve(model, time, envlist, sobmtxC[i][j], useparams, ic)
                yC[i][j] = compare_data(xpdata, outlistnorm, xspairlist, vardata)
                spinner(j)

    if fileobj:
        if norm:
            writetofile(fileobj, params, outlistnorm, objout)
        else:
            writetofile(fileobj, params, outlist, objout)

    return yA, yB, yC

def getvarsens(yA, yB, yC):
    """Calculate the array of S_i and ST_i for each parameter given yA, yB, yC matrices
    from the multi-sampling runs. Calculate S_i and ST_i as follows:

    Parameter sensitivity:
    ----------------------
            U_j - E^2
    S_j = ------------
               V(y)

    U_j = 1/n \sum yA * yC_j

    E^2 = 1/n \sum yA * 1/n \sum yB

    Total effect sensitivity (i.e. non additive part):
    --------------------------------------------------
                  U_-j - E^2
     ST_j = 1 - -------------
                      V(y)

    U_-j = 1/n \sum yB * yC_j

    E^2 = { 1/n \sum yB * yB }^2

    In both cases, calculate V(y) from yA and yB

    """
    nparms = yC.shape[0] # should be the number of parameters
    nsamples = yC.shape[1] # should be the number of samples from the original matrix
    nobs = yC.shape[-1]    # the number of observables (this is linked to BNG usage, generalize?)

    #first get V(y) from yA and yB

    varyA = numpy.var(yA, axis=0, ddof=1)
    varyB = numpy.var(yB, axis=0, ddof=1)

    # now get the E^2 values for the S and ST calculations
    E_s  = numpy.average((yA * yB), axis=0)
    E_st = numpy.average(yB, axis=0) ** 2

    #allocate the S_i and ST_i arrays
    Sens = numpy.zeros((nparms,nobs))
    SensT = numpy.zeros((nparms,nobs))

    # now get the U_j and U_-j values and store them
    for i in range(nparms):
        Sens[i]  =        (((yA * yC[i]).sum(axis=0)/(nsamples-1.)) - E_s ) / varyA
        SensT[i] = 1.0 - ((((yB * yC[i]).sum(axis=0)/(nsamples-1.)) - E_st) / varyB)

    return Sens, SensT

def writetofile(fout, simparms, simdata, temperature):
    imax, jmax = simdata.shape
    nparms = len(simparms)

    fout.write('# TEMPERATURE\n{}\n'.format(temperature))
    fout.write('# PARAMETERS ({})\n'.format(len(simparms)))
    for i in range(nparms):
        fout.write('{}'.format(simparms[i]))
        if (i !=0 and i%5 == 0) or (i == nparms-1):
            fout.write('\n')
        else:
            fout.write(', ')

    fout.write('# SIMDATA ({},{})\n'.format(imax, jmax))
    for i in range(imax):
        fout.write('# {}\n'.format(i))
        for j in range(jmax):
            fout.write('{}'.format(simdata[i][j]))
            if (j != 0 and j%10 == 0) or (j == jmax-1):
                fout.write('\n')
            else:
                fout.write(', ')
    fout.write('#-------------------------------------------------------------------------------------------------\n')
    return


# yourapp/views.py

from django.shortcuts import render, redirect
from django.http import HttpResponse
from .scripts import main as script
from .scripts import validate
from .scripts.visualization import visualize
import os
import pandas as pd

data_processed = False
image_directory = os.path.join(os.getcwd(), 'yourapp/static/graphs')

def home(request):
    global data_processed
    if request.method == "POST":
        minority_class = request.POST.get("minority_class")
        minority_class_column = request.POST.get("minority_class_column")
        algorithm = request.POST.get("algorithm")
        csv_file = request.FILES.get("csv_file")

        if csv_file is None or not csv_file.name.endswith(".csv"):
            return redirect('error', error_message="No CSV file uploaded")

        input_df = pd.read_csv(csv_file)
        validation_code, message = validate.validate_input(input_df, minority_class, minority_class_column)
        if validation_code == 0:
            return redirect('error', error_message=message)

        status_code = script.function(input_df, minority_class, minority_class_column, algorithm)
        if status_code == 1:
            data_processed = True
            return redirect('download')
        else:
            return redirect('error', error_message="Internal Error, Please try again.")

    return render(request, "templates/index.html")

def error(request):
    error_message = request.GET.get("error_message", "Internal Error, Please try again.")
    return render(request, "yourapp/error.html", {"message": error_message})

def custom_autoencoder(request):
    global data_processed
    if request.method == "POST":
        minority_class = request.POST.get("minority_class")
        minority_class_column = request.POST.get("minority_class_column")
        csv_file = request.FILES.get("csv_file")
        encoder_dense_layers = request.POST.get("encoder_dense_layers")
        bottle_neck = request.POST.get("bottle_neck")
        decoder_dense_layers = request.POST.get("decoder_dense_layers")
        epochs = request.POST.get("epochs")
        decoder_activation = request.POST.get("decoder_activation")

        if csv_file is None or not csv_file.name.endswith(".csv"):
            return redirect('error', error_message="No CSV file uploaded")

        input_df = pd.read_csv(csv_file)
        validation_code, message = validate.validate_input(
            input_df, minority_class, minority_class_column,
            custom=True, algorithm="autoencoder",
            encoder_dense_layers=encoder_dense_layers,
            bottle_neck=bottle_neck,
            decoder_dense_layers=decoder_dense_layers,
            epochs=epochs
        )

        if validation_code == 0:
            return redirect('error', error_message=message)

        try:
            status_code = script.function(
                input_df, minority_class, minority_class_column,
                algorithm="custom_autoencoder",
                encoder_dense_layers=encoder_dense_layers,
                bottle_neck=bottle_neck,
                decoder_dense_layers=decoder_dense_layers,
                epochs=epochs, decoder_activation=decoder_activation
            )
        except Exception as e:
            print(e)
            return redirect('error', error_message="Internal Error, Please try again.")

        if status_code == 1:
            data_processed = True
            return redirect('download')

    return render(request, "yourapp/custom_autoencoder.html")

def download(request):
    global data_processed
    if data_processed:
        return render(request, "yourapp/download.html")
    else:
        return redirect('home')

def visualization(request):
    if request.method == "POST":
        minority_class = request.POST.get("minority_class")
        minority_class_column = request.POST.get("minority_class_column")
        original_file = request.FILES.get("original_file")
        synthetic_file = request.FILES.get("synthetic_file")
        pure_files_checkbox = request.POST.get("pure_files")
        pure_files_checkbox = True if pure_files_checkbox else False

        if original_file is None or not original_file.name.endswith(".csv") or synthetic_file is None or not synthetic_file.name.endswith(".csv"):
            return redirect('error', error_message="No CSV file uploaded")

        original_df = pd.read_csv(original_file)
        synthetic_df = pd.read_csv(synthetic_file)

        validation_code_1, message_1 = validate.validate_input(original_df, minority_class, minority_class_column)
        validation_code_2, message_2 = validate.validate_input(synthetic_df, minority_class, minority_class_column)

        if validation_code_1 == 0:
            return redirect('error', error_message=message_1)
        if validation_code_2 == 0:
            return redirect('error', error_message=message_2)

        try:
            status_code = visualize(original_df, synthetic_df, minority_class, minority_class_column, pure_files_checkbox)
        except Exception as e:
            print(e)
            return redirect('error', error_message="Internal Error, Please try again.")

        if status_code == 1:
            image_filenames = ['density_plot.png', 'heatmap_plot.png', 'scatter_plot.png']
            return render(request, 'yourapp/visualization_plot.html', {'image_filenames': image_filenames})

    return render(request, "yourapp/visualization.html")

def download_synthetic_data(request):
    if data_processed:
        synthetic_csv_path = "synthetic_data.csv"
        return HttpResponse(synthetic_csv_path, content_type='text/csv')
    else:
        return redirect('home')

def serve_image(request, filename):
    return redirect(os.path.join(image_directory, filename))


from datetime import datetime
from classes.field import Field

class Birthday(Field):
    @Field.value.setter
    def value(self, value=None):
        if value and type(value) == str:
            value = value.replace('.', '-')
            try:
                value = datetime.strptime(value, '%d-%m-%Y').date()
            except:
                raise ValueError('Invalid date format. Must be dd-mm-yyyy or dd.mm.yyyy')
        elif value != None:
            raise ValueError('Invalid date format. Must be string dd-mm-yyyy or dd.mm.yyyy')
        self._value = value

    def __repr__(self):
        return datetime.strftime(self.value, '%d-%m-%Y')


import logging
from typing import List

from volatility3.framework import constants, exceptions, renderers, interfaces
from volatility3.framework.configuration import requirements
from volatility3.framework.objects import utility
from volatility3.plugins.windows import pslist

vollog = logging.getLogger(__name__)

class CmdLine(interfaces.plugins.PluginInterface):
    """Lists process command line arguments."""

    _required_framework_version = (2, 0, 0)
    _version = (1, 0, 0)

    @classmethod
    def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
        # Since we're calling the plugin, make sure we have the plugin's requirements
        return [
            requirements.ModuleRequirement(
                name="kernel",
                description="Windows kernel",
                architectures=["Intel32", "Intel64"],
            ),
            requirements.PluginRequirement(
                name="pslist", plugin=pslist.PsList, version=(2, 0, 0)
            ),
            requirements.ListRequirement(
                name="pid",
                element_type=int,
                description="Process IDs to include (all other processes are excluded)",
                optional=True,
            ),
        ]

    @classmethod
    def get_cmdline(
        cls, context: interfaces.context.ContextInterface, kernel_table_name: str, proc
    ):
        """Extracts the cmdline from PEB

        Args:
            context: the context to operate upon
            kernel_table_name: the name for the symbol table containing the kernel's symbols
            proc: the process object

        Returns:
            A string with the command line
        """

        proc_layer_name = proc.add_process_layer()

        peb = context.object(
            kernel_table_name + constants.BANG + "_PEB",
            layer_name=proc_layer_name,
            offset=proc.Peb,
        )
        result_text = peb.ProcessParameters.CommandLine.get_string()

        return result_text

    def _generator(self, procs):
        kernel = self.context.modules[self.config["kernel"]]

        for proc in procs:
            process_name = utility.array_to_string(proc.ImageFileName)
            proc_id = "Unknown"

            try:
                proc_id = proc.UniqueProcessId
                result_text = self.get_cmdline(
                    self.context, kernel.symbol_table_name, proc
                )

            except exceptions.SwappedInvalidAddressException as exp:
                result_text = f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)"

            except exceptions.PagedInvalidAddressException as exp:
                result_text = f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)"

            except exceptions.InvalidAddressException as exp:
                result_text = "Process {}: Required memory at {:#x} is not valid (incomplete layer {}?)".format(
                    proc_id, exp.invalid_address, exp.layer_name
                )

            yield (0, (proc.UniqueProcessId, process_name, result_text))

    def run(self):
        kernel = self.context.modules[self.config["kernel"]]
        filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None))

        return renderers.TreeGrid(
            [("PID", int), ("Process", str), ("Args", str)],
            self._generator(
                pslist.PsList.list_processes(
                    context=self.context,
                    layer_name=kernel.layer_name,
                    symbol_table=kernel.symbol_table_name,
                    filter_func=filter_func,
                )
            ),
        )


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('behavior_subjects', '0007_auto_20151119_1117'),
    ]

    operations = [
        migrations.AddField(
            model_name='session',
            name='exh_inh_delay',
            field=models.IntegerField(blank=True, verbose_name='Exh-inh delay', default=-1),
        ),
    ]


import pandas as pd
import numpy as np
import pickle

from sklearn.base import TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.pipeline import Pipeline
from sklearn.metrics import f1_score, cohen_kappa_score, accuracy_score, recall_score, precision_score, roc_auc_score, confusion_matrix
from sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.model_selection import GridSearchCV, train_test_split, cross_val_score, RandomizedSearchCV
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression

from xgboost import XGBClassifier

def performance(y_true, y_pred, id_model):

    """

    Calculate performance metrics

    :param y_true: array with true values
    :param y_pred: array with predicted values
    :param id_model: model identifier
    :return: DataFrame with scores

    """

    # Calculate metrics
    accuracy = accuracy_score(y_true, y_pred)
    recall = recall_score(y_true, y_pred)
    precision = precision_score(y_true, y_pred)
    kappa = cohen_kappa_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred)
    auc = roc_auc_score(y_true, y_pred)

    model_performance_dict = {
        'accuracy': accuracy,
        'recall': recall,
        'precision': precision,
        'kappa': kappa,
        'f1': f1,
        'auc': auc
    }

    perf = pd.DataFrame(
        list(model_performance_dict.items()),
        index=['a', 'b', 'c', 'd', 'e', 'f'],
        columns=['metric', 'score_' + id_model]
    )

    return perf

def logistic_reg(df, class_column, id_model):

    """

    Train Logistic Regression

    :param df: dataframe with features and class
    :param class_column: column name for class
    :param id_model: model identifier
    :return model_rlog: trained model
    :return performance_rlog: dataframe with perfomance metrics
    :return cm_log: confusion matrix

    """

    # Partition in training and testing
    x = df.loc[:, df.columns != class_column] #select all columns except the class
    y = df[class_column]

    x_train, x_test, y_train, y_test = train_test_split(
        x,
        y,
        stratify=y,
        test_size=0.20,
        random_state=1234
    )

    # Scaling
    scaler = StandardScaler().fit(x)
    x_train_sc = scaler.transform(x_train)
    x_test_sc  = scaler.transform(x_test)

    # Generate LR model
    log_model = LogisticRegression(solver='saga',
                                   penalty='elasticnet', # for elastic net penalty
                                   l1_ratio=0.5,
                                   class_weight='balanced', # class balance
                                   max_iter=5000)

    # Fitting
    model_rlog = log_model.fit(x_train_sc, y_train)

    # Testing
    y_pred = model_rlog.predict(x_test_sc)

    # Evaluating performance
    performance_rlog = performance(y_test, y_pred, id_model)

    # Confusion Matrix
    cm_log = confusion_matrix(y_test, y_pred)

    return model_rlog, performance_rlog, cm_log

def random_forest(df, class_column, id_model):

    """

    Train Random Forest Classifier

    :param df: dataframe with features and class
    :param class_column: column name for class
    :param id_model: model identifier
    :return model_rlog: trained model
    :return performance_rlog: dataframe with perfomance metrics
    :return cm_log: confusion matrix

    """

    # Partition in training and testing
    x = df.loc[:, df.columns != class_column] #select all columns except the class
    y = df[class_column]

    # Prepare hyperparameters to test
    random_grid = {
        'n_estimators': [250, 500, 1000, 1500, 2000], # Number of trees in random forest
        'max_features': ['auto'], # Number of features to consider at every split
        'max_depth': [10, 25, 50, 75, 100], # Maximum number of levels in tree
        'min_samples_split': [5, 10, 15, 20, 30, 40], # Minimum number of samples required to split a node
        'min_samples_leaf': [5, 10, 15, 20] # Minimum number of samples required at each leaf node
    }

    # Create a base model to adjust
    rf = RandomForestClassifier()

    # Perform a Random Search of parameters (3 cross validation, 100 different combinations, use all available cores)
    rf_random = RandomizedSearchCV(
        estimator=rf,
        scoring='f1',
        param_distributions=random_grid,
        n_iter=100,
        v=3,
        verbose=2,
        random_state=42,
        n_jobs=-1
    )

    # Fitting
    rf_random.fit(x_train, y_train)

    # Testing
    y_pred = rf_random.predict(x_test)

    # Evaluating performance
    performance_randomf = performance(y_test, y_pred, modelo)

    # Confusion Matrix
    cm_rf = confusion_matrix(y_test, y_pred)

    return rf_random, performance_randomf, cm_rf

def xgboost(df, class_column, id_model):

    """

    Train XGBoost Classifier

    :param df: dataframe with features and class
    :param class_column: column name for class
    :param id_model: model identifier
    :return model_rlog: trained model
    :return performance_rlog: dataframe with perfomance metrics
    :return cm_log: confusion matrix

    """

    # Partition in training and testing
    x = df.loc[:, df.columns != class_column] #select all columns except the class
    y = df[class_column]

    # prepare the hyperparameters to test
    random_grid_xgb = {
        # Parameters that we are going to tune.
        'learning_rate' : [0.01, 0.02, 0.04, 0.06, 0.08, 0.1],
        'n_estimators' : [500, 750, 1000, 1250, 1500],
        'max_depth':[5, 10, 20, 30],
        'subsample': [0.8, 0.9, 1],
        'colsample_bytree': [0.4, 0.6, 0.8, 1],
        'gamma': [0.1, 0.5, 0.7, 1]
    }

    xgb = XGBClassifier() # base model
    xgb_random = RandomizedSearchCV(
        estimator=xgb,
        scoring='f1',
        param_distributions=random_grid_xgb,
        n_iter=100,
        cv=3,
        verbose=2,
        random_state=42,
        n_jobs=-1
    )

    # Fitting
    xgb_random.fit(x_train, y_train)

    # Testing
    y_pred = xgb_random.predict(x_test)

    # Evaluating performance
    performance_xgb = performance(y_test, y_pred, modelo)

    # Confusion Matrix
    cm_xgb = confusion_matrix(y_test, y_pred)

    return xgb_random, performance_xgb, cm_xgb


import torch

def nanminmax(tensor, operation='min', dim=None, keepdim=False):
    if operation not in ['min','max']:
        raise ValueError("Operation must be 'min' or 'max'.")

    mask = torch.isnan(tensor)
    replacement = float('-inf') if operation == 'max' else float('inf')
    replacement = torch.tensor(replacement, dtype=tensor.dtype, device=tensor.device)
    tensor_masked = torch.where(mask, replacement, tensor)

    if operation == 'min':
        values, _ = torch.min(tensor_masked, dim=dim, keepdim=keepdim)
    else:
        values, _ = torch.max(tensor_masked, dim=dim, keepdim=keepdim)

    values = torch.where(torch.all(mask, dim=dim, keepdim=keepdim), torch.tensor(float('nan'), \
        dtype=tensor.dtype, device=tensor.device), values)
    return values

def nanstd(tensor, dim=None):
    mean = torch.nanmean(tensor, dim=dim, keepdim=True)
    deviations = tensor - mean
    squared_deviations = torch.square(deviations)
    nanvar = torch.nanmean(squared_deviations, dim=dim, keepdim=True)
    nanstd = torch.sqrt(nanvar)
    return nanstd

def zscore(tensor, dim=None):
    mean = torch.nanmean(tensor, dim=dim, keepdim=True)
    std = nanstd(tensor, dim=dim)
    zscore = (tensor - mean) / std
    return zscore

def torch_nan_to_num(input_tensor, nan=0.0, posinf=None, neginf=None):
    output_tensor = torch.where(torch.isnan(input_tensor), torch.tensor(nan), input_tensor)

    if posinf is not None:
        output_tensor = torch.where(torch.isposinf(output_tensor), torch.tensor(posinf), output_tensor)

    if neginf is not None:
        output_tensor = torch.where(torch.isneginf(output_tensor), torch.tensor(neginf), output_tensor)

    return output_tensor

class MinMaxScaler:
    def __init__(self):
        self.min = None
        self.max = None

    def fit(self, data):
        self.min = nanminmax(data, 'min', dim=0)
        self.max = nanminmax(data, 'max', dim=0)

    def transform(self, data):
        return (data - self.min) / (self.max - self.min + 1e-10)

    def fit_transform(self, data):
        self.fit(data)
        return self.transform(data)

class QuantileTransformer:
    def __init__(self, n_quantiles=100):
        self.n_quantiles = n_quantiles
        self.quantiles = None

    def fit(self, data):
        self.quantiles = torch.quantile(data, torch.linspace(0, 1, self.n_quantiles))

    def transform(self, data):
        sorted_data = torch.sort(data, dim=0).values
        ranks = torch.searchsorted(self.quantiles, sorted_data)
        transformed_data = ranks / (self.n_quantiles - 1)
        return transformed_data

    def fit_transform(self, data):
        self.fit(data)
        return self.transform(data)


import torch
import numpy as np

'''
목표 MobileNet v2에 빠르게 QIL 적용

1. alexnet training  <- success
2. apply QIL to alexnet (= reproduce)
   - know How to quantize ex ) aware quantization - > pytorch source
   - know How to apply QIL on alexnet
   - conduct training
3. Apply QIL to MobileNet v2
'''

class Quantizer():

    discretization_level = 32
    def __init__(self):
        torch.manual_seed(0)
        self.cw,self.dw,self.cx,self.dx,self.gamma = torch.randn(5) - 0.5
        print(self.cw,self.dw,self.cx,self.dx,self.gamma)

    def quantize(self, target, param):
        if target is 'weight':
            self.transfomer(param,self.cw,self.dw,self.gamma)
        elif target is 'activation':
            self.transfomer(param,self.cx,self.dx)
        else:
            print("Warning : Target is not weight or activation")
        self.discretizer(param)

    def transfomer(self,tensor,c_delta,d_delta,r = 1):
        outplane,inplane,kh,kw = tensor.size()
        for o in range(outplane):
            for i in range(inplane):
                for h in range(kh):
                    for w in range(kw):
                        t = tensor[o][i][h][w]
                        if abs(t) < c_delta - d_delta:
                            t = 0
                        elif abs(t) > c_delta + d_delta:
                            t.sign_()
                        else:
                            a,b = (0.5 / d_delta) , (-0.5*c_delta / d_delta + 0.5)
                            t = (a*abs(w) + b)**r * t.sign()
                        tensor[o][i][h][w] = t

    def discretizer(self,tensor):
        q_D = pow(2, Quantizer.discretization_level)
        torch.round_(tensor.mul_(q_D))
        tensor.div_(q_D)

if __name__ == '__main__':
    weights = torch.rand(2,2,2,2).sub_(0.5)
    activation = torch.rand(4,4,4,4)
    q = Quantizer()
    print("before \n",weights)
    q.quantize('weight', weights)
    #q.transfomer_weights(param,c_delta,d_delta,r)
    #q.discretizer(weights)
    print("after \n",weights)


import plotly.offline as py
import plotly.graph_objs as go
import sensor_input
import random
from pandas import Series

def plot_spectrum_from_files(name,files):
    spectrums = []
    for file in files:
        spectrums.append(sensor_input.read_spectrum_from_file(file))
    plot_spectrums(name,spectrums)

def plot_data_from_pandas(pandas):
    data = []
    for panda in pandas:
        data.append(go.Scatter(
            x=panda,
            y=spectrum["power"],
            marker = dict(
                color = 'rgba(10, 10, 240, .9)'
            )))
    return data

def get_plot_data(name, spectrums):
    data = []
    r = random.randrange(0, 255)
    g = random.randrange(0, 255)
    b = random.randrange(0, 255)
    for spectrum in spectrums:
        data.append(go.Scatter(
            x=spectrum["wavelength"],
            y=spectrum["power"],
            name=name,
            marker = dict(
                color = 'rgba(%s, %s, %s, .9)'%(r,g,b)
            )
        ))
    return data

def read_plot_data_from_files(name,files):
    spectrums = []
    for file in files:
        spectrums.append(sensor_input.read_spectrum_from_file(file))
    return get_plot_data(name,spectrums)

def plot_spectrum_from_files_group(file_groups):
    data = []
    for key in file_groups.keys():
        for plot_data in read_plot_data_from_files(key,file_groups[key]):
            data.append(plot_data)

    py.plot(data, filename="all_ploted")

def plot_spectrums(name,spectrums):
    data = get_plot_data(name,spectrums)
    py.plot(data, filename=name)


import pytest
from unittest.mock import sentinel, Mock
import bokeh.palettes
import pandas as pd
import pandas.testing as pdt
import datetime as dt
import numpy as np
import glob
import forest.drivers
from forest.drivers import earth_networks

LINES = [
    "1,20190417T000001.440,+02.7514400,+031.9206400,-000001778,000,15635,007,001",
    "1,20190417T000001.093,+02.6388400,+031.9008800,+000002524,000,14931,007,012",
]

def test_earth_networks(tmpdir):
    path = str(tmpdir / "sample.txt")
    with open(path, "w") as stream:
        stream.writelines(LINES)

    loader = earth_networks.Loader()
    frame = loader.load([path])
    result = frame.iloc[0]
    atol = 0.000001
    if isinstance(result["date"], dt.datetime):
        # Pandas <0.25.x
        assert result["date"] == dt.datetime(2019, 4, 17, 0, 0, 1, 440000)
    else:
        # Pandas 1.0.x
        assert result["date"] == np.datetime64("2019-04-17T00:00:01.440000000")
    assert result["flash_type"] == "IC"
    assert abs(result["latitude"] - 2.75144) < atol
    assert abs(result["longitude"] - 31.92064) < atol

def test_dataset():
    dataset = forest.drivers.get_dataset("earth_networks")
    assert isinstance(dataset, forest.drivers.earth_networks.Dataset)

def get_navigator(settings):
    dataset = forest.drivers.get_dataset("earth_networks", settings)
    return dataset.navigator()

def test_dataset_navigator():
    navigator = get_navigator({"pattern": "*.txt"})
    assert isinstance(navigator, forest.drivers.earth_networks.Navigator)

def test_navigator_variables():
    navigator = earth_networks.Navigator([])
    assert set(navigator.variables(None)) == set(
        [
            "Strike density (cloud-ground)",
            "Strike density (intra-cloud)",
            "Strike density (total)",
            "Time since flash (cloud-ground)",
            "Time since flash (intra-cloud)",
            "Time since flash (total)",
        ]
    )

def test_view_render_density():
    locator = Mock(specs=["find"])
    loader = Mock(specs=["load"])
    loader.load.return_value = pd.DataFrame(
        {
            "flash_type": [],
            "longitude": [],
            "latitude": [],
        }
    )
    view = earth_networks.View(loader, locator)
    view.render(
        {
            "variable": "Strike density (cloud-ground)",
            "valid_time": "1970-01-01T00:00:00Z",
        }
    )
    expect = bokeh.palettes.all_palettes["Spectral"][8]
    assert view.color_mappers["image"].palette == expect

def test_view_render_time_since_flash():
    locator = Mock(specs=["find"])
    loader = Mock(specs=["load"])
    loader.load.return_value = pd.DataFrame(
        {
            "date": [],
            "flash_type": [],
            "longitude": [],
            "latitude": [],
        }
    )
    view = earth_networks.View(loader, locator)
    view.render(
        {
            "variable": "Time since flash (cloud-ground)",
            "valid_time": "1970-01-01T00:00:00Z",
        }
    )
    expect = bokeh.palettes.all_palettes["RdGy"][8]
    assert view.color_mappers["image"].palette == expect

@pytest.mark.parametrize(
    "variable, expect",
    [
        pytest.param(
            "Time since flash (intra-cloud)",
            [
                ("Variable", "@variable"),
                ("Time window", "@window{00:00:00}"),
                ("Period start", "@date{%Y-%m-%d %H:%M:%S}"),
                ("Since start", "@image{00:00:00}"),
            ],
            id="time since flash",
        ),
        pytest.param(
            "Strike density (cloud-ground)",
            [
                ("Variable", "@variable"),
                ("Time window", "@window{00:00:00}"),
                ("Period start", "@date{%Y-%m-%d %H:%M:%S}"),
                ("Value", "@image @units"),
            ],
            id="strike density",
        ),
    ],
)
def test_view_tooltips(variable, expect):
    assert earth_networks.View.tooltips(variable) == expect

@pytest.mark.parametrize(
    "variable, expect",
    [
        pytest.param(
            "Time since flash (intra-cloud)",
            {"@date": "datetime", "@window": "numeral", "@image": "numeral"},
            id="time since flash",
        ),
        pytest.param(
            "Strike density (cloud-ground)",
            {"@date": "datetime", "@window": "numeral"},
            id="strike density",
        ),
    ],
)
def test_view_formatters(variable, expect):
    assert earth_networks.View.formatters(variable) == expect

def test_view_since_flash():
    view = earth_networks.View(Mock(), Mock())
    strike_times = ["2020-01-01T00:00:00Z", "2020-01-01T01:00:00Z"]
    period_start = "2020-01-01T00:00:00Z"
    result = view.since_flash(strike_times, period_start)
    expect = pd.Series([0.0, 3600.0])
    pdt.assert_series_equal(result, expect)


from pprint import pprint
from azdev.operations.regex import (
    get_all_tested_commands_from_regex,
    search_argument,
    search_argument_context,
    search_command,
    search_command_group)

# pylint: disable=line-too-long
# one line test
def test_one_line_regex():
    lines = [
        # start with self.cmd.
        'self.cmd(\'image builder create -n {tmpl_02} -g {rg} --identity {ide} --scripts {script} --image-source {img_src} --build-timeout 22\')\n',
        # start with blanks, match self.cmd("").
        '            self.cmd("role assignment create --assignee {assignee} --role {role} --scope {scope}")\n',
        # start with blanks, match self.cmd('').
        '            self.cmd(\'role assignment create --assignee {assignee} --role {role} --scope {scope}\')\n',
        # start with multiple blanks, characters, self.cmd
        '     identity_id = self.cmd(\'identity create -g {rg} -n {ide}\').get_output_in_json()[\'clientId\']\n',
        # start with blanks, match self.cmd, use fstring, ''
        '        self.cmd(f\'afd profile usage -g {resource_group} --profile-name {profile_name}\', checks=usage_checks)',
        # start with blanks, match self.cmd, use fstring, ""
        '        self.cmd(f"afd profile usage -g {resource_group} --profile-name {profile_name}", checks=usage_checks)',
        # one line docstring '''
        '        self.cmd("""afd profile usage -g {resource_group} --profile-name {profile_name}""", checks=usage_checks)',
        # one line docstring """
        '        self.cmd("""afd profile usage -g {resource_group} --profile-name {profile_name}""", checks=usage_checks)',
        # .format
        '        self.cmd("afd profile usage -g {} --profile-name {}".format(group, name)',
        # %s
        '        self.cmd("afd profile usage -g %s --profile-name %s", group, name)',
        # end with hashtag, should match.
        '        self.cmd(f"afd profile usage -g {resource_group} --profile-name {profile_name}", checks=usage_checks) #  xxx',
        # start with hashtag, shouldn't match.
        '        # self.cmd(f"afd profile usage -g {resource_group} --profile-name {profile_name}", checks=usage_checks)',
        # start with blanks, match *_cmd = ''.
        '    stop_cmd = \'aks stop --resource-group={resource_group} --name={name}\'\n',
        # start with blanks, match *_cmd = "".
        '    enable_cmd = "aks enable-addons --addons confcom --resource-group={resource_group} --name={name} -o json"\n',
        # start with blanks, match *_cmd = f''.
        '    disable_cmd = f\'aks disable-addons --addons confcom --resource-group={resource_group} --name={name} -o json\'\n',
        # start with blanks, match *_cmd = f"".
        '    browse_cmd = f"aks browse --resource-group={resource_group} --name={name} --listen-address=127.0.0.1 --listen-port=8080 --disable-browser"\n',
    ]
    ref = get_all_tested_commands_from_regex(lines)
    pprint(ref, width=1000)
    assert len(ref) == 15

# multiple lines test
def test_multiple_lines_regex():
    lines = [
        # start with blanks, self.cmd, one cmd line, multiple checks.
        '        self.cmd(\'aks list -g {resource_group}\', checks=[\n',
        '            self.check(\'[0].type\', \'{resource_type}\'),\n',
        '            StringContainCheck(aks_name),\n',
        '            StringContainCheck(resource_group)\n',
        '        ])\n',
        # start with blanks, self.cmd, multiple cmd lines, multiple checks.
        '        self.cmd(\'image builder create -n {tmpl_02} -g {rg} --identity {ide} --scripts {script} --image-source {img_src} --build-timeout 22\'\n',
        '                 \' --managed-image-destinations img_1=westus \' + out_3,\n',
        '                 checks=[\n',
        '                     self.check(\'name\', \'{tmpl_02}\'), self.check(\'provisioningState\', \'Succeeded\'),\n',
        '                     self.check(\'length(distribute)\', 2),\n',
        '                     self.check(\'distribute[0].imageId\', \'/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/images/img_1\'),\n',
        '                     self.check(\'distribute[0].location\', \'westus\'),\n',
        '                     self.check(\'distribute[0].runOutputName\', \'img_1\'),\n',
        '                     self.check(\'distribute[0].type\', \'ManagedImage\'),\n',
        '                     self.check(\'distribute[1].imageId\', \'/subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Compute/images/img_2\'),\n',
        '                     self.check(\'distribute[1].location\', \'centralus\'),\n',
        '                     self.check(\'distribute[1].runOutputName\', \'img_2\'),\n',
        '                     self.check(\'distribute[1].type\', \'ManagedImage\'),\n',
        '                     self.check(\'buildTimeoutInMinutes\', 22)\n',
        '                 ])\n',
        # start with blanks, characters, self.cmd, but have line break at self.cmd(
        '    ipprefix_id = self.cmd(\n',
        '        \'az network public-ip prefix create -g {rg} -n {ipprefix_name} --location {location} --length 29\'). \\n',
        '        get_output_in_json().get("id")\n',
        # start with blanks, match *_cmd = '', multiple lines.
        '    create_cmd = \'aks create -g {resource_group} -n {name} -p {dns_name_prefix} --ssh-key-value {ssh_key_value} \' \\n',
        '                 \'-l {location} --service-principal {service_principal} --client-secret {client_secret} -k {k8s_version} \' \\n',
        '                 \'--node-vm-size {vm_size} \' \\n',
        '                 \'--tags scenario_test -c 1 --no-wait\'\n',
        '    update_cmd = \'aks update --resource-group={resource_group} --name={name} \' \\n',
        '                 \'--aad-admin-group-object-ids 00000000-0000-0000-0000-000000000002 \' \\n',
        '                 \'--aad-tenant-id 00000000-0000-0000-0000-000000000003 -o json\'\n',
        '    enable_autoscaler_cmd = \'aks update --resource-group={resource_group} --name={name} \' \\n',
        '                            \'--tags {tags} --enable-cluster-autoscaler --min-count 2 --max-count 5\'\n',
        '    disable_autoscaler_cmd = \'aks update --resource-group={resource_group} --name={name} \' \\n',
        '                             \'--tags {tags} --disable-cluster-autoscaler\'\n',
        '    create_spot_node_pool_cmd = \'aks nodepool add \' \\n',
        '                                \'--resource-group={resource_group} \' \\n',
        '                                \'--cluster-name={name} \' \\n',
        '                                \'-n {spot_node_pool_name} \' \\n',
        '                                \'--priority Spot \' \\n',
        '                                \'--spot-max-price {spot_max_price} \' \\n',
        '                                \'-c 1\'\n',
        '    create_ppg_node_pool_cmd = \'aks nodepool add \' \\n',
        '                               \'--resource-group={resource_group} \' \\n',
        '                               \'--cluster-name={name} \' \\n',
        '                               \'-n {node_pool_name} \' \\n',
        '                               \'--ppg={ppg} \'\n',
        '    upgrade_node_image_only_cluster_cmd = \'aks upgrade \' \\n',
        '                                          \'-g {resource_group} \' \\n',
        '                                          \'-n {name} \' \\n',
        '                                          \'--node-image-only \' \\n',
        '                                          \'--yes\'\n',
        '    upgrade_node_image_only_nodepool_cmd = \'aks nodepool upgrade \' \\n',
        '                                           \'--resource-group={resource_group} \' \\n',
        '                                           \'--cluster-name={name} \' \\n',
        '                                           \'-n {node_pool_name} \' \\n',
        '                                           \'--node-image-only \' \\n',
        '                                           \'--no-wait\'\n',
        '    get_nodepool_cmd = \'aks nodepool show \' \\n',
        '                       \'--resource-group={resource_group} \' \\n',
        '                       \'--cluster-name={name} \' \\n',
        '                       \'-n {node_pool_name} \'\n',
        # start with blanks, match *_cmd = '', multiple lines, use .format
        '    install_cmd = \'aks install-cli --client-version={} --install-location={} --base-src-url={} \' \\n',
        '                  \'--kubelogin-version={} --kubelogin-install-location={} --kubelogin-base-src-url={}\'.format(version,\n',
        '                                                                                ctl_temp_file,\n',
        '                                                                                "",\n',
        '                                                                                version,\n',
        '                                                                                login_temp_file,\n',
        '                                                                                "")\n',
        # start with blanks, match *_cmd, use docstring """
        '        create_cmd = """storage account create -n {sc} -g {rg} -l eastus2euap --enable-files-adds --domain-name\n',
        '    {domain_name} --net-bios-domain-name {net_bios_domain_name} --forest-name {forest_name} --domain-guid\n',
        '    {domain_guid} --domain-sid {domain_sid} --azure-storage-sid {azure_storage_sid}"""\n',
        '        update_cmd = """storage account update -n {sc} -g {rg} --enable-files-adds --domain-name {domain_name}\n',
        '    --net-bios-domain-name {net_bios_domain_name} --forest-name {forest_name} --domain-guid {domain_guid}\n',
        '    --domain-sid {domain_sid} --azure-storage-sid {azure_storage_sid}"""\n',
        # start with blanks, match *_cmd, use docstring '''
        '        update_cmd = \'\'\'storage account update -n {sc} -g {rg} --enable-files-adds --domain-name {domain_name}\n',
        '    --net-bios-domain-name {net_bios_domain_name} --forest-name {forest_name} --domain-guid {domain_guid}\n',
        '    --domain-sid {domain_sid} --azure-storage-sid {azure_storage_sid}\'\'\'\n',
        # start with blanks, match *_cmd*, .format
        '        create_cmd1 = \'az storage account create -n {} -g {} --routing-choice MicrosoftRouting --publish-microsoft-endpoint true\'.format('
        '        name1, resource_group)',
        # start with blanks, match .cmd
        '    test.cmd(\'az billing account list \'',
        '             \'--expand "soldTo,billingProfiles,billingProfiles/invoiceSections"\',',
        '             checks=[])',
        # start with blanks, match *Command
        '        runCommand = \'aks command invoke -g {resource_group} -n {name} -o json -c "kubectl get pods -A"\'',
        '        self.cmd(runCommand, [',
        '            self.check(\'provisioningState\', \'Succeeded\'),',
        '            self.check(\'exitCode\', 0),',
        '        ])',
        # start with blanks, match *command, use fstring, single quotation marks
        '    command = f\'afd origin-group update -g {resource_group_name} --profile-name {profile_name} \' ',
        '              f\'--origin-group-name {origin_group_name}\'',
        # string splicing (+)
        '    self.cmd(\'spring-cloud app deployment create -g {resourceGroup} -s {serviceName} --app {app} -n green\'\n',
        '             + \' --container-image {containerImage} --registry-username PLACEHOLDER --registry-password PLACEHOLDER\',\n',
        '             checks=[\n',
        '                 self.check(\'name\', \'green\'),\n',
        '             ])\n',
        # --format vs .format
        '    self.cmd(\n',
        '        \'appconfig kv import -n {config_store_name} -s {import_source} --path "{strict_import_file_path}" --format {imported_format} --profile {profile} --strict -y\')',
    ]
    ref = get_all_tested_commands_from_regex(lines)
    pprint(ref, width=1000)
    assert len(ref) == 22

def test_detect_new_command():
    commands = []
    lines = [
        'with self.command_group(\'disk\', compute_disk_sdk, operation_group=\'disks\', min_api=\'2017-03-30\') as g:',
        # 1.`+    g.command(xxx)`
        '+    g.command(\'list-instances\', \'list\', command_type=compute_vmss_vm_sdk)',
        # 2.`+    g.custom_command(xxx)`
        '+    g.custom_command(\'create\', \'create_managed_disk\', supports_no_wait=True, table_transformer=transform_disk_show_table_output, validator=process_disk_or_snapshot_create_namespace)',
        # 3.`+    g.custom_show_command(xxx)`
        '+    g.custom_show_command(\'show\', \'get_vmss\', table_transformer=get_vmss_table_output_transformer(self, False))',
        # 4.`+    g.wait_command(xxx)`
        '+    g.wait_command(\'wait\', getter_name=\'get_vmss\', getter_type=compute_custom)',
    ]
    for row_num, line in enumerate(lines):
        command = search_command(line)
        if command:
            cmd = search_command_group(row_num, lines, command)
            if cmd:
                commands.append(cmd)
    pprint(commands)
    assert commands == ['disk list-instances', 'disk create', 'disk show', 'disk wait']

def test_detect_new_params():
    parameters = []
    lines = [
        # without scope
        '    with self.argument_context(\'disk\') as c:',
        '+        c.argument(\'network_policy\')',
        '+        c.argument(\'zone\', zone_type, min_api=\'2017-03-30\', options_list=[\'--zone\']) ',
        # scope
        '    for scope in [\'disk\', \'snapshot\']:',
        '        with self.argument_context(scope) as c:',
        '+            c.argument(\'size_gb\', options_list=[\'--size-gb\', \'-z\'], help=\'size in GB. Max size: 4095 GB (certain preview disks can be larger).\', type=int)',
        # scope with multi args
        '    for scope in [\'signalr create\', \'signalr update\']:',
        '        with self.argument_context(scope, arg_group=\'Network Rule\') as c:',
        '+            c.argument(\'default_action\', arg_type=get_enum_type([\'Allow\', \'Deny\']), help=\'Default action to apply when no rule matches.\', required=False)',
        # scope AND format
        '    for scope in [\'create\', \'update\']:',
        '        with self.argument_context(\'vmss run-command {}\'.format(scope)) as c:',
        '+            c.argument(\'vmss_name\', run_cmd_vmss_name)',
        # scope AND format
        '    for scope in [\'vm\', \'vmss\']:',
        '        with self.argument_context(\'{} stop\'.format(scope)) as c:',
        '+            c.argument(\'skip_shutdown\', action=\'store_true\', help=\'Skip shutdown and power-off immediately.\', min_api=\'2019-03-01\')',
        # multiple `[]`
        '    with self.argument_context(\'acr connected-registry create\') as c:',
        '+        c.argument(\'client_token_list\', options_list=[\'--client-tokens\'], nargs=\'+\', help=\'Specify the client access to the repositories in the connected registry. It can be in the format [TOKEN_NAME01] [TOKEN_NAME02]...\')',
        '+        c.argument(\'notifications\', options_list=[\'--notifications\'], nargs=\'+\', help=\'List of artifact pattern for which notifications need to be generated. Use the format "--notifications [PATTERN1 PATTERN2 ...]".\')',
        # multiple lines
        '    with self.argument_context(\'webapp update\') as c:',
        '+        c.argument(\'skip_custom_domain_verification\',',
        '+                   help="If true, custom (non *.azurewebsites.net) domains associated with web app are not verified",',
        '+                   arg_type=get_three_state_flag(return_label=True), deprecate_info=c.deprecate(expiration=\'3.0.0\'))',
        # options_list=[""] double quotes
        '    with self.argument_context(\'webapp update\') as c:',
        '+        c.argument(\'minimum_elastic_instance_count\', options_list=["--minimum-elastic-instance-count", "-i"], type=int, is_preview=True, help="Minimum number of instances. App must be in an elastic scale App Service Plan.")',
        '+        c.argument(\'prewarmed_instance_count\', options_list=["--prewarmed-instance-count", "-w"], type=int, is_preview=True, help="Number of preWarmed instances. App must be in an elastic scale App Service Plan.")',
        # self.argument_context with multi args
        '    with self.argument_context(\'appconfig kv import\', arg_group=\'File\') as c:',
        '+        c.argument(\'strict\', validator=validate_strict_import, arg_type=get_three_state_flag(), help="Delete all other key-values in the store with specified prefix and label", is_preview=True)',
        '    with self.argument_context(\'snapshot\', resource_type=ResourceType.MGMT_COMPUTE, operation_group=\'snapshots\') as c:',
        '+        c.argument(\'snapshot_name\', existing_snapshot_name, id_part=\'name\', completer=get_resource_name_completion_list(\'Microsoft.Compute/snapshots\'))',
    ]
    # pattern = r'\+\s+c.argument\((.*)\)?'
    for row_num, line in enumerate(lines):
        params, _ = search_argument(line)
        if params:
            cmds = search_argument_context(row_num, lines)
            # print(cmds)
            for cmd in cmds:
                parameters.append([cmd, params])
                continue
    pprint(parameters)
    assert parameters[0] == ['disk', ['--network-policy']]
    assert parameters[1] == ['disk', ['--zone']]
    assert parameters[2] == ['disk', ['--size-gb', '-z']]
    assert parameters[3] == ['snapshot', ['--size-gb', '-z']]
    assert parameters[4] == ['signalr create', ['--default-action']]
    assert parameters[5] == ['signalr update', ['--default-action']]
    assert parameters[6] == ['vmss run-command create', ['--vmss-name']]
    assert parameters[7] == ['vmss run-command update', ['--vmss-name']]
    assert parameters[8] == ['vm stop', ['--skip-shutdown']]
    assert parameters[9] == ['vmss stop', ['--skip-shutdown']]
    assert parameters[10] == ['acr connected-registry create', ['--client-tokens']]
    assert parameters[11] == ['acr connected-registry create', ['--notifications']]
    assert parameters[12] == ['webapp update', ['--skip-custom-domain-verification']]
    assert parameters[13] == ['webapp update', ['--minimum-elastic-instance-count', '-i']]
    assert parameters[14] == ['webapp update', ['--prewarmed-instance-count', '-w']]
    assert parameters[15] == ['appconfig kv import', ['--strict']]
    assert parameters[16] == ['snapshot', ['--snapshot-name']]

if __name__ == '__main__':
    test_one_line_regex()
    test_multiple_lines_regex()
    test_detect_new_command()
    test_detect_new_params()


#!/usr/bin/env python

# list of packages that should be imported for this code to work
import cobra.mit.access
import cobra.mit.session
import cobra.mit.request
import cobra.mit.naming
import yaml
import argparse
import warnings
warnings.filterwarnings("ignore")

def get_args():
    parser = argparse.ArgumentParser(description="Looking for a ports in filter. Return the contract using the filter")
    parser.add_argument('-p', dest='port', help='TCP/UDP port', type=int)
    args = parser.parse_args()
    return args

# since some of the port values are encoded as string I need to map them back to the port interger value
def port_to_int(port):
    port_map = {'unspecified':0, 'https':443, 'http':80, 'smtp':25, 'ftpData':21, 'dns':53, 'pop3':110,'rtsp':554}
    try:
        return int(port)
    except:
        return port_map[port]

# Get command line arguments
args = get_args()

# open yaml files with the credentials
f = open('credentials_camfab.yaml', 'r')
credentials = yaml.load(f)
f.close()

# log into an APIC and create a directory object
ls = cobra.mit.session.LoginSession(credentials['host'], credentials['user'], credentials['pass'], secure=False, timeout=180)
md = cobra.mit.access.MoDirectory(ls)
md.login()

#Qury the APIC for all the filters
q = cobra.mit.request.ClassQuery('vzFilter')
q.subtree = 'children'
# Run the query
filters = md.query(q)

flt_list = ""

#iterate over the filters
for filt in filters:
    # print filt.dn
    # "uni/tn-common/flt-arp/e-arp" ==> E-arp == Entry
    # iterate over the ACLs
    for acl in filt.e:
        # get the filter name withiht the "E" class preposition
        flt_name = filt.name.split('/e-')[0]
        if port_to_int(acl.sFromPort) <= args.port <= port_to_int(acl.sToPort):
            flt_list += flt_name + '|'
            break
        if port_to_int(acl.dFromPort) <= args.port <= port_to_int(acl.dToPort) :
            flt_list += flt_name + '|'

#Remove the trailing | character
flt_list = flt_list[:-1]

# Get all the contracts in the fabric
q = cobra.mit.request.ClassQuery('vzBrCP')
q.subtree = 'full'

# From the subtree return only the child that are of class vzRsSubjFiltAtt --> relationship between filter ans subject
q.subtreeClassFilter = 'vzRsSubjFiltAtt'

# Return only the Subjects that are using one of the filters that are matching the port we need.
# Basicaly we check that the DN of the vzRsSubjFiltAtt contains the name of the filter. That means that we do get back all the contracts
# however only the one that match our filter have a subject attached
q.subtreePropFilter = 'wcard(vzRsSubjFiltAtt.dn, "{}")'.format(flt_list)
# Run the query
contracts = md.query(q)

for contract in contracts:
    # If the returned contract has a subject
    if contract.subj:
        print("Contract {} is using the following filter matching the port".format(str(contract.dn).strip('/uni')))
        for subjects in contract.subj:
            for flt in subjects.rssubjFiltAtt:
                print("\t {}".format(str(flt.dn).strip('/uni')))
        print("\n")


"""
the simulation, simulates the motors and work in the same coordinate system.
"""
import pygame
import numpy as np
import time
import math
import matplotlib.pyplot as plt
import ArduinoCommunication as Ac
import Algorithmics as Algo

# ---------- CONSTANTS -------------
# COLORS
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
BLUE = (0, 0, 255)
RED = (255, 0, 0)
GREEN = (0, 255, 0)

# MECHANICS
SCREEN = (16, 12)  # dimensions of 10'' screen
ARMS = (15, 10)  # length of arm links in cm
d = 15  # distance from screen in cm
STEPS_PER_REVOLUTION = 200  # number of full steps to make a full round
STEPS_FRACTION = 8  # resolution of micro-stepping
MINIMAL_ANGLE = 2 * np.pi / (STEPS_PER_REVOLUTION * STEPS_FRACTION)  # the minimal angle step of the motor in rad (it is
BITS_PER_BYTE = 8
LENGTH_OF_COMMAND = 6  # how many chars are in one command

# TIME
WANTED_RPS = 0.5  # revolutions per second of motors
ONE_STEP_DELAY = 5.0 / WANTED_RPS / STEPS_FRACTION / 1000.0  # in sec
SERIAL_BPS = 19200  # bits per second the serial can read and write
WRITE_DELAY = 1.0/(SERIAL_BPS/BITS_PER_BYTE/LENGTH_OF_COMMAND)  # delay in sec after writing to prevent buffer overload
T = 1  # time of one slice in sec
dt_serial = WRITE_DELAY * 4  # time between 2 readings from serial in sec
dt_motor = ONE_STEP_DELAY * 4  # time of writing to the serial in sec
times_ideal = int(T / dt_motor)  # the size of the vectors for the simulation
times_serial = int(T / dt_serial)  # the amount of different values for the
TIME_TO_QUIT_SIMULATION = 2  # time to quit the simulation after finished in sec

first_point = 0, 0  # theta, phi - will be updated in the code
theta_simulation, phi_simulation = [], []

# ---------- ALGORITHMIC FUNCTION ---------------
def get_xy_by_t_line_acceleration(t):  # gets time in sec
    """
    example of trajectory function that tells the arm to make a straight line with acceleration in the beginning and
    the end of the line
    :param t: time
    :return: tuple (x, y)
    """
    acc = 1800.0
    x_0 = -SCREEN[0] / 2
    y_0 = 0.8 * SCREEN[1]
    d_a = SCREEN[0] / 4.0
    t_a = math.sqrt(2 * d_a / acc)
    v = acc * t_a

    x = x_0
    y = y_0

    if t < t_a:
        x = x_0 + 0.5 * acc * math.pow(t, 2)
    elif t_a < t < T - t_a:
        x = x_0 + d_a + v * (t - t_a)
    elif t > T - t_a:
        x = x_0 + d_a + v * (T - 2 * t_a) + v * (t - (T - t_a)) - 0.5 * acc * (t - (T - t_a)) ** 2
    return x, y

def get_xy_by_t_line_const_speed(t):  # gets time in sec
    """
    example of trajectory function that tells the arm to make a straight line with constant speed
    :param t: time
    :return: tuple (x, y)
    """
    x_0 = -SCREEN[0] / 2
    y_0 = 0.5 * SCREEN[1]

    x = x_0 + SCREEN[0] * t / T
    y = y_0

    return x, y

# ----------- PLOTS AND GRAPHS FUNCTIONS -----------
def plot_screen(screen):
    """
    plots the lines for the tablet screen in the pygame simulation
    :param screen: pygame screen object - pygame.display.set_mode((WIDTH, HEIGHT))
    """
    draw_line([SCREEN[0] / 2, d], [SCREEN[0] * 3 / 2, d], screen)
    draw_line([SCREEN[0] / 2, d + SCREEN[1]], [SCREEN[0] * 3 / 2, d + SCREEN[1]], screen)
    draw_line([SCREEN[0] / 2, d], [SCREEN[0] / 2, d + SCREEN[1]], screen)
    draw_line([SCREEN[0] * 3 / 2, d], [SCREEN[0] * 3 / 2, d + SCREEN[1]], screen)
    draw_circle([SCREEN[0], 0], 2, screen, RED)

def draw_line(start_pos, end_pos, screen):
    """
    draws a line in the pygame simulation
    :param start_pos: tuple (x, y)
    :param end_pos: tuple (x, y)
    :param screen: pygame screen object - pygame.display.set_mode((WIDTH, HEIGHT))
    """
    pygame.draw.line(screen, BLUE, [cm_to_pixels(start_pos[0]), cm_to_pixels(
        start_pos[1])], [cm_to_pixels(end_pos[0]), cm_to_pixels(end_pos[1])], 1)
    return

def draw_circle(pos, radius, screen, color):
    """
    draws a circle in the pygame simulation
    :param pos: tuple (x, y)
    :param radius: double
    :param screen: pygame screen object - pygame.display.set_mode((WIDTH, HEIGHT))
    :param color: color of circle
    """
    pygame.draw.circle(screen, color, [cm_to_pixels(pos[0]), cm_to_pixels(pos[1])],
                       radius, 0)

def cm_to_pixels(length):
    """
    returns the length in number of pixels
    :param length: double in cm
    :return: int - number of pixels
    """
    return int(20 * length)

def draw_graph(x, y, title, xlabel, ylabel):
    """
    draws a graph in matplotlib
    :param x: vector for x axis
    :param y: vector for y axis
    :param title: string for title
    :param xlabel: string for x axis lable
    :param ylabel: string for y axis lable
    """
    plt.plot(x, y)
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.show()

# width and height of the window of the pygame simulation
WIDTH = cm_to_pixels(2 * SCREEN[0])
HEIGHT = cm_to_pixels(2 * (SCREEN[1] + d))

# ------------- HELPING FUNCTIONS ------------
def modulo_by_1(num):
    """
    makes a modulo by 1 that returns a positive number for positive parameter and a negative number for a negative
    parameter
    :param num: double
    """
    if num > 0:
        return num % 1
    else:
        return num % 1 - 1

def xy_by_theta_phi(theta, phi):
    """
    returns the location in (x, y) according to the location in (theta, phi) by the trigonometric connection - this
    is the location of the pen (the end of the 2nd arm)
    :param theta: double
    :param phi: double
    :param x_0: double
    :return: tuple of doubles (x, y)
    """
    x = ARMS[0] * np.cos(theta) + ARMS[1] * np.cos(phi)
    y = ARMS[0] * np.sin(theta) + ARMS[1] * np.sin(phi) - d
    return x, y

def xy_by_theta(theta, x_0):
    """
    returns the location in (x, y) according to the location in (theta) by the trigonometric connection - this
    is the location of the link (the end of the 1st arm)
    :param theta: double
    :param x_0: double
    :return: tuple of doubles (x, y)
    """
    x = x_0 + ARMS[0] * np.cos(theta)
    y = ARMS[0] * np.sin(theta)
    return x, y

def unite_vector(a):
    united = []
    for i in a:
        united += i
    return united

def draw_points(points_to_draw):
    screen = pygame.display.set_mode((WIDTH, HEIGHT))
    x_0, y_0 = SCREEN[0], 0
    green = 0

    # quiting option
    event = pygame.event.poll()
    if event.type == pygame.QUIT:
        pygame.quit()

    # plotting the screen
    screen.fill(WHITE)
    plot_screen(screen)

    for point in points_to_draw:
        x_point = point[0]
        y_point = point[1]

        x_point_simulation, y_point_simulation = xy_board_to_xy_simulation(x_point, y_point)
        color = (255, green, 0)
        draw_circle([x_point_simulation, y_point_simulation], 2, screen, color)
        # change the color to be more blue
        green += 255 // len(points_to_draw)

    # display the simulation
    pygame.display.flip()

    # quiting the simulation
    time.sleep(TIME_TO_QUIT_SIMULATION)
    pygame.display.quit()

def xy_board_to_xy_simulation(x_board, y_board):
    return x_board + SCREEN[0], y_board + d

# -------------- SLICE TRAJECTORY ------------------
def make_slice_by_trajectory(points, invert=True):
    """
    Sends commands to Arduino according to the given route from the algorithmic module.
    :param points: list of tuples, each tuple is a point the arm should go through
    :param invert: if true then make also invert slice
    """
    steps_theta, steps_phi = list(), list()
    for i in range(len(points)-1):
        current_point = Ac.xy2angles(points[i])  # in (theta,phi)
        next_point = Ac.xy2angles(points[i+1])  # in (theta,phi)
        current_steps_theta, current_steps_phi = Ac.generate_steps_list(Ac.rad2steps(next_point[0] - current_point[0]),
                                                                     Ac.rad2steps(next_point[1] - current_point[1]))
        for j in range(len(current_steps_theta)):
            steps_theta.append(current_steps_theta[j])
            steps_phi.append(current_steps_phi[j])
    global first_point
    if len(points) != 0:
        first_point = Ac.xy2angles(points[0])
    move_2_motors_simulation(steps_theta, steps_phi)
    if invert:
        i_steps_theta, i_steps_phi = Ac.invert_slice(steps_theta, steps_phi)
        if len(points) != 0:
            first_point = Ac.xy2angles(points[-1])

        move_2_motors_simulation(i_steps_theta, i_steps_phi, True)

def duplicate_theta_and_phi_values_for_simulation(theta_vector, phi_vector):
    """
    returns theta vector and phi vector to show in simulation - with the dt of the simulation
    :param theta_vector: list of theta angles - double list
    :param phi_vector: list of phi angles - double list
    :param steps_theta: steps for the theta motor - int list
    :param steps_phi: steps for the phi motor - int list
    :return: tuple of 2 lists (theta vector, phi vector) in the right dt interval
    """
    factor = int(times_ideal/times_serial)
    theta_mul = [[angle]*factor for angle in theta_vector]
    phi_mul = [[angle]*factor for angle in phi_vector]
    theta_simulation = unite_vector(theta_mul)
    phi_simulation = unite_vector(phi_mul)

    return theta_simulation, phi_simulation

def get_theta_and_phi_vectors_by_steps_vectors(steps_theta, steps_phi):
    theta_vector = []
    phi_vector = []
    theta_vector.append(first_point[0])
    phi_vector.append(first_point[1])
    for i in range(len(steps_theta)):
        theta_vector.append(theta_vector[-1] + steps_theta[i] * MINIMAL_ANGLE)
        phi_vector.append(phi_vector[-1] + steps_phi[i] * MINIMAL_ANGLE)
    return theta_vector, phi_vector

def move_2_motors_simulation(steps_theta, steps_phi, inverse=False):
    global theta_simulation, phi_simulation
    theta_vector, phi_vector = get_theta_and_phi_vectors_by_steps_vectors(steps_theta, steps_phi)
    theta_simulation, phi_simulation = duplicate_theta_and_phi_values_for_simulation(theta_vector, phi_vector)

# ------------- FRUIT TRAJECTORY ------------------
def xy_by_fruit_trajectory(trajectory, total_time, dt):
    """
    returns vector of x and vector of y in the simulation dt interval in cm
    :param trajectory: function that gets double t and returns a tuple (x, y) of the fruit location by the estimated
    trajectory
    :param total_time: the total time of the fruit on the screen (from the moment that the trajectory was calculated)
    :param dt: the dt of the simulation
    :return: tuple of 2 lists (x of the fruit, y of the fruit) in the simulation dt interval in cm
    """
    # calculation of the right dt interval in order to show it in the simulation
    dt_trajectory = total_time / (T / dt)
    times = range(int(T / dt))
    x_fruit, y_fruit = [0 for _ in times], [0 for _ in times]
    for i in times:
        x_fruit[i], y_fruit[i] = trajectory(i * dt_trajectory)
        x_fruit[i], y_fruit[i] = Algo.algo_to_mech((x_fruit[i], y_fruit[i]))
        # # TODO the next 2 lines are a bit "fishy". check why it is necessary to add values for the conversion
        # x_fruit[i] += SCREEN[0] / 2
        # y_fruit[i] += d
    return x_fruit, y_fruit

def get_fruit_xy_vectors(fruits):
    def zero_trajectory(_):
        return 0, 0

    fruit_trajectories = [fruit.trajectory for fruit in fruits]

    # get the trajectory of the first fruit - (x,y) by t
    if len(fruit_trajectories) > 0:
        first_trajectory = []
        first_trajectory_total_time = []
        for i in range(len(fruit_trajectories)):
            # first_trajectory_object.append(fruit_trajectories[i])
            first_trajectory.append(fruit_trajectories[i].calc_trajectory())
            first_trajectory_total_time.append(fruit_trajectories[i].calc_life_time())
            #     # do not have to get into the 2 else down
            # else:
            #     first_trajectory = zero_trajectory
            #     first_trajectory_total_time = 1
    else:
        first_trajectory = []
        first_trajectory_total_time = 1

    xy_of_fruits_list = []
    for j in range(len(first_trajectory)):
        xy_of_fruits_list.append(xy_by_fruit_trajectory(first_trajectory[j],first_trajectory_total_time[j], dt_motor ))

    return xy_of_fruits_list

def init_multi():
    global SCREEN, d
    SCREEN = (12.0,8.0)
    d = 17.8

# ------------- CALCULATE LOCATIONS -------------
def run_simulation(points_to_go_through, fruits_sliced):
    """
    Runs the simulation.
    :param func: function of (x,y)(t), route of slice
    :param fruits_trajectories_and_starting_times:
    """
    # print("points to go through:")
    # print(points_to_go_through)
    # draw_points(points_to_go_through)

    global theta_simulation, phi_simulation

    make_slice_by_trajectory(points_to_go_through, False)

    xy_of_fruits_list = get_fruit_xy_vectors(fruits_sliced)

    # ------------- PLOT -------------------
    screen = pygame.display.set_mode((WIDTH, HEIGHT))
    x_0, y_0 = SCREEN[0], 0

    times = len(theta_simulation)

    x_practical_vector = [0 for _ in range(times)]
    y_practical_vector = [0 for _ in range(times)]

    # loop of plot
    for i in range(times):
        # quiting option
        event = pygame.event.poll()
        if event.type == pygame.QUIT:
            pygame.quit()

        # plotting the screen
        screen.fill(WHITE)
        plot_screen(screen)

        # draw fruits locations
        for k in range(len(xy_of_fruits_list)):
            x_fruit, y_fruit = xy_board_to_xy_simulation(xy_of_fruits_list[k][0][i], xy_of_fruits_list[k][1][i])
            draw_circle([x_fruit, y_fruit], 10, screen, GREEN)

        # real locations
        x_practical, y_practical = xy_by_theta_phi(theta_simulation[i], phi_simulation[i])
        x_practical, y_practical = xy_board_to_xy_simulation(x_practical, y_practical)
        x_practical_vector[i], y_practical_vector[i] = x_practical, y_practical
        x_link_practical, y_link_practical = xy_by_theta(theta_simulation[i], x_0)
        draw_circle([x_practical, y_practical], 2, screen, RED)
        draw_line([x_link_practical, y_link_practical], [x_0, y_0], screen)
        draw_line([x_link_practical, y_link_practical], [x_practical, y_practical], screen)

        # display the simulation
        pygame.display.flip()

        # sleep for the simulation dt (dt_motor is the simulation dt)
        time.sleep(dt_motor)

    # quiting the simulation
    time.sleep(TIME_TO_QUIT_SIMULATION)
    pygame.display.quit()

if __name__ == '__main__':
    for i in range(1):
        run_simulation([(0.6,0.0), (-7.0,4.0), (7.0,4.0), (0.6,0.0)], [])
        time.sleep(1)


from functools import lru_cache
import csv

@lru_cache
def read(path):
    data = []
    with open(path) as file:
        jobs_reader = csv.DictReader(file, delimiter=",", quotechar='"')
        for row in jobs_reader:
            data.append(row)
    return data


import pygame
import time

pygame.mixer.init()
pygame.init()
pygame.mixer.music.set_volume(1)
window = pygame.display.set_mode((1800, 450))
pygame.display.set_caption('Guitar -> Piano Visualizer')

#### Maybe use this later for adding text to pygame screen #####
font = pygame.font.Font('freesansbold.ttf', 30)
eText = font.render('RED : E', True, (255,0,0), (127,0,0))
eTextRect = eText.get_rect()
eTextRect.center = (150, 425)
window.blit(eText, eTextRect)

aText = font.render('ORANGE : A', True, (255,165,0), (127,84,0))
aTextRect = aText.get_rect()
aTextRect.center = (450, 425)
window.blit(aText, aTextRect)

dText = font.render('YELLOW : D', True, (255,255,0), (127,127,0))
dTextRect = dText.get_rect()
dTextRect.center = (750, 425)
window.blit(dText, dTextRect)

gText = font.render('GREEN : G', True, (0,255,0), (0,127,0))
gTextRect = gText.get_rect()
gTextRect.center = (1050, 425)
window.blit(gText, gTextRect)

bText = font.render('BLUE : B', True, (0,0,255), (0,0,127))
bTextRect = bText.get_rect()
bTextRect.center = (1350, 425)
window.blit(bText, bTextRect)

EText = font.render('PURPLE : High E', True, (255,0,255), (127,0,127))
ETextRect = EText.get_rect()
ETextRect.center = (1650, 425)
window.blit(EText, ETextRect)

#key colors
white = (255,255,255)
black = (0,0,0)

#used for calculations
scottBases = [7, 0, 5, 10, 2, 7]
willBases = [0, 5, 10, 15, 19, 24]

#active colors
red = (255,0,0)
orange = (255,165,0)
yellow = (255,255,0)
green = (0,255,0)
blue = (0,0,255)
purple = (255,0,255)
# used for coloring keyboard
activeColors = [red, orange, yellow, green, blue, purple]

#passive colors
red2 = (255/2,0,0)
orange2 = (255/2,165/2,0)
yellow2 = (255/2,255/2,0)
green2 = (0,255/2,0)
blue2 = (0,0,255/2)
purple2 = (255/2,0,255/2)
#used for coloring keyboard
passiveColors = [red2, orange2, yellow2, green2, blue2, purple2]

class Key:
    individualKeys = []
    def __init__(self, sharp, num, xPos, yPos, width, height, pianoSound, guitarSound):
        self.__class__.individualKeys.append(self)
        self.self = self
        self.sh = sharp
        self.n = num
        self.x = xPos
        self.y = yPos
        self.w = width
        self.h = height
        self.ps = pianoSound
        self.gs = guitarSound

    def whiten(self):
        if self.sh == 1:
            pygame.draw.rect(window, white, (self.x, self.y, self.w, self.h), 5)
        else:
            pygame.draw.rect(window, white, (self.x, self.y, self.w, self.h), 0)

    def activate(self, colornum):
        pygame.draw.rect(window, activeColors[colornum], (self.x, self.y, self.w, self.h), 0)

    def makePassive(self, colornum):
        pygame.draw.rect(window, passiveColors[colornum], (self.x, 0, self.w, 50), 0)
        pygame.draw.rect(window, black, (self.x, self.y, self.w, self.h), 0)
        pygame.draw.rect(window, white, (self.x, self.y, self.w, self.h), 5)
        #pygame.draw.rect(window, passiveColors[colornum], (self.x, self.y, self.w, self.h), 0)

    def displayNumber(self):
        font = pygame.font.Font('freesansbold.ttf', 15)
        numText = font.render(str(self.n), True, (255,255,255), (0,0,0))
        numTextRect = numText.get_rect()
        if self.sh == 1:
            numTextRect.center = ((self.x + 12.5), 360)
        else:
            numTextRect.center = (self.x + 25, 375)
        window.blit(numText, numTextRect)

    def playGuitarSound(self):
        pygame.mixer.Sound(self.gs).play()
        pygame.display.update()

### SET KEYS ###

e1 = Key(0,0,0,50,50,300,'e1.mp3','Ge1.mp3')
f1 = Key(0,1,50,50,50,300,'f1.mp3','Gf1.mp3')
fsharp1 = Key(1,2,100, 0,25,300,'fsharp1.mp3','Gfsharp1.mp3')
g1 = Key(0,3,125,50,50,300,'g1.mp3','Gg1.mp3')
gsharp1 = Key(1,4,175, 0,25,300,'gsharp1.mp3','Ggsharp1.mp3')
a1 = Key(0,5,200,50,50,300,'a1.mp3','Ga1.mp3')
asharp1 = Key(1,6,250, 0,25,300,'asharp1.mp3','Gasharp1.mp3')
b1 = Key(0,7,275,50,50,300,'b1.mp3','Gb1.mp3')
c1 = Key(0,8,325,50,50,300,'c1.mp3','Gc1.mp3')
csharp1 = Key(1,9,375, 0,25,300,'csharp1.mp3','Gcsharp1.mp3')
d1 = Key(0,10,400,50,50,300,'d1.mp3','Gd1.mp3')
dsharp1 = Key(1,11,450, 0,25,300,'dsharp1.mp3','Gdsharp1.mp3')
e2 = Key(0,12,475,50,50,300,'e2.mp3','Ge2.mp3')
f2 = Key(0,13,525,50,50,300,'f2.mp3','Gf2.mp3')
fsharp2 = Key(1,14,575, 0,25,300,'fsharp2.mp3','Gfsharp2.mp3')
g2 = Key(0,15,600,50,50,300,'g2.mp3','Gg2.mp3')
gsharp2 = Key(1,16,650, 0,25,300,'gsharp2.mp3','Ggsharp2.mp3')
a2 = Key(0,17,675,50,50,300,'a2.mp3','Ga2.mp3')
asharp2 = Key(1,18,725, 0,25,300,'asharp2.mp3','Gasharp2.mp3')
b2 = Key(0,19,750,50,50,300,'b2.mp3','Gb2.mp3')
c2 = Key(0,20,800,50,50,300,'c2.mp3','Gc2.mp3')
csharp2 = Key(1,21,850, 0,25,300,'csharp2.mp3','Gcsharp2.mp3')
d2 = Key(0,22,875,50,50,300,'d2.mp3','Gd2.mp3')
dsharp2 = Key(1,23,925, 0,25,300,'dsharp2.mp3','Gdsharp2.mp3')
e3 = Key(0,24,950,50,50,300,'e3.mp3','Ge3.mp3')
f3 = Key(0,25,1000,50,50,300,'f3.mp3','Gf3.mp3')
fsharp3 = Key(1,26,1050, 0,25,300,'fsharp3.mp3','Gfsharp3.mp3')
g3 = Key(0,27,1075,50,50,300,'g3.mp3','Gg3.mp3')
gsharp3 = Key(1,28,1125, 0,25,300,'gsharp3.mp3','Ggsharp3.mp3')
a3 = Key(0,29,1150,50,50,300,'a3.mp3','Ga3.mp3')
asharp3 = Key(1,30,1200, 0,25,300,'asharp3.mp3','Gasharp3.mp3')
b3 = Key(0,31,1225,50,50,300,'b3.mp3','Gb3.mp3')
c3 = Key(0,32,1275,50,50,300,'c3.mp3','Gc3.mp3')
csharp3 = Key(1,33,1325, 0,25,300,'csharp3.mp3','Gcsharp3.mp3')
d3 = Key(0,34,1350,50,50,300,'d3.mp3','Gd3.mp3')
dsharp3 = Key(1,35,1400, 0,25,300,'dsharp3.mp3','Gdsharp3.mp3')
e4 = Key(0,36,1425,50,50,300,'e4.mp3','Ge4.mp3')
f4 = Key(0,37,1475,50,50,300,'f4.mp3','Gf4.mp3')
fsharp4 = Key(1,38,1525, 0,25,300,'fsharp4.mp3','Gfsharp4.mp3')
g4 = Key(0,39,1550,50,50,300,'g4.mp3','Gg4.mp3')
gsharp4 = Key(1,40,1600, 0,25,300,'gsharp4.mp3','Ggsharp4.mp3')
a4 = Key(0,41,1625,50,50,300,'a4.mp3','Ga4.mp3')
asharp4 = Key(1,42,1675, 0,25,300,'asharp4.mp3','Gasharp4.mp3')
b4 = Key(0,43,1700,50,50,300,'b4.mp3','Gb4.mp3')
c4 = Key(0,44,1750,50,50,300,'c4.mp3','Gc4.mp3')

scottNotes = []
willNotes = []
totalNumberOfChords = 0

### RUN THIS TO ENTER FRETS AND SEE KEYS ###

def chooseNewChord():

    print("Hello!")
    stringOfFrets = input("For each string, say which fret is played, seperated by spaces (E A D G B E): ")
    print()
    frets = stringOfFrets.split()
    fretInts = map(int, frets)
    listFretInts = list(fretInts) #int list of fret nums

    for i in range(6):
        scottNotes.append(scottBases[i] + listFretInts[i])

    for i in range(6):
        if listFretInts[i] >= 0:
            willNotes.append(willBases[i] + listFretInts[i])
        else:
            willNotes.append(-1)

def displayCurrentchord():
    j = -1
    for i in range(44):
        if (Key.individualKeys[i].n in willNotes) and (Key.individualKeys[i].n >= 0):
            j += 1
            Key.individualKeys[i].activate(j)
            pygame.mixer.Sound(Key.individualKeys[i].ps).play()
            pygame.display.update()
            time.sleep(0.3)
    time.sleep(3)

    for i in range(44):
        if (Key.individualKeys[i].n in willNotes) and (Key.individualKeys[i].n >= 0):
            pygame.mixer.Sound(Key.individualKeys[i].gs).play()
            pygame.display.update()
            time.sleep(0.1)

    #### Maybe use this later for adding text to pygame screen #####
    font = pygame.font.Font('freesansbold.ttf', 30)
    eText = font.render('RED : E - ' + str(scottNotes[0]), True, (255,0,0), (127,0,0))
    eTextRect = eText.get_rect()
    eTextRect.center = (150, 425)
    window.blit(eText, eTextRect)

    aText = font.render('ORANGE : A - ' + str(scottNotes[1]), True, (255,165,0), (127,84,0))
    aTextRect = aText.get_rect()
    aTextRect.center = (450, 425)
    window.blit(aText, aTextRect)

    dText = font.render('YELLOW : D - ' + str(scottNotes[2]), True, (255,255,0), (127,127,0))
    dTextRect = dText.get_rect()
    dTextRect.center = (750, 425)
    window.blit(dText, dTextRect)

    gText = font.render('GREEN : G - ' + str(scottNotes[3]), True, (0,255,0), (0,127,0))
    gTextRect = gText.get_rect()
    gTextRect.center = (1050, 425)
    window.blit(gText, gTextRect)

    bText = font.render('BLUE : B - ' + str(scottNotes[4]), True, (0,0,255), (0,0,127))
    bTextRect = bText.get_rect()
    bTextRect.center = (1350, 425)
    window.blit(bText, bTextRect)

    EText = font.render('PURPLE : High E - ' + str(scottNotes[5]), True, (255,0,255), (127,0,127))
    ETextRect = EText.get_rect()
    ETextRect.center = (1650, 425)
    window.blit(EText, ETextRect)
    pygame.display.update()

def displayPreviousChord(listKeys):
    j = -1
    for i in range(44):
        if Key.individualKeys[i].n in listKeys:
            j += 1
            Key.individualKeys[i].makePassive(j)
    pygame.display.update()
    print('previous wills notez' , listKeys)

##########################################
#            Makes Chord Class           #
##########################################

class Chord:
    bank = []
    def __init__(self, listKeyInts):
        self.__class__.bank.append(self)
        self.noteList = listKeyInts
        self.e = listKeyInts[0]
        self.a = listKeyInts[1]
        self.d = listKeyInts[2]
        self.g = listKeyInts[3]
        self.b = listKeyInts[4]
        self.E = listKeyInts[5]

####################################
#### Initilize the White Piano #####
####################################

for i in range(45):
    Key.individualKeys[i].whiten()
    Key.individualKeys[i].displayNumber()
pygame.display.update()

def play():
    chooseNewChord()
    displayCurrentchord()
    yayNay = input("Start fresh? press 'e' to exit: ")

    if yayNay == 'e' :
        exit()
    else :
        print("u tried!")

play()


import cv2

print(cv2.__version__)

cam = cv2.VideoCapture(0)

while True:
    output1, frame = cam.read()

    greyFrame = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)

    cv2.imshow("CameraWindow", greyFrame)
    cv2.moveWindow("CameraWindow", 500,0)

    if cv2.waitKey(1) & 0xff == ord("q"):
        break

cam.release()


import os
import zipfile

if not os.path.isdir("dist"):
    os.mkdir("dist")

zf = zipfile.ZipFile("dist/gcp-dumper-function.zip", "w")
for dirname, subdirs, files in os.walk("libs"):
    if not dirname.endswith("__pycache__"):
        zf.write(dirname)
        for filename in files:
            zf.write(os.path.join(dirname, filename))

zf.write("gcp_dumper_function.py", "main.py")
zf.write("requirements.txt")
zf.write("indexGroup-by-provider.csv")

zf.close()


"""For each root element in a XML file, count how many lines in the element.
The print out a list of the elements and the number of lines they have.
Optionally pass a flag to sort the elements by length
"""

from lxml import etree
import argparse
import re
import os
from pathlib import Path

from lxml.etree import XMLParser
from typing import Dict

replaceRegex = re.compile(r'/xs:schema/xs:element\[.\]/')

def countElements(infile: str, sort: bool = False) -> Dict[str, int]:
    """Count number of lines for each root element.

    Args:
        infile: File to read in from
        sort: Whether or not to sort the element by size

    Returns:
        A dict of all of the root elements and the number of lines they have.

    """

    elementCounts: Dict[str, int] = {}
    # Load in the XSD file and then get the root elements.
    # Make sure the parser know to remove comments so they do not cause glitches
    schema: etree._ElementTree = etree.parse(infile)
    rootElements = list(schema.getroot())
    # Iterate backwards through all root element. (Backwards works better for some reason. )
    for rootElement in rootElements:
        asText = etree.tostring(rootElement, pretty_print=True)
        newLineCount = len(re.findall(br"\n", asText))
        try:
            elementCounts[rootElement.attrib['name']] = newLineCount
        except KeyError:
            pass

    if sort:
        elementCounts = {k: v for k, v in sorted(elementCounts.items(), key=lambda item: item[1])}
    return elementCounts

def main():
    """Function to run when called directly from CLI."""

    # Use the argparse library to process arguments from the command line.
    parser = argparse.ArgumentParser(description='Count lines in each root element')
    parser.add_argument("infile", type=str, help="Path to the file to read")
    parser.add_argument("-s", "--sorted", action='store_true', help="Sort by line number")
    args = parser.parse_args()

    elementCounts = countElements(args.infile, args.sorted)
    for key, value in elementCounts.items():
        print(f"{value}:", key)

    print(f'{sum(elementCounts.values())}: Total')

if __name__ == "__main__":
    main()