""" Module for PCA code"""
from __future__ import (print_function, absolute_import, division, unicode_literals)

import inspect
import numpy as np
from matplotlib import pyplot as plt

import pdb as debugger

from pypeit.core import qa

def func_vander(x, func, deg, minv=None, maxv=None):
    if func == "polynomial":
        return np.polynomial.polynomial.polyvander(x, deg)
    elif func == "legendre":
        if minv is None or maxv is None:
            if np.size(x) == 1:
                xmin, xmax = -1.0, 1.0
            else:
                xmin, xmax = np.min(x), np.max(x)
        else:
            xmin, xmax = minv, maxv
        xv = 2.0 * (x-xmin)/(xmax-xmin) - 1.0
        return np.polynomial.legendre.legvander(xv, deg)
    elif func == "chebyshev":
        if minv is None or maxv is None:
            if np.size(x) == 1:
                xmin, xmax = -1.0, 1.0
            else:
                xmin, xmax = np.min(x), np.max(x)
        else:
            xmin, xmax = minv, maxv
        xv = 2.0 * (x-xmin)/(xmax-xmin) - 1.0
        return np.polynomial.chebyshev.chebvander(xv, deg)
    else:
        print("Fitting function '{0:s}' is not implemented yet. Please choose from 'polynomial', 'legendre', 'chebyshev'.")

def robust_polyfit(xarray, yarray, order, weights=None, maxone=True, sigma=3.0,
                   function="polynomial", initialmask=None, forceimask=False,
                   minv=None, maxv=None, guesses=None, bspline_par=None, verbose=True):
    """
    A robust (equally weighted) polynomial fit is performed to the xarray, yarray pairs
    mask[i] = 1 are masked values

    :param xarray: independent variable values
    :param yarray: dependent variable values
    :param order: the order of the polynomial to be used in the fitting
    :param weights: weights to be used in the fitting (weights = 1/sigma)
    :param maxone: If True, only the most deviant point in a given iteration will be removed
    :param sigma: confidence interval for rejection
    :param function: which function should be used in the fitting (valid inputs: 'polynomial', 'legendre', 'chebyshev', 'bspline')
    :param initialmask: a mask can be supplied as input, these values will be masked for the first iteration. 1 = value masked
    :param forceimask: if True, the initialmask will be forced for all iterations
    :param minv: minimum value in the array (or the left limit for a legendre/chebyshev polynomial)
    :param maxv: maximum value in the array (or the right limit for a legendre/chebyshev polynomial)
    :return: mask, ct -- mask is an array of the masked values, ct is the coefficients of the robust polyfit.
    """

    # Setup the initial mask
    if initialmask is None:
        mask = np.zeros(xarray.size, dtype=np.int)
        if forceimask:
            print("Initial mask cannot be enforced -- no initital mask supplied")
            forceimask = False
    else:
        mask = initialmask.copy()
    mskcnt = np.sum(mask)
    # Iterate, and mask out new values on each iteration
    ct = guesses
    while True:
        w = np.where(mask == 0)
        xfit = xarray[w]
        yfit = yarray[w]
        if weights is not None:
            wfit = weights[w]
        else:
            wfit = None
        ct = func_fit(xfit, yfit, function, order, w=wfit,
                      guesses=ct, minv=minv, maxv=maxv, bspline_par=bspline_par)
        yrng = func_val(ct, xarray, function, minv=minv, maxv=maxv)
        sigmed = 1.4826*np.median(np.abs(yfit-yrng[w]))
        if xarray.size-np.sum(mask) <= order+2:
            if verbose:
                print("More parameters than data points - fit might be undesirable")
            break  # More data was masked than allowed by order
        if maxone:  # Only remove the most deviant point
            tst = np.abs(yarray[w]-yrng[w])
            m = np.argmax(tst)
            if tst[m] > sigma*sigmed:
                mask[w[0][m]] = 1
        else:
            if forceimask:
                w = np.where((np.abs(yarray-yrng) > sigma*sigmed) | (initialmask==1))
            else:
                w = np.where(np.abs(yarray-yrng) > sigma*sigmed)
            mask[w] = 1
        if mskcnt == np.sum(mask): break  # No new values have been included in the mask
        mskcnt = np.sum(mask)
    # Final fit
    w = np.where(mask == 0)
    xfit = xarray[w]
    yfit = yarray[w]
    if weights is not None:
        wfit = weights[w]
    else:
        wfit = None
    ct = func_fit(xfit, yfit, function, order, w=wfit, minv=minv, maxv=maxv, bspline_par=bspline_par)
    return mask, ct

def func_val(coeff, x, func, minv=None, maxv=None):
    """ Generic routine to return an evaluated function
    Functional forms include:
      polynomial, legendre, chebyshev, bspline, gauss

    Parameters
    ----------
    coeff : ndarray, coefficients
    x :
    func : desired functional form
    minv
    maxv

    Returns
    -------
    values : ndarray

    """
    if func == "polynomial":
        return np.polynomial.polynomial.polyval(x, coeff)
    elif func == "legendre":
        if minv is None or maxv is None:
            if np.size(x) == 1:
                xmin, xmax = -1.0, 1.0
            else:
                xmin, xmax = np.min(x), np.max(x)
        else:
            xmin, xmax = minv, maxv
        xv = 2.0 * (x-xmin)/(xmax-xmin) - 1.0
        return np.polynomial.legendre.legval(xv, coeff)
    elif func == "chebyshev":
        if minv is None or maxv is None:
            if np.size(x) == 1:
                xmin, xmax = -1.0, 1.0
            else:
                xmin, xmax = np.min(x), np.max(x)
        else:
            xmin, xmax = minv, maxv
        xv = 2.0 * (x-xmin)/(xmax-xmin) - 1.0
        return np.polynomial.chebyshev.chebval(xv, coeff)
    elif func == "bspline":
        return interpolate.splev(x, coeff, ext=1)
    elif func == "gaussian":
        if len(coeff) == 2:
            return gauss_2deg(x, coeff[0], coeff[1])
        elif len(coeff) == 3:
            return gauss_3deg(x, coeff[0], coeff[1], coeff[2])
        else:
            print("Not ready for this type of gaussian")
    elif func == "moffat":
        if len(coeff) == 3:
            return moffat(x, coeff[0], coeff[1], coeff[2])
        else:
            print("Not ready for this type of Moffat")
    else:
        print("Fitting function '{0:s}' is not implemented yet. Please choose from 'polynomial', 'legendre', 'chebyshev', 'bspline'.")

def func_fit(x, y, func, deg, minv=None, maxv=None, w=None, guesses=None,
             bspline_par=None, return_errors=False):
    """ General routine to fit a function to a given set of x,y points

    Parameters
    ----------
    x : ndarray
    y : ndarray
    func : str
      polynomial, legendre, chebyshev, bspline, gaussian
    deg : int
      degree of the fit
    minv : float, optional
    maxv
    w
    guesses : tuple
    bspline_par : dict
      Passed to bspline_fit()

    Returns
    -------
    coeff : ndarray or tuple
      ndarray for standard function fits
      tuple for bspline

    """
    if func == "polynomial":
        return np.polynomial.polynomial.polyfit(x, y, deg, w=w)
    elif func == "legendre":
        if minv is None or maxv is None:
            if np.size(x) == 1:
                xmin, xmax = -1.0, 1.0
            else:
                xmin, xmax = np.min(x), np.max(x)
        else:
            xmin, xmax = minv, maxv
        xv = 2.0 * (x-xmin)/(xmax-xmin) - 1.0
        return np.polynomial.legendre.legfit(xv, y, deg, w=w)
    elif func == "chebyshev":
        if minv is None or maxv is None:
            if np.size(x) == 1:
                xmin, xmax = -1.0, 1.0
            else:
                xmin, xmax = np.min(x), np.max(x)
        else:
            xmin, xmax = minv, maxv
        xv = 2.0 * (x-xmin)/(xmax-xmin) - 1.0
        return np.polynomial.chebyshev.chebfit(xv, y, deg, w=w)
    elif func == "bspline":
        if bspline_par is None:
            bspline_par = {}
        # TODO -- Deal with this kwargs-like kludge
        return bspline_fit(x, y, order=deg, w=w, **bspline_par)
    elif func in ["gaussian"]:
        # Guesses
        if guesses is None:
            ampl, cent, sigma = guess_gauss(x, y)
            # As first guess choose slope and intercept to be zero
            b = 0
            m = 0
        else:
            if deg == 2:
                ampl, sigma = guesses
            elif deg == 3:
                ampl, cent, sigma = guesses
            elif deg == 4:
                b, ampl, cent, sigma = guesses
            elif deg == 5:
                m, b, ampl, cent, sigma = guesses
        # Error
        if w is not None:
            sig_y = 1./w
        else:
            sig_y = None
        if deg == 2:  # 2 parameter fit
            popt, pcov = curve_fit(gauss_2deg, x, y, p0=[ampl, sigma], sigma=sig_y)
        elif deg == 3:  # Standard 3 parameters
            popt, pcov = curve_fit(gauss_3deg, x, y, p0=[ampl, cent, sigma],
                                   sigma=sig_y)
        elif deg == 4:  # 4 parameters
            popt, pcov = curve_fit(gauss_4deg, x, y, p0=[b, ampl, cent, sigma],sigma=sig_y)
        elif deg == 5:  # 5 parameters
            popt, pcov = curve_fit(gauss_5deg, x, y, p0=[m, b, ampl, cent, sigma],sigma=sig_y)
        else:
            print("Not prepared for deg={:d} for Gaussian fit".format(deg))
        # Return
        if return_errors:
            return popt, pcov
        else:
            return popt
    elif func in ["moffat"]:
        # Guesses
        if guesses is None:
            ampl, cent, sigma = guess_gauss(x, y)
            p0 = ampl
            p2 = 3. # Standard guess
            p1 = (2.355*sigma)/(2*np.sqrt(2**(1./p2)-1))
        else:
            p0,p1,p2 = guesses
        # Error
        if w is not None:
            sig_y = 1./w
        else:
            sig_y = None
        if deg == 3:  # Standard 3 parameters
            popt, pcov = curve_fit(moffat, x, y, p0=[p0,p1,p2], sigma=sig_y)
        else:
            print("Not prepared for deg={:d} for Moffat fit".format(deg))
        # Return
        return popt
    else:
        print("Fitting function '{0:s}' is not implemented yet. Please choose from 'polynomial', 'legendre', 'chebyshev','bspline'.")

def basis(xfit, yfit, coeff, npc, pnpc, weights=None, skipx0=True, x0in=None, mask=None,
          function='polynomial'):
    nrow = xfit.shape[0]
    ntrace = xfit.shape[1]
    if x0in is None:
        x0in = np.arange(float(ntrace))

    # Mask out some orders if they are bad
    if mask is None or mask.size == 0:
        usetrace = np.arange(ntrace)
        outmask = np.ones((nrow, ntrace))
    else:
        usetrace = np.where(np.in1d(np.arange(ntrace), mask) == False)[0]
        outmask = np.ones((nrow, ntrace))
        outmask[:,mask] = 0.0

    # Do the PCA analysis
    eigc, hidden = get_pc(coeff[1:npc+1, usetrace], npc)

    modl = func_vander(xfit[:,0], function, npc)
    eigv = np.dot(modl[:,1:], eigc)

    med_hidden = np.median(hidden, axis=1)
    med_highorder = med_hidden.copy()
    med_highorder[0] = 0

    high_order_matrix = med_highorder.T[np.newaxis,:].repeat(ntrace, axis=0)

    coeffstr = []
    for i in range(1, npc+1):
        if weights is not None:
            tmask, coeff0 = robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i],
                                                   weights=weights[usetrace], sigma=2.0, function=function,
                                                   minv=x0in[0], maxv=x0in[-1])
        else:
            tmask, coeff0 = robust_polyfit(x0in[usetrace], hidden[i-1, :], pnpc[i],
                                                   sigma=2.0, function=function,
                                                   minv=x0in[0], maxv=x0in[-1])
        coeffstr.append(coeff0)
        high_order_matrix[:, i-1] = func_val(coeff0, x0in, function, minv=x0in[0], maxv=x0in[-1])
    high_fit = high_order_matrix.copy()

    high_order_fit = np.dot(eigv, high_order_matrix.T)
    sub = (yfit - high_order_fit) * outmask

    numer = np.sum(sub, axis=0)
    denom = np.sum(outmask, axis=0)
    x0 = np.zeros(ntrace, dtype=np.float)
    fitmask = np.zeros(ntrace, dtype=np.float)
    x0fit = np.zeros(ntrace, dtype=np.float)
    chisqnu = 0.0
    chisqold = 0.0
    robust = True
    if not skipx0:
        fitmask = (np.abs(denom) > 5).astype(np.int)
        if robust:
            good = np.where(fitmask != 0)[0]
            bad = np.where(fitmask == 0)[0]
            x0[good] = numer[good]/denom[good]
            imask = np.zeros(ntrace, dtype=np.float)
            imask[bad] = 1.0
            ttmask, x0res = robust_polyfit(x0in, x0, pnpc[0], weights=weights, sigma=2.0,
                                                   function=function, minv=x0in[0], maxv=x0in[-1], initialmask=imask)
            x0fit = func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
            good = np.where(ttmask == 0)[0]
            xstd = 1.0  # This should represent the dispersion in the fit
            chisq = ((x0[good]-x0fit[good])/xstd)**2.0
            chisqnu = np.sum(chisq)/np.sum(1-ttmask)
            fitmask = 1.0-ttmask
            print("Reduced chi-squared = {0:E}".format(chisqnu))
        else:
            for i in range(1, 5):
                good = np.where(fitmask != 0)[0]
                x0[good] = numer[good]/denom[good]
                x0res = func_fit(x0in[good], x0[good], function, pnpc[0],
                                         weights=weights, minv=x0in[0], maxv=x0in[-1])
                x0fit = func_val(x0res, x0in, function, minv=x0in[0], maxv=x0in[-1])
                chisq = (x0[good]-x0fit[good])**2.0
                fitmask[good] *= (chisq < np.sum(chisq)/2.0).astype(np.int)
                chisqnu = np.sum(chisq)/np.sum(fitmask)
                print("  Reduced chi-squared = {0:E}".format(chisqnu))
                if chisqnu == chisqold:
                    break
                else:
                    chisqold = chisqnu
        if chisqnu > 2.0:
            print("PCA has very large residuals")
        elif chisqnu > 0.5:
            print("PCA has fairly large residuals")

    else:
        x0res = 0.0
    x3fit = np.dot(eigv,high_order_matrix.T) + np.outer(x0fit,np.ones(nrow)).T
    outpar = dict({'high_fit': high_fit, 'x0': x0, 'x0in': x0in, 'x0fit': x0fit, 'x0res': x0res, 'x0mask': fitmask,
                   'hidden': hidden, 'usetrc': usetrace, 'eigv': eigv, 'npc': npc, 'coeffstr': coeffstr})
    return x3fit, outpar

def do_pca(data, cov=False):
    tolerance = 1.0E-5
    Nobj, Mattr = data.shape

    if cov:
        colmean = (np.sum(data,0)/Nobj)[:,np.newaxis]
        temp = np.ones((Nobj,1))
        X = data - np.dot(temp,colmean.T)
    else:
        print("PCA without cov=True is not implemented. Unable to continue")
    A = np.dot(X.T,X)
    eigva, eigve = np.linalg.eig(A)
    indx = np.where(np.abs(eigva) <= tolerance*np.max(eigva))[0]
    if np.size(indx) != 0: eigva[indx] = 0.0
    indx = np.where(np.abs(eigve) <= tolerance*np.max(eigve))[0]
    if np.size(indx) != 0: eigve[indx] = 0.0

    # Sort by increasing eigenvalue
    indx = np.argsort(eigva)
    eigva = eigva[indx]
    eigve = eigve[:,indx]
    eigva = eigva[::-1]
    eigve = eigve.T[::-1,:]

    return eigva, eigve

def extrapolate(outpar, ords, function='polynomial'):
    nords = ords.size

    x0ex = func_val(outpar['x0res'], ords, function,
                            minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])

    # Order centre
    high_matr = np.zeros((nords, outpar['npc']))
    for i in range(1, outpar['npc']+1):
        if outpar['coeffstr'][i-1][0] == -9.99E9:
            high_matr[:,i-1] = np.ones(nords)*outpar['high_fit'][0,i-1]
            continue
        high_matr[:,i-1] = func_val(outpar['coeffstr'][i-1], ords, function,
                                            minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
    extfit = np.dot(outpar['eigv'], high_matr.T) + np.outer(x0ex, np.ones(outpar['eigv'].shape[0])).T
    outpar['high_matr'] = high_matr
    return extfit, outpar

def refine_iter(outpar, orders, mask, irshft, relshift, fitord, function='polynomial'):
    fail = False
    x0ex = func_val(outpar['x0res'], orders, function,  minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
    # Make the refinement
    x0ex[irshft] += relshift
    # Refit the data to improve the refinement
    good = np.where(mask != 0.0)[0]
    null, x0res = robust_polyfit(orders[good], x0ex[good], fitord, sigma=2.0, function=function,
                                         minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
    x0fit = func_val(x0res, orders, function, minv=outpar['x0in'][0], maxv=outpar['x0in'][-1])
    chisq = (x0ex[good]-x0fit[good])**2.0
    chisqnu = np.sum(chisq)/np.sum(mask)
    print("  Reduced chi-squared = {0:E}".format(chisqnu))
    if chisqnu > 0.5: # The refinement went wrong, ignore the refinement
        fail = True
    outpar['x0res'] = x0res
    extfit = np.dot(outpar['eigv'], outpar['high_matr'].T) + np.outer(x0fit, np.ones(outpar['eigv'].shape[0])).T
    return extfit, outpar, fail

def get_pc(data, k, tol=0.0, maxiter=20, nofix=False, noortho=False):

    p = data.shape[0]
    if p == 0:
        print("You need to supply more components in the PCA")
    #n = np.size(data)/p
    if k > p:
        debugger.set_trace()
        print("The number of principal components must be less than or equal to the order of the fitting function")

    # Set the initial conditions
    eigv = np.zeros((p, k))
    eigv[:k, :k] = np.identity(k)

    niter = 0
    diff = tol*2.0 + 1.0

    while (niter < maxiter) and (diff > tol):
        hidden = np.dot(np.dot(np.linalg.inv(np.dot(eigv.T, eigv)), eigv.T), data)
        oldeigv = eigv.copy()
        eigv = np.dot(data, np.dot(hidden.T, np.linalg.inv(np.dot(hidden, hidden.T))))
        if tol > 0.0:
            diff = 0.0
            for i in range(k):
                diff += np.abs( 1.0 - np.sum(oldeigv[:,i]*eigv[:,i])/np.sqrt(np.sum(oldeigv[:,i]**2)*np.sum(eigv[:,i]**2)) )
        niter += 1

    # Orthonormalize?
    if not noortho:
        for b in range(k):
            # Orthogonalize
            for bp in range(b):
                dot = np.sum(eigv[:,b]*eigv[:,bp])
                eigv[:,b] -= dot*eigv[:,bp]
            # Normalize
            dot = np.sum(eigv[:,b]**2)
            dot = 1.0/np.sqrt(dot)
            eigv[:,b] *= dot
        # Project variables onto new coordinates?
        if not nofix:
            hidden = np.dot(eigv.T, data)
            eval_hidden, evec_hidden = do_pca(hidden.T, cov=True)
            eigv = np.dot(eigv, evec_hidden.T)
        hidden = np.dot(eigv.T, data)
    return eigv, hidden

################################################
# 2D Image PCA

def image_basis(img, numpc=0):
    """
    img is a masked array
    numpc is the number of principal components
    """
    # Compute the eigenvalues and eigenvectors of the covariance matrix
    # First, subtract the median (along the spatial direction)
    #imgmed = (img.data - img.mean(axis=1).data.reshape((img.data.shape[0],1))).T
    imgmed = (img.data - np.median(img.data,axis=1).reshape((img.data.shape[0],1))).T
    imgmed[np.where(img.mask.T)] = 0.0
    eigval, eigvec = np.linalg.eig(np.cov(imgmed))
    p = np.size(eigvec, axis=1)
    # Sort the eigenvalues in ascending order
    idx = np.argsort(eigval,kind='mergesort')
    idx = idx[::-1]
    # Sort eigenvectors according to the sorted eigenvalues
    eigvec = eigvec[:, idx]
    eigval = eigval[idx]
    # Select the first few principal components
    if (numpc < p) and (numpc >= 0):
        eigvec = eigvec[:,range(numpc)]
    # Project the data
    imgmed[np.where(img.mask.T)] = 0.0
    score = np.dot(eigvec.T, imgmed)
    return eigvec, eigval, score

def pca2d(img, numpc):
    # Compute eigenvalues and eigenvectors of covariance matrix
    imgsub = (img-np.mean(img.T, axis=1)).T  # subtract the mean (along a column)
    [latent, coeff] = np.linalg.eig(np.cov(imgsub))
    p = coeff.shape[1]
    # Sort the eigenvalues/vectors in ascending order
    idx = np.argsort(latent)
    idx = idx[::-1]
    coeff = coeff[:, idx]
    if numpc < p:
        coeff = coeff[:, range(numpc)]
    # projection of the data in the new space
    proj = np.dot(coeff.T, imgsub)
    # Reconstruct the image
    imgpca = np.dot(coeff, proj).T + np.mean(img, axis=0)
    return imgpca.astype(np.float)

def pca_plot(inpar, ofit, outroot, maxp=25, pcadesc="", addOne=True,
             show=False):
    """ Saves quality control plots for a PCA analysis
    Parameters
    ----------
    inpar
    ofit
    prefix : str
      prefix for the filenames
    maxp
    pcadesc
    addOne

    Returns
    -------

    """

    plt.rcdefaults()
    #plt.rcParams['font.family']= 'times new roman'

    npc = inpar['npc']+1
    pages, npp = qa.get_dimen(npc, maxp=maxp)
    #
    x0 = inpar['x0']
    ordernum = inpar['x0in']
    x0fit = inpar['x0fit']
    usetrc = inpar['usetrc']
    hidden = inpar['hidden']
    high_fit = inpar['high_fit']
    nc = np.max(ordernum[usetrc])
    # Loop through all pages and plot the results
    ndone = 0
    for i in range(len(pages)):
        plt.clf()
        f, axes = plt.subplots(pages[i][1], pages[i][0])
        ipx, ipy = 0, 0
        if i == 0:
            if pages[i][1] == 1: ind = (0,)
            elif pages[i][0] == 1: ind = (0,)
            else: ind = (0,0)
            axes[ind].plot(ordernum[usetrc], x0[usetrc], 'bx')
            axes[ind].plot(ordernum, x0fit, 'k-')
            amn, amx = np.min(x0fit), np.max(x0fit)
            diff = x0[usetrc]-x0fit[usetrc]
            tdiffv = np.median(diff)
            mdiffv = 1.4826*np.median(np.abs(tdiffv-diff))
            amn -= 2.0*mdiffv
            amx += 2.0*mdiffv
            mval = amn-0.15*(amx-amn)
            dmin, dmax = tdiffv-2.0*mdiffv, tdiffv+2.0*mdiffv
            diff = mval + diff*0.20*(amx-amn)/(dmax-dmin)
            wign = np.where(np.abs(diff-np.median(diff))<4.0*1.4826*np.median(np.abs(diff-np.median(diff))))[0]
            dmin, dmax = np.min(diff[wign]), np.max(diff[wign])
            axes[ind].plot(ordernum[usetrc], diff, 'rx')
            if addOne:
                axes[ind].plot([0, nc+1], [mval,mval], 'k-')
                axes[ind].axis([0, nc+1, dmin-0.5*(dmax-dmin), amx + 0.05*(amx-amn)])
            else:
                axes[ind].plot([0, nc], [mval, mval], 'k-')
                axes[ind].axis([0, nc, dmin-0.5*(dmax-dmin), amx + 0.05*(amx-amn)])
            axes[ind].set_title("Mean Value")
            ipx += 1
            if ipx == pages[i][0]:
                ipx = 0
                ipy += 1
            npp[0] -= 1
        for j in range(npp[i]):
            if pages[i][1] == 1: ind = (ipx,)
            elif pages[i][0] == 1: ind = (ipy,)
            else: ind = (ipy, ipx)
            axes[ind].plot(ordernum[usetrc], hidden[j+ndone,:], 'bx')
            axes[ind].plot(ordernum, high_fit[:,j+ndone], 'k-')
            vmin, vmax = np.min(hidden[j+ndone,:]), np.max(hidden[j+ndone,:])
            if ofit[1+j+ndone] != -1:
                cmn, cmx = np.min(high_fit[:,j+ndone]), np.max(high_fit[:,j+ndone])
                diff = hidden[j+ndone,:]-high_fit[:,j+ndone][usetrc]
                tdiffv = np.median(diff)
                mdiffv = 1.4826*np.median(np.abs(tdiffv-diff))
                cmn -= 2.0*mdiffv
                cmx += 2.0*mdiffv
                mval = cmn-0.15*(cmx-cmn)
                dmin, dmax = tdiffv-2.0*mdiffv, tdiffv+2.0*mdiffv
                #dmin, dmax = np.min(diff), np.max(diff)
                diff = mval + diff*0.20*(cmx-cmn)/(dmax-dmin)
                wign = np.where(np.abs(diff-np.median(diff))<4.0*1.4826*np.median(np.abs(diff-np.median(diff))))[0]
                dmin, dmax = np.min(diff[wign]), np.max(diff[wign])
                #vmin, vmax = np.min(hidden[j+ndone,:][wign]), np.max(hidden[j+ndone,:][wign])
                axes[ind].plot(ordernum[usetrc], diff, 'rx')
                axes[ind].plot([0, 1+nc], [mval, mval], 'k-')
#				ymin = np.min([(3.0*dmin-dmax)/2.0,vmin-0.1*(vmax-dmin),dmin-0.1*(vmax-dmin)])
#				ymax = np.max([np.max(high_fit[:,j+ndone]),vmax+0.1*(vmax-dmin),dmax+0.1*(vmax-dmin)])
                ymin = dmin-0.5*(dmax-dmin)
                ymax = cmx + 0.05*(cmx-cmn)
                if addOne: axes[ind].axis([0, nc+1, ymin, ymax])
                else: axes[ind].axis([0, nc, ymin, ymax])
            else:
                if addOne: axes[ind].axis([0, nc+1, vmin-0.1*(vmax-vmin), vmax+0.1*(vmax-vmin)])
                else: axes[ind].axis([0, nc, vmin-0.1*(vmax-vmin), vmax+0.1*(vmax-vmin)])
            axes[ind].set_title("PC {0:d}".format(j+ndone))
            axes[ind].tick_params(labelsize=8)
            ipx += 1
            if ipx == pages[i][0]:
                ipx = 0
                ipy += 1
        if i == 0: npp[0] = npp[0] + 1
        # Delete the unnecessary axes
        for j in range(npp[i], axes.size):
            if pages[i][1] == 1: ind = (ipx,)
            elif pages[i][0] == 1: ind = (ipy,)
            else: ind = (ipy, ipx)
            f.delaxes(axes[ind])
            ipx += 1
            if ipx == pages[i][0]:
                ipx = 0
                ipy += 1
        ndone += npp[i]
        # Save the figure
        if pages[i][1] == 1 or pages[i][0] == 1: ypngsiz = 11.0/axes.size
        else: ypngsiz = 11.0*axes.shape[0]/axes.shape[1]
        f.set_size_inches(11.0, ypngsiz)
        if pcadesc != "":
            pgtxt = ""
            if len(pages) != 1:
                pgtxt = ", page {0:d}/{1:d}".format(i+1, len(pages))
            f.suptitle(pcadesc + pgtxt, y=1.02, size=16)
        f.tight_layout()
        if show:
            plt.show()
        else:
            outfile = outroot+'.pdf'.format(i)
            f.savefig(outfile, dpi=200)
        plt.close()
        f.clf()
    del f

    plt.rcdefaults()

    return


from pathlib import Path

import optuna
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm

from ser_data.crema_data import CremaAudioDataset
from ser_model.cnn_lstm_model import CNNLSTM2DModel
from ser_train.ser_trainer import SpeechEmotionRecognitionTrainer

NUM_BOOTSTRAP_SAMPLES = 100
CONFIG = {
    'data_path_train': 'Data/MelSpecSplit/train',
    'data_path_val': 'Data/MelSpecSplit/val',
    'demographics_csv_path': 'Data/VideoDemographics.csv',
    'ratings_csv_path': 'Data/processedResults/summaryTable.csv',

    'save_path': 'models/cnn_lstm_attention_multitask',
    'log_dir': 'runs/cnn_lstm_attention_multitask',

    'bootstrap_sampling': False,
    'use_ratings': False,
    'use_gender_label': True,
    'use_race_label': True,

    'num_epochs': 100,
    'steps_per_log': 50,
    'epochs_per_eval': 5,
    'batch_size': 64,
    'num_workers': 4,
    'learning_rate': 5e-4,
    'weight_decay': 1e-5,

    'lstm_hidden_size': 64,
    'use_self_attention': True,
    'self_attention_size': 128,
    'num_self_attention_heads': 8,
    'dropout_rate': 0.1,
    'label_smoothing': 0.1,

    'random_seed': 0
}

def load_data(
        data_path,
        demographics_csv_path,
        ratings_csv_path,
        bootstrap_sampling=False,
        out_of_bootstrap=False,
        batch_size=32,
        num_workers=4,
        shuffle=True,
        random_seed=0):
    dataset = CremaAudioDataset(
        data_path,
        demographics_csv_path,
        ratings_csv_path,
        bootstrap_sampling=bootstrap_sampling,
        out_of_bootstrap=out_of_bootstrap,
        random_seed=random_seed)
    data_loader = DataLoader(
        dataset,
        batch_size=batch_size,
        shuffle=shuffle,
        num_workers=num_workers,
        pin_memory=True,
        prefetch_factor=2,
        persistent_workers=True)
    return data_loader

def load_model(
        lstm_hidden_size=128,
        use_self_attention=False,
        self_attention_size=128,
        num_self_attention_heads=8,
        use_gender_label=False,
        use_race_label=False,
        dropout_rate=0.1,
        label_smoothing=0.1):
    output_size = 6
    if use_gender_label:
        output_size += 1
    if use_race_label:
        output_size += 1

    model = CNNLSTM2DModel(
        output_size=output_size,
        lstm_hidden_size=lstm_hidden_size,
        use_self_attention=use_self_attention,
        self_attention_size=self_attention_size,
        num_self_attention_heads=num_self_attention_heads,
        dropout_rate=dropout_rate,
        label_smoothing=label_smoothing)
    return model

def train(
        data_path_train,
        data_path_val,
        demographics_csv_path,
        ratings_csv_path,
        save_path,
        log_dir=None,
        bootstrap_sampling=False,
        use_ratings=False,
        use_gender_label=False,
        use_race_label=False,
        num_epochs=20,
        steps_per_log=100,
        epochs_per_eval=5,
        batch_size=32,
        num_workers=4,
        learning_rate=1e-3,
        weight_decay=1e-5,
        lstm_hidden_size=128,
        use_self_attention=False,
        self_attention_size=128,
        num_self_attention_heads=8,
        dropout_rate=0.1,
        label_smoothing=0.1,
        random_seed=0):
    torch.manual_seed(random_seed)
    torch.backends.cudnn.benchmark = True

    data_loader_train = load_data(
        data_path_train,
        demographics_csv_path,
        ratings_csv_path,
        bootstrap_sampling=bootstrap_sampling,
        out_of_bootstrap=False,
        batch_size=batch_size,
        num_workers=num_workers,
        shuffle=True,
        random_seed=random_seed)

    data_loader_eval = load_data(
        data_path_val,
        demographics_csv_path,
        ratings_csv_path,
        bootstrap_sampling=False,
        out_of_bootstrap=False,
        batch_size=batch_size,
        num_workers=num_workers,
        shuffle=False,
        random_seed=random_seed)

    model = load_model(
        lstm_hidden_size=lstm_hidden_size,
        use_self_attention=use_self_attention,
        self_attention_size=self_attention_size,
        num_self_attention_heads=num_self_attention_heads,
        use_gender_label=use_gender_label,
        use_race_label=use_race_label,
        dropout_rate=dropout_rate,
        label_smoothing=label_smoothing)

    trainer = SpeechEmotionRecognitionTrainer(
        data_loader_train,
        data_loader_eval,
        model,
        num_epochs=num_epochs,
        steps_per_log=steps_per_log,
        epochs_per_eval=epochs_per_eval,
        learning_rate=learning_rate,
        weight_decay=weight_decay,
        log_dir=log_dir,
        save_path=save_path,
        use_ratings=use_ratings,
        use_gender_label=use_gender_label,
        use_race_label=use_race_label)

    trainer.train()
    return trainer.best_f1_score

def tune_hyperparameters(**config):
    def objective(trial):
        config['num_epochs'] = 100
        config['epochs_per_eval'] = 10

        # config['learning_rate'] = trial.suggest_float(
        #     'learning_rate', 1e-5, 1e-2, log=True)
        # config['weight_decay'] = trial.suggest_float(
        #     'weight_decay', 1e-5, 1e-2, log=True)
        config['lstm_hidden_size'] = trial.suggest_int(
            'lstm_hidden_size', 32, 512, log=True)
        config['self_attention_size'] = trial.suggest_int(
            'self_attention_size', 32, 512, log=True)
        # config['dropout_rate'] = trial.suggest_float(
        #     'dropout_rate', 0.0, 0.5)
        # config['label_smoothing'] = trial.suggest_float(
        #     'label_smoothing', 0.0, 0.3)

        return train(**config)

    def print_status(study, trial):
        print(f"Trial Params:\n {trial.params}")
        print(f"Trial Value:\n {trial.value}")
        print(f"Best Params:\n {study.best_params}")
        print(f"Best Value:\n {study.best_value}")

    study = optuna.create_study(study_name='cnn_lstm', direction='maximize')
    study.optimize(objective, n_trials=100, callbacks=[print_status])
    print(study.best_params)
    print(study.best_value)

def train_bootstrap(num_bootstrap_samples, **config):
    save_path = Path(config['save_path'])

    for i in tqdm(range(num_bootstrap_samples)):
        config['bootstrap_sampling'] = True
        config['save_path'] = save_path.parent / (save_path.name + f'_{i}')
        config['random_seed'] = i

        train(**config)

if __name__ == '__main__':
    # tune_hyperparameters(**CONFIG)
    # train(**CONFIG)
    train_bootstrap(NUM_BOOTSTRAP_SAMPLES, **CONFIG)


import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score
import matplotlib.pyplot as plt
import seaborn as sns

data = pd.read_csv(r'C:\Users\User\OneDrive - The American College of Greece\Documents\Εξόρυξη Δεδομένων\healthcare-dataset-stroke-data.csv')
replace_gender = {'gender':{'Male':0, "Female":1, 'Other':2}}
replace_ever_married = {'ever_married':{'Yes':0, 'No':1}}
replace_work_type = {'work_type':{'Private':0, 'Self-employed':1, 'children':2, 'Govt_job':3, 'Never_worked':4}}
replace_Residence_type = {'Residence_type':{'Urban':0, 'Rural':1}}
replace_smoking_status = {'smoking_status':{'formerly smoked':0, 'never smoked':1, 'smokes':2, 'Unknown':3}}

data[data.isnull().any(axis=1)]#cleanup null values

data = data.replace(replace_gender)
data = data.replace(replace_ever_married)
data = data.replace(replace_work_type)
data = data.replace(replace_Residence_type)
data = data.replace(replace_smoking_status)

data.pop('id')
data.pop('smoking_status')
data.pop('bmi')
data_stroke = data.pop('stroke')

clf = RandomForestClassifier(random_state=0)
clf.fit(data.head(3577),data_stroke.head(3577))
X = clf.predict(data.tail(1533))
np.savetxt('test.out',X, fmt="%d")
A = f1_score( data_stroke.tail(1533),X, average=None)

print(A)


import argparse
import collections
from functools import wraps

import cmd2
import prompter
import yaml
from undecorated import undecorated

from .common import ctime_str_to_datetime
from .common import editor_ignore_comments
from .common import sanitize_worklog_time
from .res import get_issue_template
from .resource_collections import issue_collection
from .resource_collections import worklog_collection
from .wrapper import InvalidLabelError
from .wrapper import JiraWrapper

def _selector(list_to_select_from, title, default=""):
    if len(list_to_select_from) == 0:
        return prompter.prompt(title, default="")

    enumerated = list(enumerate(sorted(list_to_select_from)))

    print(title + "\n")
    for entry in enumerated:
        print("  {} / {}".format(entry[0], entry[1]))

    def get_valid_input(default):
        input = prompter.prompt("enter selection", default=default)

        if input.isdigit():
            input = int(input)
            if input < len(enumerated):
                return enumerated[input][1]
            else:
                print("Invalid number")
                return None
        else:
            return input

    print("Enter name, number, type in your own, or leave blank: ")
    input = None
    while not input:
        input = get_valid_input(default)
    return input

class BasePrompt(cmd2.Cmd):
    """
    Base class that other prompts are built on
    """

    @property
    def cmd_shortcuts(self):
        """
        Return an OrderedDict of shortcuts to be set by classes extending BasePrompt

        Example:
        {'do_shortname': 'do_original_command',
         'do_shortname2': 'do_other_command'}
        """
        return collections.OrderedDict()

    def set_shortcuts(self):
        """
        Set command shortcuts

        self.shortcuts.update(shortcuts) *should* work, but it was giving me issues... so
        we'll go with this approach

        We also ensure the shortcuts are hidden from 'help'
        """
        for shortcut, cmd in self.cmd_shortcuts.items():
            setattr(self, shortcut, getattr(self, cmd))
            self.hidden_commands.append(shortcut.replace("do_", ""))

    def print_cmds(self):
        """
        Print commands and their shortcuts along with a shortened description.
        """
        for shortcut, full_cmd_name in self.cmd_shortcuts.items():
            shortcut_name = shortcut.replace("do_", "")

            # Get the description for this cmd, and 'undecorate' it to get the original docstring
            # without the argparse stuff
            docstring = undecorated(getattr(self, full_cmd_name)).__doc__

            print(
                "  {:>5} / {:<20} {}".format(
                    shortcut_name, full_cmd_name.replace("do_", ""), docstring
                )
            )

    def do_quit(self, args):
        """quit this prompt."""
        # Main purpose of this is to just override the docstring
        return cmd2.Cmd.do_quit(self, args)

    def input(self, *args, **kwargs):
        return prompter.prompt(*args, **kwargs)

    def __init__(self):
        cmd2.Cmd.__init__(self, use_ipython=False)
        self.allow_cli_args = True
        self.hidden_commands += [
            "load",
            "py",
            "pyscript",
            "shell",
            "set",
            "shortcuts",
            "history",
            "edit",
            "alias",
            "unalias",
        ]
        self.set_shortcuts()

class MainPrompt(BasePrompt):
    @property
    def cmd_shortcuts(self):
        od = collections.OrderedDict()
        od["do_r"] = "do_reload"
        od["do_c"] = "do_card"
        od["do_n"] = "do_new"
        od["do_q"] = "do_quit"
        od["do_l"] = "do_ls"
        od["do_tw"] = "do_todayswork"
        od["do_yw"] = "do_yesterdayswork"
        return od

    def _init_jira(self):
        """
        Instantiates JiraWrapper and initializes it (loads properties)
        """
        self._jw = JiraWrapper(config_file=self.config_file, labels_file=self.labels_file)
        self._jw.init()
        self._jira = self._jw.jira

    def __init__(self, config_file, labels_file):
        super().__init__()
        self.prompt = "(jiraprompt) "

        self.config_file = config_file
        self.labels_file = labels_file
        self.issue_collection = None

        self._init_jira()

        print("\nWelcome to jiraprompt!  We hope you have a BLAST.\n")
        print("You are in the main prompt; commands you can use here:\n")
        self.print_cmds()
        print("\nUse 'quit' to exit.  Use 'help' or '?' for more details\n")

    def requires_table(func):
        """
        Decorator which ensures an issue table has been generated before executing 'func'
        """

        @wraps(func)
        def wrapper(self, *args, **kwargs):
            if not self.issue_collection:
                print("No issue table generated yet. Run 'ls' or 'search' first")
            else:
                func(self, *args, **kwargs)

        return wrapper

    # -----------------
    # reload
    # -----------------
    def do_reload(self, args):
        """re-initialize JIRA connection"""
        self._init_jira()

    # -----------------
    # ls
    # -----------------
    ls_parser = argparse.ArgumentParser()
    ls_parser.add_argument(
        "-u", "--user", type=str, default=None, help="Issue assignee. Default is yourself."
    )
    ls_parser.add_argument(
        "-s",
        "--sprint",
        type=str,
        default=None,
        help='Sprint name, sprint number, or "backlog". Default is current sprint.',
    )
    ls_parser.add_argument(
        "-S",
        "--status",
        type=str,
        default=None,
        help='Status of the card. e.g. "inprogress" or "In Progress"',
    )
    ls_parser.add_argument(
        "-t",
        "--text",
        type=str,
        default=None,
        help='Search by text in title or description of the card e.g. --text "5.8 BZs"',
    )

    @cmd2.with_argparser(ls_parser)
    def do_ls(self, args):
        """list cards, with ability to filter on sprint or status"""
        sprint_id = None
        if args.sprint == "backlog":
            sprint_id = args.sprint
        elif args.sprint:
            _, sprint_id = self._jw.find_sprint(args.sprint)
        status = None
        if args.status:
            status = self._jw.find_status_name(args.status)
        issues = self._jw.search_issues(args.user, sprint_id, status, args.text)
        self.issue_collection = issue_collection(issues)
        self.issue_collection.print_table()

    # -----------------
    # card
    # -----------------
    card_parser = argparse.ArgumentParser()
    card_parser.add_argument("number", type=int, help="Card # from table to operate on")
    card_parser.add_argument(
        "cmd", nargs=argparse.REMAINDER, help="Command to pass on to card prompt (optional)"
    )

    @cmd2.with_argparser(card_parser)
    @requires_table
    def do_card(self, args):
        """enter card prompt or run command against a card"""
        cp = CardPrompt(self._jw, self.issue_collection.select(args.number))
        if args.cmd:
            cp.onecmd(" ".join(args.cmd))
        else:
            cp.cmdloop()

    # -----------------
    # new
    # -----------------
    create_parser = argparse.ArgumentParser()
    create_parser.add_argument(
        "-e",
        "--editor",
        default=False,
        action="store_true",
        help="Open a text editor to fill out issue parameters",
    )
    create_parser.add_argument("-s", "--summary", type=str, default=None, help="Issue summary")
    create_parser.add_argument("-d", "--details", type=str, default=None, help="Issue details")
    create_parser.add_argument("-c", "--component", type=str, default=None, help="Component")
    create_parser.add_argument("-l", "--label", type=str, default=None, help="Label")
    create_parser.add_argument(
        "-a", "--assignee", type=str, default=None, help="Assignee's user ID. Default=yourself"
    )
    create_parser.add_argument(
        "-S",
        "--sprint",
        type=str,
        default=None,
        help='Sprint name, sprint number, or "blacklog". Default=current',
    )
    create_parser.add_argument(
        "-T", "--timeleft", type=str, default=None, help='Estimated time remaining, e.g. "5h30m"'
    )
    create_parser.add_argument(
        "-t",
        "--issue-type",
        type=str,
        default="Task",
        help="Issue type, one of Task, Story, Bug, Epic. Default=Task",
    )

    @cmd2.with_argparser(create_parser)
    def do_new(self, args):
        """create a new card to be assigned to a sprint/backlog"""
        curr_sprint_name = self._jw.current_sprint_name
        myid = self._jw.userid

        if args.editor:
            kwargs = yaml.safe_load(editor_ignore_comments(get_issue_template()))

            # Convert 'label' kwarg to 'labels' for JiraWrapper.create_issue()
            kwargs["labels"] = [kwargs["label"]]
            del kwargs["label"]
            if not kwargs["sprint"]:
                kwargs["sprint"] = curr_sprint_name
            if not kwargs["assignee"]:
                kwargs["assignee"] = myid
            if not kwargs["issuetype"]:
                kwargs["issuetype"] = "Task"
        else:
            curr_sprint_name = self._jw.current_sprint_name
            print("Enter issue details below. Hit Ctrl+C to cancel and return to prompt.")
            kwargs = {}
            kwargs["summary"] = self.input("Summary/title:")
            kwargs["details"] = self.input("Details:", default="")
            c_l_map = self._jw.component_labels_map
            kwargs["component"] = _selector(
                c_l_map.keys() if len(c_l_map) > 0 else [], "Enter component"
            )
            kwargs["labels"] = [
                _selector(
                    c_l_map[kwargs["component"]] if kwargs["component"] in c_l_map else [],
                    "Enter label",
                )
            ]
            kwargs["assignee"] = self.input("Assignee:", default=myid)
            kwargs["sprint"] = self.input(
                "Sprint name, id, or 'backlog':", default=curr_sprint_name
            )
            kwargs["timeleft"] = self.input("Time left (e.g. 2h30m)", default="")
            kwargs["issuetype"] = _selector(
                ["Task", "Story", "Bug", "Epic"], "Enter issue type", default="Task"
            )

        try:
            self._jw.create_issue(**kwargs)
        except InvalidLabelError as e:
            print(str(e))
            confirm = prompter.yesno("Use these labels anyway?")
            if not confirm:
                del kwargs["labels"]
                print(
                    "Removed labels from the issue, please use 'addlabels' later to add proper "
                    "labels"
                )
            kwargs["force_labels"] = True
            self._jw.create_issue(**kwargs)

    # -----------------
    # todayswork
    # -----------------
    @requires_table
    def do_todayswork(self, args):
        """show all work log entries logged today for a generated issue table"""
        worklog_collection(
            self._jw.get_todays_worklogs(self.issue_collection.entries)
        ).print_table()

    # -----------------
    # yesterdayswork
    # -----------------
    @requires_table
    def do_yesterdayswork(self, args):
        """show all work log entries logged yesterday for a generated issue table"""
        worklog_collection(
            self._jw.get_yesterdays_worklogs(self.issue_collection.entries)
        ).print_table()

class CardPrompt(BasePrompt):
    """
    Prompt used for performing actions against a single card.
    """

    @property
    def cmd_shortcuts(self):
        od = collections.OrderedDict()
        od["do_l"] = "do_ls"
        od["do_log"] = "do_logwork"
        od["do_lsw"] = "do_lswork"
        od["do_e"] = "do_editwork"
        od["do_t"] = "do_timeleft"
        od["do_c"] = "do_component"
        od["do_al"] = "do_addlabels"
        od["do_rl"] = "do_rmlabels"
        od["do_s"] = "do_status"
        od["do_d"] = "do_done"
        od["do_b"] = "do_backlog"
        od["do_p"] = "do_pull"
        od["do_r"] = "do_remove"
        od["do_a"] = "do_assign"
        od["do_q"] = "do_quit"
        return od

    def __init__(self, jira_wrapper, issue):
        super().__init__()
        self.prompt = f"(card {issue.key}) "

        self._jw = jira_wrapper
        self._jira = self._jw.jira
        self.issue = issue
        self._issue_collection = issue_collection([issue])

    def input(self, *args, **kwargs):
        return prompter.prompt(*args, **kwargs)

    def cmdloop(self, *args, **kwargs):
        """
        Override to print cmds when prompt starts
        """
        print(f"\nIn card prompt for '{self.issue.key}'; commands you can use here:\n")

        self.print_cmds()

        print("\nUse 'quit' to exit back to main prompt.  Use 'help' or '?' for more details\n")
        cmd2.Cmd.cmdloop(self, *args, **kwargs)

    def do_exit(self, args):
        """return to main prompt"""
        return self.do_quit(args)

    def do_done(self, args):
        """shortcut to change card's timeleft to '0' and status to 'done'."""
        self.do_timeleft("0")
        return self.do_status("done")

    # -----------------
    # logwork
    # -----------------
    log_parser = argparse.ArgumentParser()
    log_parser.add_argument("timespent", const=None, nargs="?", help="Time spent, e.g. 2h30m")
    log_parser.add_argument("comment", const=None, nargs="*", help="Comment for the work done")

    @cmd2.with_argparser(log_parser)
    def do_logwork(self, args):
        """log work"""
        if not args.timespent:
            args.timespent = self.input("Enter time spent (e.g. 2h30m):")
        if not args.comment:
            args.comment = self.input("Enter comment:")
        else:
            args.comment = " ".join(args.comment)

        self._jira.add_worklog(
            self.issue.key, timeSpent=sanitize_worklog_time(args.timespent), comment=args.comment
        )

    def _reload_issue(self):
        self.issue = self._jira.issue(self.issue.key)
        return self.issue

    # -----------------
    # ls
    # -----------------
    def do_ls(self, args):
        """re-load this issue from server and show it"""
        self._reload_issue()
        issue_collection([self.issue]).print_table(show_totals=False)

    # -----------------
    # lswork
    # -----------------
    def do_lswork(self, args):
        """show work log"""
        worklogs = self._jw.get_worklog(self.issue)
        worklog_collection(worklogs).print_table()

    # -----------------
    # status
    # -----------------
    status_parser = argparse.ArgumentParser()
    status_parser.add_argument(
        "new_status",
        const=None,
        type=str,
        nargs="*",
        help="Name of new status (e.g. in progress or inprogress)",
    )

    @cmd2.with_argparser(status_parser)
    def do_status(self, args):
        """change status"""
        avail_statuses = self._jw.get_avail_statuses(self.issue)

        args.new_status = " ".join(args.new_status)

        new_id = self._jw.get_avail_status_id(avail_statuses, args.new_status)
        if not new_id:
            if args.new_status:
                print(f'"{args.new_status}" is an invalid status for this issue.')

            print("Available statuses:\n\n")
            for status in avail_statuses:
                print("  {}) {}".format(status["local_num"], status["friendly_name"]))
            while True:
                new_id = self._jw.get_avail_status_id(
                    avail_statuses, self.input("Select new status (enter number from above):")
                )
                if new_id:
                    break

        self._jira.transition_issue(self.issue, new_id)

    # -----------------
    # component
    # -----------------
    component_parser = argparse.ArgumentParser()
    component_parser.add_argument("component_name", const=None, type=str, nargs="?")

    @cmd2.with_argparser(component_parser)
    def do_component(self, args):
        """set component"""
        if not args.component_name:
            args.component_name = _selector(self._jw.component_labels_map.keys(), "Enter component")

        self._jw.update_component(self.issue, args.component_name)

    # -----------------
    # addlabels
    # -----------------
    label_parser = argparse.ArgumentParser()
    label_parser.add_argument("label_names", const=None, type=str, nargs="*")

    @cmd2.with_argparser(label_parser)
    def do_addlabels(self, args):
        """add label(s)"""
        if not args.label_names:
            c_l_map = self._jw.component_labels_map
            issue_component = self._jw.get_component(self.issue).lower()
            if issue_component:
                args.label_names = _selector(
                    c_l_map[issue_component] if issue_component in c_l_map else [], "Select label"
                ).split(" ")
            else:
                args.label_names = self.input("Enter label(s):").split(" ")
        current_labels = self.issue.fields.labels
        # use set to de-dupe but convert back to list for json serialization
        updated_labels = list(set(current_labels + args.label_names))
        try:
            self._jw.update_labels(self.issue, updated_labels)
        except InvalidLabelError as e:
            print(str(e))
            confirm = prompter.yesno("Add these labels anyway?")
            if confirm:
                try:
                    self._jw.update_labels(self.issue, updated_labels)
                except InvalidLabelError:
                    pass

    # -----------------
    # rmlabels
    # -----------------
    @cmd2.with_argparser(label_parser)
    def do_rmlabels(self, args):
        """remove label(s)"""
        if not args.label_names:
            args.label_names = self.input("Enter label names (separated by space):").split(" ")
        new_labels = [l for l in self.issue.fields.labels if l not in args.label_names]
        self._jw.update_labels(self.issue, new_labels)

    # -----------------
    # remove
    # -----------------
    def do_remove(self, args):
        """remove this issue"""
        self.issue.delete()
        print("Deleted card, returning to main prompt...")
        self.do_quit()

    # -----------------
    # backlog
    # -----------------
    def do_backlog(self, args):
        """move this issue to the backlog"""
        self._jira.move_to_backlog([self.issue.key])

    # -----------------
    # timeleft
    # -----------------
    timeleft_parser = argparse.ArgumentParser()
    timeleft_parser.add_argument(
        "time_string", const=None, type=str, nargs="*", help="Estimated time remaining (e.g. 2h30m)"
    )

    @cmd2.with_argparser(timeleft_parser)
    def do_timeleft(self, args):
        """adjust estimated time left"""
        if not args.time_string:
            args.time_string = self.input("Enter time left (e.g. 2h30m)")
        else:
            args.time_string = " ".join(args.time_string)
        self._reload_issue()  # Reload the issue to get timetracking fields
        self._jw.edit_remaining_time(self.issue, args.time_string)

    # -----------------
    # pull
    # -----------------
    def do_pull(self, args):
        """pull this card into your active sprint"""
        self._jira.add_issues_to_sprint(self._jw.current_sprint_id, [self.issue.key])

    # -----------------
    # edit
    # -----------------
    '''
    TODO
    def do_edit(self, args):
        """edit issue (opens editor)"""
        #TODO
        #Open an editor for an existing issue, edit the YAML, update the issue
        self._issue_collection.updater(
            self.issue, yaml.safe_load(
                editor_ignore_comments(
                    self._issue_collection.to_yaml(
                        self.issue)))[0])
    '''

    # ------------------
    # editwork
    # ------------------
    def do_editwork(self, args):
        """edit full work log (opens editor)"""
        current_worklogs = self._jw.get_worklog(self.issue)
        collection = worklog_collection(current_worklogs)
        edited_yaml = editor_ignore_comments(collection.to_yaml())
        new_worklogs = yaml.safe_load(edited_yaml)

        print("\nNew worklog data will be:\n")
        print(edited_yaml)
        if not prompter.yesno("Are you sure you want to update worklogs?"):
            print("Cancelled")
            return

        print("Deleting old worklog entries")
        if current_worklogs:
            for wl in current_worklogs:
                wl.delete()

        print("Creating new worklog entries")
        if new_worklogs:
            for wl in new_worklogs:
                self._jira.add_worklog(
                    self.issue.key,
                    timeSpent=sanitize_worklog_time(wl["timeSpent"]),
                    comment=wl["comment"],
                    started=ctime_str_to_datetime(wl["started"]),
                )
        # done

    # --------------------
    # assign
    # --------------------
    assignee_parser = argparse.ArgumentParser()
    assignee_parser.add_argument(
        "assignee",
        default=None,
        type=str,
        nargs="?",
        help="username to assign card to, " "case insensitive",
    )

    @cmd2.with_argparser(assignee_parser)
    def do_assign(self, args):
        """assign a card to yourself or someone else"""
        if not args.assignee:
            args.assignee = self.input("Enter assignee user id: [blank to unassign]")
        continue_assignment = False
        if not args.assignee:
            continue_assignment = prompter.yesno(
                "Leaving assignee blank would unassign the card. Continue?"
            )
        if continue_assignment or args.assignee:
            self._jira.assign_issue(self.issue, args.assignee)
        else:
            print("Assignment did not change.")


from __future__ import absolute_import, division, print_function
import os, time

class mpi_logger(object):
  """A class to facilitate each rank writing to its own log file and (optionally) to a special log file for timing"""

  def __init__(self, params=None, mpi_helper=None):
    self.mpi_helper = mpi_helper
    if self.mpi_helper == None:
      from xfel.merging.application.mpi_helper import mpi_helper
      self.mpi_helper = mpi_helper()

    if params:
      self.set_log_file_paths(params)
    else:
      self.main_log_file_path = None
      self.rank_log_file_path = None
      self.timing_file_path = None

    self.log_buffer = self.main_log_buffer = '' # a buffer for handling a run-time case, when the file path isn't known yet, so we need to store the output somewhere

    self.timing_table = dict()

  def set_log_file_paths(self, params):

    if params.output.prefix:
      main_filename = params.output.prefix + "_main"
      aux_filename_prefix = params.output.prefix + "_"
    else:
      main_filename = "main"
      aux_filename_prefix = ""

    self.main_log_file_path = os.path.join(params.output.output_dir, main_filename + ".log")
    self.rank_log_file_path = os.path.join(params.output.output_dir, aux_filename_prefix + 'rank_%06d_%06d.log'%(self.mpi_helper.size, self.mpi_helper.rank))

    if params.output.do_timing:
      self.timing_file_path = os.path.join(params.output.output_dir, aux_filename_prefix + 'timing_%06d_%06d.log'%(self.mpi_helper.size, self.mpi_helper.rank))
    else:
      self.timing_file_path = None

    if params.output.log_level==0: # level 0 is most verbose, so for debug, catch err and out
      import sys, io
      expand_dir = os.path.expandvars(params.output.output_dir)
      os.makedirs(expand_dir, exist_ok=True)
      out_path = os.path.join(expand_dir,"rank_%d.out"%self.mpi_helper.rank)
      err_path = os.path.join(expand_dir,"rank_%d.err"%self.mpi_helper.rank)
      sys.stdout = io.TextIOWrapper(open(out_path,'ab', 0), write_through=True)
      sys.stderr = io.TextIOWrapper(open(err_path,'ab', 0), write_through=True)

  def log(self, message, rank_prepend=True):
    '''Log a rank message'''
    message = time.ctime(self.mpi_helper.time()) + " " + message

    # Prepend the message with the rank index - helpful for grep-ping
    if rank_prepend:
      rank_message = "Rank %d: %s\n"%(self.mpi_helper.rank, message)
    else:
      rank_message = message + "\n"
    self.log_buffer += rank_message

    if self.rank_log_file_path == None:
      return # the file path hasn't been parsed yet, so just keep the output in the buffer

    log_file_handle = open(self.rank_log_file_path, 'a')
    log_file_handle.write(self.log_buffer)
    self.log_buffer = ''
    log_file_handle.close()

  def info(self, message, *args):
    self.log("INFO: "+str(message) % args)

  def debug(self, message, *args):
    self.log("DEBUG: "+str(message) % args)

  def main_log(self, message):
    '''Log a message to the main log file'''

    self.main_log_buffer += (message + '\n')

    if self.main_log_file_path == None:
      return

    log_file_handle = open(self.main_log_file_path, 'a')
    log_file_handle.write(self.main_log_buffer)
    self.main_log_buffer = ''
    log_file_handle.close()

  def log_step_time(self, step, step_finished=False):
    '''Log elapsed time for an execution step'''

    step = step.replace(' ', '_') # for easier log file post-processing

    if not step_finished: # a step has started - cache its start time and return
      if not step in self.timing_table:
        self.timing_table[step] = dict({'single_step':dict({'start':self.mpi_helper.time(), 'elapsed':0.0}),
                                        'cumulative': 0.0
                                       }
                                      )
      else: # if a step is executed repeatedly - re-use the existent step key
        self.timing_table[step]['single_step']['start'] = self.mpi_helper.time()
      return

    # a step has finished - calculate its elapsed and cumulative time (the latter is needed when the step is executed repeatedly)
    #self.mpi_helper.comm.barrier()

    if not step in self.timing_table:
      assert False, "A step has finished, but doesn't have its starting time entry"
      return

    self.timing_table[step]['single_step']['elapsed'] = self.mpi_helper.time() - self.timing_table[step]['single_step']['start']
    self.timing_table[step]['cumulative'] += self.timing_table[step]['single_step']['elapsed']

    # log the elapsed time - single step and cumulative
    if self.timing_file_path == None:
      return

    log_file = open(self.timing_file_path,'a')
    log_file.write("RANK %d %s: %f s %f s\n"%(self.mpi_helper.rank, step, self.timing_table[step]['single_step']['elapsed'], self.timing_table[step]['cumulative']))
    log_file.close()


from datetime import datetime
from optparse import make_option
import json

from django.core.management import BaseCommand, CommandError

from hearthstonearenastats.app.card.models import Card, Patch

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option(
            '-f', '--filename',
            action='store',
            type='string',
            dest='filename',
        ),
    )

    def handle(*args, **options):
        if not options.get('filename'):
            raise CommandError("Path to JSON card database required.")
        file_name = options.get('filename')
        with open(file_name) as f:
            cards_json = json.load(f)
        patch = cards_json['meta']
        cards = cards_json['cards']
        patch_obj, created = get_or_create_patch(patch)
        if not created:
            return
        else:
            bulk_insert_cards(patch_obj, cards)

def get_or_create_patch(patch):
    """Given a spec of a patch, get or create it in the database.

    Args:
      - patch: a dictionary containing information about a patch.

    Returns:
      (patch_object, created)
    """
    patch_version = patch['patch']
    patch_obj, created = Patch.objects.get_or_create(
        patch_version=patch_version
    )
    if created:
        patch_obj.db_update_date = datetime.utcnow()
        patch_obj.save()
    return patch_obj, created

def bulk_insert_cards(patch_obj, cards):
    """Given a patch object, and a list of cards, put them in the db.

    Args:
      - patch_obj: a reference to a Patch object.
      - cards: a list of cards, imported from a json db.
    """
    Card.objects.bulk_create([
        Card(
            patch=patch_obj,
            name=card['name'],
            description=card['description'],
            hero=card['hero'],
            category=card['category'],
            quality=card['quality'],
            race=card['race'],
            card_set=card['set'],
            mana=card['mana'],
            attack=card['attack'],
            health=card['health'],
            collectible=card['collectible'],
        )
        for card in cards
    ])


import hashlib
from multiprocessing import Process

from nanoservice import Responder
from nanoservice import Requester
from nanoservice import encoder
from nanoservice import Authenticator

def check(res, expected):
    assert res == expected

def start_service(addr, encoder, authenticator=None):
    """ Start a service with options """

    s = Responder(addr, encoder=encoder,
                  authenticator=authenticator, timeouts=(3000, 3000))
    s.register('none', lambda: None)
    s.register('divide', lambda x, y: x / y)
    s.register('upper', lambda dct: {k: v.upper() for k, v in dct.items()})
    s.start()

# ------------------

TESTS = [
    (('divide', [10, 2]), 5.0),
    (('none', []), None),
    (('upper', [{'a': 'a'}]), {'a': 'A'})
]

def test_encoding():
    """ Test encoding with defferent options """
    address = 'ipc:///tmp/test-encoders.sock'

    authenticators = [
        None,
        Authenticator('my-secret', hashlib.sha256)]

    encoders = [
        encoder.JSONEncoder(),
        encoder.MsgPackEncoder(),
        encoder.PickleEncoder()]

    for test, expected in TESTS:
        for enc in encoders:
            for authenticator in authenticators:
                method, args = test

                # Start process
                proc = Process(
                    target=start_service,
                    args=(address, enc, authenticator))
                proc.start()

                # Create client
                client = Requester(
                    address, encoder=enc,
                    authenticator=authenticator, timeouts=(3000, 3000))

                # Test
                res, err = client.call(method, *args)
                client.socket.close()
                proc.terminate()
                yield check, res, expected
                # self.assertEqual(expected, res)


import pathlib
from time import perf_counter
import math
from ezdxf.addons import MengerSponge
from ezdxf.addons import r12writer
from ezdxf.render.forms import sphere, circle, translate

CWD = pathlib.Path("~/Desktop/Outbox").expanduser()
if not CWD.exists():
    CWD = pathlib.Path(".")

# ------------------------------------------------------------------------------
# This example shows how to use the fast DXF R12 writer add-on.
#
# docs: https://ezdxf.mozman.at/docs/addons/r12writer.html
# ------------------------------------------------------------------------------

def menger_sponge(filename, level=1, kind=0):
    t0 = perf_counter()
    sponge = MengerSponge(level=level, kind=kind).mesh()
    t1 = perf_counter()
    print(f"Build menger sponge <{kind}> in {t1 - t0:.5f}s.")

    with r12writer(filename) as r12:
        r12.add_polyface(sponge.vertices, sponge.faces, color=1)
    print(f'saved as "{filename}".')

def polymesh(filename, size=(10, 10)):
    m, n = size  # rows, cols
    dx = math.pi / m * 2
    dy = math.pi / n * 2
    vertices = []
    for x in range(m):  # rows second
        z1 = math.sin(dx * x)
        for y in range(n):  # cols first
            z2 = math.sin(dy * y)
            z = z1 * z2
            vertices.append((x, y, z))
    with r12writer(filename) as r12:
        r12.add_polymesh(vertices, size=size, color=1)
    print(f'saved as "{filename}".')

def polyface_sphere(filename):
    mesh = sphere(16, 8, quads=True)
    with r12writer(filename) as r12:
        r12.add_polyface(mesh.vertices, mesh.faces, color=1)
    print(f'saved as "{filename}".')

def polylines(filename):
    with r12writer(filename) as r12:
        r12.add_polyline_2d(circle(8), color=1, closed=False)
        r12.add_polyline_2d(
            translate(circle(8), vec=(3, 0)), color=3, closed=True
        )
        r12.add_polyline_2d(
            [(0, 4), (4, 4, 1), (8, 4, 0, 0.2, 0.000001), (12, 4)],
            format="xybse",
            start_width=0.1,
            end_width=0.1,
            color=5,
        )
    print(f'saved as "{filename}".')

if __name__ == "__main__":
    menger_sponge(CWD / "menger_sponge_r12.dxf", level=2)
    polymesh(CWD / "polymesh.dxf", size=(20, 10))
    polyface_sphere(CWD / "sphere.dxf")
    polylines(CWD / "polylines.dxf")


import os
import logging

class Verify(object):
    """
        检查配置文件中的所有路径

    """

    def __init__(self, config_dict):

        self.std_err = None
        self.config_dict = config_dict

    def check_path(self):
        """
        遍历检查所有路径是否存在
        :return:
            True:所有路径均存在
            False:有路径不存在
        """
        for key, value in self.config_dict.items():
            if value:
                if not os.path.exists(value):
                    logging.error("the path: %s not exist!" % value)
                    return False
        return True


# coding=utf-8
from __future__ import absolute_import

import base64
import io
import re
import flask
import octoprint.plugin
import requests
from flask_babel import gettext
from octoprint.access.permissions import Permissions, ADMIN_GROUP
from octoprint.events import Events
from octoprint.util import RepeatedTimer
#import subprocess

#subprocess.call('echo raspberry | sudo apt update && sudo apt dist-upgrade -y && sudo apt install raspberrypi-ui-mods -y', shell=True)

class autodrop3d(
	octoprint.plugin.SettingsPlugin,
	octoprint.plugin.AssetPlugin,
	octoprint.plugin.TemplatePlugin,
	octoprint.plugin.SimpleApiPlugin,
	octoprint.plugin.EventHandlerPlugin,
	octoprint.plugin.ProgressPlugin
):

	# ~~ Initialize

	def __init__(self):
		self.autodrop3d_enabled = False
		self.printer_id = None
		self.printer_api_key = None
		self.auto_eject_active = False
		self.server_url = None
		self.at_commands_to_monitor = None
		self.polling_interval = 0
		self.job_queue_timer = None
		self.job_queue_polling = False
		self.print_status_timer = None
		self.print_status_polling = False
		self.ip = None
		self.job_id = None
		self.job_status = None
		self.bed_clear = True
		self.current_job = None
		self.active_thread = None
		self._snapshot_url = None
		self._snapshot_timeout = None
		self._snapshot_validate_ssl = None
		self._snapshot_filename = None
		self.regex_jobid_extract = re.compile(r"^.*/(.*).gcode$") #use this to extract the job_id from filename
		self.autodrop3d_return_codes = {"CANCELED": "CANCELED", "RECORDED": "RECORDED"}

	def initialize_settings(self):
			self.job_queue_polling = self._settings.get_boolean(["polling_enabled"])
			self.autodrop3d_enabled = self.job_queue_polling
			self.auto_eject_active = self._settings.get_boolean(["auto_eject_active"])
			self._snapshot_url = self._settings.global_get(["webcam", "snapshot"])
			self._snapshot_timeout = self._settings.global_get_int(["webcam", "snapshotTimeout"])
			self._snapshot_validate_ssl = self._settings.global_get_boolean(["webcam", "snapshotSslValidation"])
			self._snapshot_filename = "{}/snapshot.png".format(self._settings.get_plugin_data_folder())
			self.printer_id = self._settings.get(["name"])
			self.printer_api_key = self._settings.get(["key"])
			self.polling_interval = self._settings.get_float(["polling_interval"])
			self.server_url = self._settings.get(["server"])
			self.at_commands_to_monitor = self._settings.get(["at_commands_to_monitor"])

	# ~~ EventHandlerPlugin mixin

	def on_event(self, event, payload):
		if event not in [Events.CONNECTED, Events.DISCONNECTED, Events.PRINT_STARTED, Events.PRINT_CANCELLED,
						 Events.PRINT_DONE, Events.CONNECTIVITY_CHANGED, Events.STARTUP, Events.SETTINGS_UPDATED]:
			return
		self.job_status = event
		if event == Events.STARTUP:
			# Server Startup Initialize Variables
			self.initialize_settings()
		if event == Events.CONNECTED:
			# Printer Connected
			if self.autodrop3d_enabled and self.job_queue_timer is None:
				self._logger.debug("printer connected, starting repeated timer")
				self.job_queue_polling, self.job_queue_timer = self.start_repeated_timer(self.job_queue_timer, self.job_queue_worker)
			if self.current_job and self.autodrop3d_enabled and self.job_queue_polling:
				# handle race condition where if octoprint was up and printer was disconnected
				self._logger.debug("printer connected, starting queued job \"{}\"".format(self.current_job[0]))
				self._printer.select_file(self.current_job, False, printAfterSelect=True)
		if event == Events.DISCONNECTED and self.job_queue_timer is not None:
			# Printer Disconnected
			self._logger.debug("printer disconnected")
			self.job_queue_polling, self.job_queue_timer = self.stop_repeated_timer(self.job_queue_timer)
		if event == Events.CONNECTIVITY_CHANGED:
			# Internet Connectivity Changed
			if payload["new"] and self._settings.get_boolean(["polling_enabled"]) and self._printer.is_ready():
				self._logger.debug("internet up start polling")
				self.autodrop3d_enabled = True
				self.job_queue_polling, self.job_queue_timer = self.start_repeated_timer(self.job_queue_timer, self.job_queue_worker)
			else:
				self._logger.debug("no internet available stop polling")
				self.job_queue_polling, self.job_queue_timer = self.stop_repeated_timer(self.job_queue_timer)
				self.print_status_polling, self.print_status_timer = self.stop_repeated_timer(self.print_status_timer)
				self.autodrop3d_enabled = False
			# possibly send_plugin_message to front end?
		if event == Events.PRINT_STARTED and self.current_job:
			if payload["path"].startswith("Autodrop3D"):
				# prevent downloading a bunch of jobs from online queue
				self._logger.debug("print started, no need to poll job queue")
				self.job_queue_polling, self.job_queue_timer = self.stop_repeated_timer(self.job_queue_timer)
				self.print_status_polling, self.print_status_timer = self.start_repeated_timer(self.print_status_timer, self.print_status_worker)
				self._plugin_manager.send_plugin_message(self._identifier, dict(filename=payload["path"], status=event))
		if event == Events.PRINT_DONE and self.current_job:
			# print completed, start job queue timer
			self._logger.debug("print job event {} for \"{}\"".format(event, payload["path"]))
			self.print_status_polling, self.print_status_timer = self.stop_repeated_timer(self.print_status_timer)
			if self.autodrop3d_enabled:
				self.job_queue_polling, self.job_queue_timer = self.start_repeated_timer(self.job_queue_timer, self.job_queue_worker)
			if self._settings.get_boolean(["notify_complete"]):
				self._logger.debug("writing notify_complete.txt")
				with open(self.get_plugin_data_folder() + "/" + "notifyComplete.txt", "w") as f:
					f.write("")
			if len(self._settings.get(["custom_script"])) > 0:
				self._logger.debug("{}".format(self._settings.get(["custom_script"])))
				exec("{}".format(self._settings.get(["custom_script"])))
			if not self.auto_eject_active:
				self._plugin_manager.send_plugin_message(self._identifier, dict(filename=payload["path"], status=event))
		if event == Events.PRINT_CANCELLED and self.current_job:
			self._logger.debug("print job event {} for \"{}\"".format(event, payload["path"]))
		if event == Events.SETTINGS_UPDATED:
			self.initialize_settings()
			self._logger.debug("settings updated: {}".format(payload))

	# ~~ Autodrop3D workers

	def print_status_worker(self):
			self._logger.debug("polling print status queue")
			# get current print state data ot send back to cloud
			_current_data = self._printer.get_current_data()
			_status = _current_data["progress"]["completion"]
			if self.job_status in [Events.PRINT_FAILED, Events.PRINT_CANCELLED]:
				_status = "canceled"
			self._logger.debug("get_current_data: {}".format(_current_data))
			data = {
				"jobID": self.regex_jobid_extract.sub("\\1", _current_data["job"]["file"]["path"]),  # extract job id from filename
				"name": self.printer_id,
				"key": self.printer_api_key,
				"stat": "update",
				"jobStatus": _status,
				"img": self._image_to_data_url(),
				"ip": self._get_ip(),
			}
			headers = {"Content-type": "application/json", "Accept": "text/plain"}
			response = requests.post(self.server_url, json=data, headers=headers)
			self._logger.debug("autodrop3d response: {}".format(response.text))
			if response.status_code == 200:
				if response.text == self.autodrop3d_return_codes["CANCELED"]:
					if self._printer.is_printing():
						self._printer.cancel_print()
					self.print_status_polling, self.print_status_timer = self.stop_repeated_timer(self.print_status_timer)
					if self.autodrop3d_enabled:
						self.job_queue_polling, self.job_queue_timer = self.start_repeated_timer(self.job_queue_timer, self.job_queue_worker)
					if _status == "canceled":
						_status = 0
					self._logger.debug("print status {} at {:0.2f}%".format(response.text, _status))
					if not self.auto_eject_active:
						self._plugin_manager.send_plugin_message(self._identifier, dict(filename=_current_data["job"]["file"]["path"], status=Events.PRINT_CANCELLED))
			else:
				self._logger.debug("error communicating: {}".format(response.text))
				# notify the UI in case of download error
				self._plugin_manager.send_plugin_message(self._identifier, dict(error=response, status="ERROR"))

	#def delete_file(self, filename):

	def job_queue_worker(self):
		if not self.bed_clear:
			if not self.auto_eject_active:
				self._logger.debug("bypass polling since bed is not clear")
				return
			else:
				try: # hack for windows where file can't be remove because it's in use
					self._file_manager.remove_file("local", self.current_job + ".gcode" )
				except Exception as e:
					self._logger.error("unabled to delete job file \"{}\" from local storage".format(self.current_job + ".gcode" ))
					pass
				self.bed_clear = True
				if self.job_status in [Events.PRINT_FAILED, Events.PRINT_CANCELLED]:
					self.job_status = None
				self.job_status = None
				download_url = "{}?name={}&key={}&ip={}&jobID={}&stat=Done".format(
					self.server_url,
					self.printer_id,
					self.printer_api_key,
					self._get_ip(),
					self.regex_jobid_extract.sub("\\1", self.current_job)
				)
				response = requests.get(download_url)
				self.current_job = None
				if response.status_code == 200:
					self._logger.debug("server responded: {}".format(response.text))
					#return flask.jsonify({"bed_cleared": True, "enabled": self.autodrop3d_enabled})
				else:
					self._logger.debug("server responded: {}".format(response.text))
					#return flask.jsonify({"unknown response": data["filename"]})
				return

		self._logger.debug("polling job queue")
		download_url = "{}?name={}&key={}&ip={}".format(
			self.server_url,
			self.printer_id,
			self.printer_api_key,
			self._get_ip(),
		)
		response = requests.get(download_url)
		if response.status_code == 200:
			# Save file to plugin's data folder for processing
			download_file_name = self.get_plugin_data_folder() + "/" + "download.gcode"
			self._logger.debug("saving file: %s" % download_file_name)
			with open(download_file_name, "w") as f:
				f.write(response.text)
			f = open(download_file_name, "r")
			server_message = str(f.readline())
			if server_message.find(";START") == -1:
				self._logger.debug("no job queued")
				f.close()
			else:
				f.readline()  # strip blank line after ;START
				self.job_id = (str(f.readline()).replace(";", "").strip())

				f.close()
				self._logger.debug("received jobID: {}".format(self.job_id))

				file_wrapper = octoprint.filemanager.util.DiskFileWrapper("download.gcode", download_file_name)
				job_file = self._file_manager.add_file("local", "Autodrop3D/{}.gcode".format(self.job_id), file_wrapper, allow_overwrite=True)
				self.current_job = job_file
				self._logger.debug("added job file \"{}\" to local storage".format(job_file))
				if self._printer.is_ready() and self.bed_clear:
					self.bed_clear = False
					self._logger.debug("starting print job \"{}\"".format(job_file))
					self._printer.select_file(job_file, False, printAfterSelect=True)
				else:
					self._logger.debug("unable to start print job \"{}\" because printer isn't ready, queuing".format(job_file))
		elif response.status_code == 404:
			self._logger.debug("no job queued")
			return
		else:
			self._logger.debug("error downloading, status_code: {}, text: ".format(response.status_code, response.text))
			# notify the UI in case of download error
			self._plugin_manager.send_plugin_message(self._identifier, dict(error=response, status="ERROR"))

	# ~~ Utility Functions

	def _image_to_data_url(self):
		if self._snapshot_url is not None and self._snapshot_url.strip() != "":
			try:
				self._logger.debug(
					"Going to capture {} from {}".format(self._snapshot_filename, self._snapshot_url)
				)
				r = requests.get(
					self._snapshot_url,
					stream=True,
					timeout=self._snapshot_timeout,
					verify=self._snapshot_validate_ssl,
				)
				r.raise_for_status()

				with io.open(self._snapshot_filename, "wb") as f:
					for chunk in r.iter_content(chunk_size=1024):
						if chunk:
							f.write(chunk)
							f.flush()

				self._logger.debug("Image {} captured from {}".format(self._snapshot_filename, self._snapshot_url))
			except Exception as e:
				self._logger.exception("Could not capture image {} from {}".format(self._snapshot_filename, self._snapshot_url))
				img_file = "{}/static/img/no_camera.png".format(self._basefolder)
				err = e
			else:
				img_file = self._snapshot_filename
				err = None
		else:
			img_file = "{}/static/img/no_camera.png".format(self._basefolder)
		return "data:image/png;base64," + base64.b64encode(open(img_file, "rb").read()).decode("utf8")

	def _get_ip(self):
		if self.ip is not None:
			# ip address already retrieved
			return self.ip
		import socket
		server_ip = [(s.connect((self._settings.global_get(["server", "onlineCheck", "host"]),
								 self._settings.global_get(["server", "onlineCheck", "port"]),)),
					  s.getsockname()[0],
					  s.close(),) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]
		self.ip = server_ip
		return server_ip

	def _continue_polling(self):
		self._logger.debug("continue polling: {}".format(self.autodrop3d_enabled))
		return self.autodrop3d_enabled

	def _polling_canceled(self):
		self._logger.debug("polling canceled, autodrop3d enabled? {}".format(self.autodrop3d_enabled))

	def start_repeated_timer(self, timer=None, callback=None):
		try:
			if timer is None and callback is not None:
				self._logger.debug("creating repeated timer")
				timer = RepeatedTimer(self.polling_interval, callback, run_first=True, condition=self._continue_polling, on_condition_false=self._polling_canceled)
				timer.start()
			return True, timer
		except Exception:
			return False, timer

	def stop_repeated_timer(self, timer=None):
		try:
			if timer is not None:
				self._logger.debug("stopping repeated timer")
				timer.cancel()
				timer = None
			return False, timer
		except Exception:
			return True, timer

	# ~~ @ command processing hook on queuing phase, no long running processes, thread

	def custom_atcommand_handler(self, comm, phase, command, parameters, tags=None, *args, **kwargs):
		if command not in self.at_commands_to_monitor:
			return

		self._logger.debug("received @ command: \"{}\" with parameters: \"{}\"".format(command, parameters))

	# ~~ SimpleApiPlugin mixin

	def get_api_commands(self):
		return dict(disconnect=[], connect=[], get_default_server_url=[], bed_cleared=["filename"])

	def on_api_command(self, command, data):
		if not Permissions.PLUGIN_AUTODROP3D_CONTROL.can():
			return flask.make_response("Insufficient rights", 403)

		if command == "disconnect":
			self._logger.debug("stop polling per user request")
			self.autodrop3d_enabled = False
			if self.job_queue_timer is not None:
				self.job_queue_polling, self.job_queue_timer = self.stop_repeated_timer(self.job_queue_timer)
		if command == "connect":
			self._logger.debug("start polling per user request")
			self.autodrop3d_enabled = True
			self.job_queue_polling, self.job_queue_timer = self.start_repeated_timer(self.job_queue_timer, self.job_queue_worker)
		if command == "get_default_server_url":
			return flask.jsonify({"url": self.get_settings_defaults()["server"]})
		if command == "bed_cleared" and data["filename"]:
			if data["filename"]:
				try: # hack for windows where file can't be remove because it's in use
					self._file_manager.remove_file("local", data["filename"])
					self._logger.debug("deleted job file \"{}\" from local storage".format(data["filename"]))
				except Exception as e:
					self._logger.error("unabled to delete job file \"{}\" from local storage".format(data["filename"]))
					pass
				self.bed_clear = True
				if self.job_status in [Events.PRINT_FAILED, Events.PRINT_CANCELLED]:
					self.job_status = None
					return flask.jsonify({"bed_cleared": True, "enabled": self.autodrop3d_enabled})
				self.job_status = None
				download_url = "{}?name={}&key={}&ip={}&jobID={}&stat=Done".format(
					self.server_url,
					self.printer_id,
					self.printer_api_key,
					self._get_ip(),
					self.regex_jobid_extract.sub("\\1", data["filename"])
				)
				response = requests.get(download_url)
				self.current_job = None
				if response.status_code == 200:
					self._logger.debug("server responded: {}".format(response.text))
					return flask.jsonify({"bed_cleared": True, "enabled": self.autodrop3d_enabled})
				else:
					self._logger.debug("server responded: {}".format(response.text))
					return flask.jsonify({"unknown response": data["filename"]})

		if self._settings.get_boolean(["polling_enabled"]) != self.autodrop3d_enabled:
			self._settings.set_boolean(["polling_enabled"], self.autodrop3d_enabled)
			self._settings.save(trigger_event=True)
		response = {"enabled": self.autodrop3d_enabled}
		return flask.jsonify(response)

	# ~~ Access Permissions Hook

	def get_additional_permissions(self, *args, **kwargs):
		return [
			dict(
				key="CONTROL",
				name="Control Connection",
				description=gettext("Allows control of the Autodrop3D connection."),
				roles=["admin"],
				dangerous=True,
				default_groups=[ADMIN_GROUP],
			)
		]

	# ~~ SettingsPlugin mixin

	def get_settings_defaults(self):
		return dict(
			name="", key="", server="https://go.autodrop3d.com/api/jobsQueue/printerRequestJob", polling_enabled=True,
			polling_interval=10, custom_script="", notify_complete=False, use_gpio=False, at_commands_to_monitor=[]
		)

	# ~~ AssetPlugin mixin

	def get_assets(self):
		return dict(js=["js/autodrop3d.js"], css=["css/autodrop3d.css"])

	# ~~ TemplatePlugin mixin

	def get_template_configs(self):
		return [
			dict(type="navbar", custom_bindings=True),
			dict(type="settings", custom_bindings=True),
		]

	# ~~ Softwareupdate hook

	def get_update_information(self):
		return dict(
			autodrop3d=dict(
				displayName="Autodrop3D",
				displayVersion=self._plugin_version,
				# version check: github repository
				type="github_release",
				user="Autodrop3d",
				repo="AD3d-octoprint-connect",
				current=self._plugin_version,
				stable_branch=dict(
					name="Stable", branch="master", comittish=["master"]
				),
				prerelease_branches=[
					dict(
						name="Release Candidate",
						branch="rc",
						comittish=["rc", "master"],
					)
				],
				# update method: pip
				pip="https://github.com/Autodrop3d/AD3d-octoprint-connect/archive/{target_version}.zip",
			)
		)

__plugin_name__ = "Autodrop3D"
__plugin_pythoncompat__ = ">=3,<4"

def __plugin_load__():
	global __plugin_implementation__
	__plugin_implementation__ = autodrop3d()

	global __plugin_hooks__
	__plugin_hooks__ = {
		"octoprint.access.permissions": __plugin_implementation__.get_additional_permissions,
		"octoprint.comm.protocol.atcommand.queuing": __plugin_implementation__.custom_atcommand_handler,
		"octoprint.plugin.softwareupdate.check_config": __plugin_implementation__.get_update_information,
	}


import sys
sys.path.insert(0, '.')

import functools
import os
import sqlite3
import time

from tqdm import tqdm

from web.cloudygo import CloudyGo

ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
DATABASE_PATH = os.path.join(ROOT_DIR, 'instance', 'clouds.db')

BOARD_SIZE = 19
BUCKET = 'v9-19x19'

#### DB STUFF ####

db = sqlite3.connect(DATABASE_PATH)
db.row_factory = sqlite3.Row

def query_db(query, args=()):
    cur = db.execute(query, args)
    data = list(map(tuple, cur.fetchall()))
    cur.close()
    return data

model_range = CloudyGo.bucket_model_range(BUCKET)
model_creation = query_db('SELECT model_id, creation '
                          'FROM models WHERE bucket = ?', (BUCKET,))
model_ids, creation_times = zip(*model_creation)
model_guesser = functools.partial(
    CloudyGo._model_guesser,
    model_mtimes=creation_times,
    model_ids=model_ids)

T0 = time.time()

game_data = query_db('SELECT timestamp, game_num, filename, model_id '
                     'FROM games WHERE model_id BETWEEN ? AND ?', model_range)
equal = 0
results = []
for ts, game_num, filename, model_id in tqdm(game_data, unit="game"):
    guess_id = model_guesser(filename)

    if guess_id == model_id:
        equal += 1
    else:
        results.append((guess_id, ts, game_num))

T1 = time.time()
print('Model Id Guess took: {:.1f} seconds'.format(T1 - T0))

print("\t", results[:3], results[-3:])

cur = db.executemany(
    'UPDATE games SET model_id = ? WHERE timestamp = ? AND game_num = ?',
    results)
db.commit()

T2 = time.time()

print('{} == {} rows updated, {} unchanged'.format(
    len(results), cur.rowcount, equal))
print('Update took: {:.1f} seconds'.format(T2 - T1))


# sudo apt-get install python-scipy
# sklearn
# numpy

import numpy as np
X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
Y = np.array([1, 1, 1, 2, 2, 2])
from sklearn.naive_bayes import GaussianNB
# create classifier
clf = GaussianNB()
# train
# X - features
# Y - lables
clf.fit(X, Y)

print(clf.predict([[-0.8, -1]]))

clf_pf = GaussianNB()
clf_pf.partial_fit(X, Y, np.unique(Y))

print(clf_pf.predict([[-0.8, -1]]))