"""here functions transfered from notebook. To be merged with other notebooks ICSO2020."""

import numpy as np
from matplotlib import pyplot as plt
from dataIO.fn_add_subfix import fn_add_subfix
from dataIO.span import span
from pyProfile.profile import crop_profile, save_profile
from pyProfile.psd import plot_psd, psd_units
import os
from scipy.stats import binned_statistic
from pySurf.readers.instrumentReader import matrixdat_reader,matrixZygo_reader, matrix_reader, matrix4D_reader
#from pySurf.readers.nid_reader import read_nid
from pySurf.readers.format_reader import read_nid
from pySurf.data2D_class import Data2D
from pyProfile.profile_class import Profile, Plist
from plotting.backends import maximize
import pdb

from time import sleep

'''
def dopsd(files,name,outfolder=None,rmsthr=None,psdrange=None,frange=None):
    plt.close('all')

    if not os.path.exists(outfolder):
        os.mkdir(outfolder)
    for f in files:
        try:
            print (f)
            datadic = read_nid(f)
            data,x,y = datadic['Gr0-Ch1']
            d = Data2D(data,x,y,units=['um','um','nm'],scale=[1000000.,1000000.,1000000000.],name=name)
            plt.figure(1)
            if f == 'Image00098.nid':
                d=d.crop(None,[30.,45])

            dd = d.level(2,axis=1)
            dd.plot(stats=[0,1],nsigma=1)
            if outfolder:
                plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'','.png',strip=True)))

            dd=dd.transpose()
            p2 = dd.psd(analysis=True,fignum=2,rmsthr=rmsthr)
            if outfolder:
                plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_psd2d','.png',strip=True)))

            fs,ps = p2.avgpsd()
            plt.figure(3)
            plot_psd(fs,ps,units=['mm','mm','nm'])
            plt.ylim(psdrange)
            plt.xlim(frange)
            plt.title(name)
            if outfolder:
                np.savetxt(os.path.join(outfolder,fn_add_subfix(f,'_psd','.dat',strip=True)),
                    np.vstack([fs,ps]).T,header='# '+','.join(psd_units(['mm','mm','nm'])[1:]))
        except KeyError:
            pass

    if outfolder:
        plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_psd','.png',strip=True)))
'''

# without hardcoded exceptions edit

def dopsd(files,titles,outfolder=None,rmsthr=None,psdrange=None,frange=None,axis=1):
    """from review.
    files: full path to input file
    titles: can be a single string or array same length of files, used for plot titles
    Make PSDs from AFM data from a list of .nid files generating some outputs if `outfolder` is provided.
    Return psds as couples f,psd with proper units for AFM images.
    rmsthr is used in calculation of psd for Data2D object.
    psdrange and frange are purely a visual parameters for the output.
    """
    plt.close('all')

    os.makedirs(outfolder,exist_ok=True)
    psds = []

    from dataIO.arrays import is_iterable

    if not is_iterable(titles): titles = [titles for f in files]
    #if np.size(titles) == 1:
    #    titles = [titles for f in files]
    #pdb.set_trace()

    for f,n in zip(files,titles):
        try:
            # replace with data,x,y = read_nid
            # units um,um,mm; scae=1e,1e6,1e96
            print (f)
            #a = read_nid(f)
            data,x,y = read_nid(f)  # a['Gr0-Ch1']
            d = Data2D(data,x,y,units=['um','um','nm'],scale=[1000000.,1000000.,1000000000.],
                       center = (0,0), name=n)

            # custom edit for some files, level all deg 2 along x (individual lines) in dd
            # if f == 'Image00098.nid':
                # d=d.crop(None,[30.,45])
            # rmsthr= 0.0007 if f == 'Image00095.nid' else rmsthr
            dd = d.level(2,axis=1)

            # plot
            plt.figure(1) #,figsize=(10,8))
            plt.clf()
            dd.plot(stats=1,nsigma=3)
            #plt.title(os.path.basename(f))
            if outfolder:
                plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'','.png',strip=True)))

            #psd of transposed
            dd=dd.transpose()
            #pdb.set_trace()
            maximize()
            p2 = dd.psd(analysis=True,fignum=2,rmsthr=rmsthr)
            #plt.title(os.path.basename(f))
            if outfolder:
                plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_psd2d','.png',strip=True)))

            # average psd
            try:
                fs,ps = p2.avgpsd()
            except:
                fs,ps = p2.avgpsd()()  # probably changed to return profile, convert to x,y
            plt.figure(3) #,figsize=(10,8))
            plot_psd(fs,ps,units=p2.units) # psd_units(p2.units))
            plt.ylim(psdrange)
            plt.xlim(frange)
            #plt.title(os.path.basename(f))

            # make outputs
            psds.append([fs,ps])
            if outfolder:
                np.savetxt(os.path.join(outfolder,fn_add_subfix(f,'_psd','.dat',strip=True)),
                    np.vstack([fs,ps]).T,header='# '+','.join(psd_units(p2.units)[1:]))
                plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_psd','.png',strip=True)))
            #    np.savetxt(os.path.join(outfolder,fn_add_subfix(f,'_psd','.dat',strip=True)),
            #        np.vstack([fs,ps]).T,header='# '+','.join(psd_units(['mm','mm','nm'])[1:]))

        except KeyError:
            pass

    return psds

'''
def mft_psd(files,outfolder,rmsthr=None,psdrange=None,frange=None):

    plt.close('all')
    plt.figure(1)
    ax1 = plt.gca()
    plt.figure(2)
    ax2 = plt.gca()
    plt.figure(3)
    ax3 = plt.gca()

    if outfolder:
        if not os.path.exists(outfolder):
            os.mkdir(outfolder)
    for f in files:
        print (f)
        plt.figure(1)
        plt.clf()
        if os.path.splitext(f)[-1] == '.csv':
            data, x, y = matrix4D_reader(f,center=(0,0),scale = [1e6,1e6,1])  #note that center works also with `None`
        elif os.path.splitext(f)[-1] == '.dat':
            data, x, y = matrixdat_reader(f,center=(0,0),
                     scale = [1.e6,1.e6,1.e9])  #note that center works also with `None`, data are in m
        else:
            raise TypeError ("Unrecognized data format for MFT")
        D = Data2D(data, x, y , units = ['um','um','nm'],name = os.path.basename(f))
        D=D.remove_nan_frame()

        D.plot(stats = 2)
        if outfolder:
            plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_MFT','.png',strip=True)))

        plt.figure(2)
        plt.clf()
        p2 = D.transpose().level(1,axis=0).psd(analysis=True)
        if outfolder:
            plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_MFT_psd2d','.png',strip=True)))

        fs,ps = p2.avgpsd()
        plt.figure(3)
        plt.clf()
        plot_psd(fs,ps,units=['um','um','nm'])
        plt.title(name)
        if outfolder:
            np.savetxt(os.path.join(outfolder,fn_add_subfix(f,'_MFT_psd','.dat',strip=True)),
                np.vstack([fs,ps]).T,header='# '+','.join(psd_units(['um','um','nm'])[1:]))
    if outfolder:
        plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_MFT_psd','.png',strip=True)))
'''

#DA REVIEW, EDITED
def mft_psd(files,outfolder=None,rmsthr=None,psdrange=None,frange=None):
    """read MFT file. Remove nan frame.
    PSD of data after individual line leveling along y.
    return list of Data2D objects.
    If outfolder is given, output files are genearted, adding _MFT subfix to each file name.
        This is needed to avoid name conflicts, data related to same sample are saved in same folder,
        even if instruments differ."""
    # TODO plot all PSD together at the end.

    res = []
    if outfolder:  # save files only if defined
        os.makedirs(outfolder,exist_ok=True)
    for f in files:
        print (f)
        plt.figure(1)
        plt.clf()
        if os.path.splitext(f)[-1] == '.csv':
            data, x, y = matrix4D_reader(f,center=(0,0),scale = [1e6,1e6,1])  #note that center works also with `None`
        elif os.path.splitext(f)[-1] == '.dat':
            data, x, y = matrixdat_reader(f,center=(0,0),
                     scale = [1.e6,1.e6,1.e9])  #note that center works also with `None`, data are in m
        else:
            raise TypeError ("Unrecognized data format for MFT")
        name = os.path.basename(f)
        D = Data2D(data, x, y , units = ['um','um','nm'],name = name)
        D=D.remove_nan_frame().level()

        res.append(D)
        D.plot(stats=1)
        if outfolder:
            plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_MFT','.png',strip=True)))
        # 2D PSD ANALYSIS, ORIZONTAL LINES INDIVIDUALLY LEVELED
        plt.figure(2)
        plt.clf()
        p2 = D.transpose().level(1,axis=0).psd(analysis=True)
        if outfolder:
            plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_MFT_psd2d','.png',strip=True)))

        # 1D PSD
        fs,ps = p2.avgpsd()
        plt.figure(3)
        plt.clf()
        plot_psd(fs,ps,units=p2.units)
        plt.title(name)
        if outfolder:
            np.savetxt(os.path.join(outfolder,fn_add_subfix(f,'_MFT_psd','.dat',strip=True)),
                np.vstack([fs,ps]).T,header='# '+','.join(psd_units(['um','um','nm'])[1:]))
            plt.savefig(os.path.join(outfolder,fn_add_subfix(f,'_MFT_psd','.png',strip=True)))

    return res # 2021/06/30

def trim_psds_group(ranges, datafiles, labels, xrange=None,yrange=None,
                    outname=None):
    """ Makes a single psd trimming and averaging multiple ones according to selection.
        An equivalent updated version (or an attempt to it) in trim_psds_group2

        this is lauched to trim a single set of psds related to same sample (or same context).
        ranges, datafiles, labels are lists with same number of elements,
        describing respectively:
        ranges for each psd (if None, the psd is not included in the output).
            If set to None, include all data.
        datafiles full paths from which to read psds
            after adding '_psd' subfix and .dat extension to filename.
            One line header is assumed, zero freq
            element is removed from starting data if present.
        labels to be used in plot.
        outname: if provided generate plot of trim and txt with resulting psd.

        """
    if ranges is None:
        ranges = np.repeat([[None],[None]],
                           len(datafiles),axis=1).T
    xtot,ytot,bins,xvals,yvals = [] , [], [], [], []
    plt.figure(figsize=(12,6))
    for ran,fn,lab in zip(ranges,datafiles,labels):
        if ran is not None:
            x,y = np.genfromtxt(fn_add_subfix(fn,'_psd','.dat'),unpack=True,skip_header=1, comments='#')
            if len(x)==0:
                raise ValueError('No data read from file '+fn_add_subfix(fn,'_psd','.dat'))
            if x[0] == 0:
                x=x[1:]
                y=y[1:]
            try:
                xx,yy = crop_profile(x,y,ran)
            except(IndexError):
                Warning('possible conflict between range and PSD values for input file\n'+
                        fn_add_subfix(fn,'_psd','.dat')+
                        'and range'+str(ran))
                _ = psds_table(tdic) # in case of problems with range, run some diagnostics

            #print(x,y)
            plot_psd(xx,yy,label=lab,units=['um','um','nm'])
            xtot.append(xx)
            ytot.append(yy)
        plt.legend( prop={'size': 12},loc = 1) #bbox_to_anchor=(1.05, 1), loc='upper left')
        plt.xlim(xrange)
        plt.ylim(yrange)
        if outname:
            plt.title(os.path.basename(outname))
        plt.grid( which='minor', linestyle='--')
        plt.tight_layout()
    if outname:
        plt.savefig(fn_add_subfix(outname,'_trimpsds','.png'))

    ### The following part is of interest only for rebinning,
    ###   not included in this version's output
    ###   because result will in general not be smooth in transitions between
    ###   different intervals.
    ###   if intervals are not overlapping, it is irrelevant.
    # makes rebinning and averaging.
    # we want to start from lower x (assuming inputs are passed not in order),
    # keep x in non overlapping regions, while averaging on common regions.
    # psd frequencies (x) are not equally spacing. If I do average, I get
    #   spikes when typically lower freqs of each psd are more spaced than higher freq
    #   of same psds. Then overlapping intervals of two psds typically have one with
    #   broader spacing. Most of intervals have points only from psds with tighter spacing
    #      when points from the other enter, you get spike. This is why interpolation is needed.
    #sort groups in xtot in ascending order
    pmin=[a.min() for a in xtot]
    igroup =np.argsort(pmin)
    for i in igroup:
        x = xtot[i]
        y = ytot[i]
        if len(bins) == 0:
            bins.append(x)
            xvals.append(x)
            yvals.append(y)
        else:
            sel = x>max(bins[-1])
            xint = np.hstack([ bins[-1][bins[-1]>=min(x)] , x[sel] ])
            #pdb.set_trace()
            xvals.append(xint)
            yvals.append(np.interp(xint,x,y))
            if any(sel):
                #pdb.set_trace()
                #resample second vector on common range
                bins.append(x[sel])
    xtot=np.hstack(xtot)
    ytot=np.hstack(ytot)
    xvals=np.hstack(xvals)
    yvals=np.hstack(yvals)
    xbins = np.hstack(bins)
    ybins = binned_statistic(xvals,yvals,bins=xbins,statistic='mean') [0]
    ###

    plot_psd(xbins[:-1],ybins,label='binned',units=['um','um','nm'],
             linestyle='--')
    #plot_psd(xtot,ytot,label='total',units=['um','um','nm'])
    if outname:
        save_profile(fn_add_subfix(outname,'_binpsd','.dat'),xbins[:-1],ybins)

    if outname:
        plt.savefig(fn_add_subfix(outname,'_trimpsds','.png'))

    return xtot, ytot

def trim_psds_group2(ranges, datafiles, labels,xrange=None,yrange=None,
                    outname=None):
    """ Makes a single psd trimming and averaging multiple ones according to selection.

    2022/11/25 attempt to create from previous version above, a function which uses profile functions
    and avoid duplication from pyProfile.merge_profile, where this was cloned.
    prepare to accept Profiles instead of datafiles.
    The version in profile merge is basically the same with output removed,
    can be handled with Plists.

    this is lauched to trim a single set of psds related to same sample (or same context).
    ranges, datafiles, labels are lists with same number of elements,
    describing respectively:
    ranges for each psd (if None, the psd is not included in the output).
        If set to None, include all data.
    datafiles full paths from which to read psds
        after adding '_psd' subfix and .dat extension to filename.
        One line header is assumed, zero freq
        element is removed from starting data if present.
    labels to be used in plot.
    outname: if provided generate plot of trim and txt with resulting psd.

        """
    def make_Plist(datafiles):
        """create Plist, this will be the data passed when finished."""

        Pl = []
        for fn in datafiles:
            x,y = np.genfromtxt(fn_add_subfix(fn,'_psd','.dat'),unpack=True,skip_header=1)
            if x[0] == 0:
                x=x[1:]
                y=y[1:]
            Pl.append(Profile(x,y,name=fn))
        return Plist(Pl)

    if ranges is None:
        ranges = np.repeat([[None],[None]],
                           len(datafiles),axis=1).T

    Profiles = make_Plist(datafiles)
    # Profiles.crop(ran) # iterate over Plist
    Profiles = [P.crop(ran) for ran,P in zip(ranges,Profiles)]
    for P,lab in zip(Profiles,labels):
        P.name = lab

    # xtot,ytot,bins,xvals,yvals = [] , [], [], [], []

    plt.figure(figsize=(12,6))  #excluded in merge
    Profiles.psd(labels,units = psd_units(['um','um','nm']))
    plt.legend( prop={'size': 12},loc = 1) #bbox_to_anchor=(1.05, 1), loc='upper left')

    #excluded in merge, da qui non ho continuato con Profile.
    plt.xlim(xrange)
    plt.ylim(yrange)
    if outname:
        plt.title(os.path.basename(outname))
    plt.grid( which='minor', linestyle='--')
    plt.tight_layout()
    if outname:  #excluded in merge
        plt.savefig(fn_add_subfix(outname,'_trimpsds','.png'))

    return Plist(Profiles)

    ### The following part is of interest only for rebinning,
    ###   not included in this version's output
    ###   because result will in general not be smooth in transitions between
    ###   different intervals.
    ###   if intervals are not overlapping, it is irrelevant.
    # makes rebinning and averaging.
    # we want to start from lower x (assuming inputs are passed not in order),
    # keep x in non overlapping regions, while averaging on common regions.
    # psd frequencies (x) are not equally spacing. If I do average, I get
    #   spikes when typically lower freqs of each psd are more spaced than higher freq
    #   of same psds. Then overlapping intervals of two psds typically have one with
    #   broader spacing. Most of intervals have points only from psds with tighter spacing
    #      when points from the other enter, you get spike. This is why interpolation is needed.
    #sort groups in xtot in ascending order
    pmin=[a.min() for a in xtot]
    igroup =np.argsort(pmin)
    for i in igroup:
        x = xtot[i]
        y = ytot[i]
        if len(bins) == 0:
            bins.append(x)
            xvals.append(x)
            yvals.append(y)
        else:
            sel = x>max(bins[-1])
            xint = np.hstack([ bins[-1][bins[-1]>=min(x)] , x[sel] ])
            #pdb.set_trace()
            xvals.append(xint)
            yvals.append(np.interp(xint,x,y))
            if any(sel):
                #pdb.set_trace()
                #resample second vector on common range
                bins.append(x[sel])
    xtot=np.hstack(xtot)
    ytot=np.hstack(ytot)
    xvals=np.hstack(xvals)
    yvals=np.hstack(yvals)
    xbins = np.hstack(bins)
    ybins = binned_statistic(xvals,yvals,bins=xbins,statistic='mean') [0]
    ###

    #excluded in merge:
    plot_psd(xbins[:-1],ybins,label='binned',units=['um','um','nm'],
             linestyle='--')
    #plot_psd(xtot,ytot,label='total',units=['um','um','nm'])
    if outname:
        save_profile(fn_add_subfix(outname,'_binpsd','.dat'),xbins[:-1],ybins)
    if outname:
        plt.savefig(fn_add_subfix(outname,'_trimpsds','.png'))

    return xtot, ytot

'''
def trim_datadic(datadic,to_trim,xrange=None,yrange=None,
                 tdic=None,outname=None):
    """"""
    if tdic is None:
        tdic = {}  #trimmed dic
    for name in to_trim:
        r,(d,l) = to_trim[name],datadic[name]   #range, path and label
        xtot, ytot = trim_psds_group(r,d,l,outname=outname)
        tdic[name] = [xtot,ytot]
    return tdic

def trim_datadic(ranges,datadic_val,xrange=None,yrange=None,
                 outname=None):
    """inutile esempio di come chiamare la funzione."""
    return trim_psds_group(ranges,*datadic_val,outname=outname,
                           xrange=xrange,yrange=yrange)
'''

def plot_all_psds(pathlabels,outfolder=None,subfix='_psd',xrange=None,yrange=None):
    """passing a datadic plots all psds for each key.

    Generate a formatted plot (N.B.: units are fixed) of all PSDs for zipping pathlabel in a single graph
    and prints their rms.
    Return a list of psds corresponding to the ones read from `pathlabels`, list of couples (file,label)
    No output files are created or returned, must be saved externally.

    Note that this was initially accepting a dictionary, of pathlabel values, generating one plot
        per each group of psds, but keys were not used, so this was deprecated.
        If this is the desired behavior, loop can be handled externally.

    `pathlabels` is a 2-el list in format [datafile_full_paths,labels] as the values of datadic,
    (modified 2023/08/20 with deprecation warning, it was {label:[(datafile1_full_path,label1),..]} i.e. datadic).
    `subfix` is the subfix of input files, whose names are in as first item. They must contains freq. e PSD on two cols.

    `outfolder` is deprecated 2023/08/20 it was only saving plots overwriting them, it can be done externally.

    e.g.:

        psds = plot_all_psds(pathlabels,outfolder=None,subfix='_psd',xrange=None,yrange=None)
        # plt.title('test')   #modify plot as you like
        plt.savefig('plot_all_psds_demo.png')

    old version: Output files are created in outfolder and sequentially numbered as ##_fullpsd.png using a sequence of numbers
    (can be improved, but keys shouldn't be used as they can contain invalid characters,
    while using info from datafiles can give raise to repetitions."""

    if type(pathlabels) == dict:
        DeprecationWarning("WARNING: deprecated, modify your code to accept a list [fullpaths_beforesubfix, labels]")
        for k in pathlabels.keys():
            plot_all_psds(pathlabels[k],
                            outfolder = outfolder, subfix = subfix, xrange = xrange, yrange = yrange)
        sleep (10)
        return

    filelist, labels = pathlabels
    psds = []

    plt.figure(figsize=(12,6))
    for d,l in zip(filelist,labels):
        #read psds from files, add to plot and tdic
        fs,ps = np.genfromtxt(fn_add_subfix(d,subfix,'.dat'), unpack=True,skip_header=1)
        if fs[0] == 0:
            fs = fs[1:]
            ps = ps[1:]
        rms = np.sqrt(np.trapz(ps,fs))
        print (os.path.basename(d),l,' : ',rms)
        plot_psd(fs,ps,label=l+', rms = %6.2f nm'%rms,units=['$\mu$m','$\mu$m','nm'])
        psds.append((fs,ps))
    plt.xlabel('Spatial frequency ($\mu$m$^{-1}$)')
    plt.ylabel('PSD (nm$^{2}$ $\mu$m)')
    plt.legend( prop={'size': 12},loc = 1) #bbox_to_anchor=(1.05, 1), loc='upper left')
    plt.xlim(xrange)
    plt.ylim(yrange)
    plt.gca().xaxis.label.set_size(12)
    plt.gca().yaxis.label.set_size(12)
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12)
    #plt.title(name)  #it doesn't make sense, as it ends up using the last key
    plt.grid( which='minor', axis = 'x', linestyle='--')
    plt.tight_layout()
    # if outfolder:
    #     fn = '%02i'%i #os.path.basename(os.path.dirname(d))  #last file containing folder
    #     plt.savefig(fn_add_subfix(fn,'_fullpsd','.png',pre=outfolder+os.path.sep))

    return psds

# MAKE PSD TABLE
from pyProfile.profile import crop_profile
def psds_table(tdic,outfolder=None,subfix='_psdtable',ranges = None):
    """ make a table of rms from trimmed PSD in tdic as sqrt of psd integral over range.

    This is mostly useless, can be incorporated in plot_all_psds or similer, as psd
    outfolder not used, kept only for format consinstency. """
    rdic = {}
    for i,name in enumerate(tdic.keys()):
        rmss=[]
        fs,ps = tdic[name]
        isort = np.argsort(fs)
        fs = fs[isort]
        ps = ps[isort]
        if min(fs) == 0:
            print('attenzione')
            fs=fs[1:]
            ps=ps[1:]
        rms_full = np.sqrt(np.trapz(ps,fs))
        print (os.path.basename(name),' : ','full freq. range [%6.3f:%6.3f], rms %6.3f'%(*(span(fs)),rms_full))
        rmss.append([span(fs),rms_full])
        #pdb.set_trace()
        if ranges is not None:   #replace with a more sofisticated filter, from other functions
            for r in ranges:
                if r[0] is None:    #None set auto
                    r[0] = np.min(fs)
                if r[1] is None:
                    r[1] = np.max(fs)

                if r[0]<fs[0] or r[1]>fs[-1]:       # if the range is larger than data, return nan
                    rms = np.nan
                else:
                    f,p = crop_profile(fs,ps,r)
                    rms = np.sqrt(np.trapz(p,f))
                print (os.path.basename(name),' : ','freq. range [%6.3f:%6.3f], rms %6.3f'%(r[0],r[1],rms))
                rmss.append([r, rms])
        rdic[name] = rmss
    return rdic

#(fs,ps,label=l+', rms = %6.2f nm'%rms,units=['um','um','nm'])

#if outfolder:
#    plt.savefig(fn_add_subfix(fn,'_fullpsd','.png',pre=outfolder+os.path.sep))

'''
# From MIRO_analysis
def psd_compare(plotnames,psdfolder,labels = None, outname=None,
                subfix='_binpsd',xrange=None,yrange=None):
    """Uses `plot_all_psds` to plot comparison between different (binned) psds.

    plotnames is a list of keys used to generate input psd files (the only requirement is that
        `"psdfolder\plotnames+subfix.dat` is a valid psd file).
        Here they can be `tdic` keys.

    outname is a full path whose last part is used both as title for plot and as filename
      for generated plot after adding '_psdcomp' subfix.

    labels if provided are used for each plot line

    """
    #   tag is used passed to plot_all_psd where is used for print and title
    #   (no output files are generated).
    #   output file is created at savefig.

    tag = os.path.basename(outname)
    if labels is None:
        labels = plotnames

    toplot = {tag:[[fn_add_subfix(name,subfix,'.dat',pre=psdfolder+os.path.sep)
                    for name in plotnames],labels]}

    plot_all_psds(toplot,outfolder=None,subfix='',xrange=xrange,yrange=yrange)
    if outname:
        plt.savefig(fn_add_subfix(outname,'_psdcomp','.png'))

'''
def psd_compare(filelist,psdfolder,labels = None,
                subfix='_binpsd',xrange=None,yrange=None):
    """Uses `plot_all_psds` to plot comparison between different (binned) psds.

    filelist is a list of files in psdfolder to use input psd files
    as `psdfolder\plotnames+subfix.dat'.
    labels if provided are used for each plot line

    2023/08/20 modified for a more consistent interface, it is mostly a different
    interface to plot_all_psds. Can be removed in future versions,
    see e.g. MUSE_analysis notebooks.
    """
    #

    if labels is None:
        labels = filelist

    filelist = [fn_add_subfix(name,subfix,'.dat',pre=psdfolder+os.path.sep)
                    for name in plotnames]

    plot_all_psds(filelist,labels,outfolder=None,subfix='',xrange=xrange,yrange=yrange)

'''

# From review
def psd_compare(plotnames,psdfolder,labels = None, outname=None,
                subfix='_binpsd',xrange=None,yrange=None):
    """Uses `plot_all_psds` to plot comparison between different (binned) psds.

    plotnames is a list of keys used to generate input psd files (the only requirement is that
        `"psdfolder\plotnames+subfix.dat` is a valid psd file).
        Here they can be `tdic` keys.

    outname is a full path whose last part is used both as title for plot and as filename
      for generated plot after adding '_psdcomp' subfix.

    labels if provided are used for each plot line

    """
    #   tag is used passed to plot_all_psd where is used for print and title
    #   (no output files are generated).
    #   output file is created at savefig.

    tag = os.path.basename(outname)
    if labels is None:
        labels = plotnames

    toplot = {tag:[[fn_add_subfix(name,subfix,'.dat',pre=psdfolder+os.path.sep)
                    for name in plotnames],labels]}

    plot_all_psds(toplot,outfolder=None,subfix='',xrange=xrange,yrange=yrange)
    if outname:
        plt.savefig(fn_add_subfix(outname,'_psdcomp','.png'))
        '''


import sys
import os
import glob
import argparse
#from ..common.bounding_box import BoundingBox
import subprocess
import json
from rh_logger.api import logger
import rh_logger
import logging
import multiprocessing as mp
import numpy as np

def add_transformation(in_file, out_file, transform, deltas):
    # load the current json file
    try:
        with open(in_file, 'r') as f:
            data = json.load(f)
    except:
        logger.report_event("Error when reading {} - Exiting".format(in_file), log_level=logging.ERROR)
        raise

    if deltas[0] != 0.0 and deltas[1] != 0.0:
        for tile in data:
            # Update the transformation
            if "transforms" not in tile.keys():
                tile["transforms"] = []
            tile["transforms"].append(transform)

            # Update the bbox
            if "bbox" in tile.keys():
                bbox = tile["bbox"]
                bbox_new = [bbox[0] - deltas[0], bbox[1] - deltas[0], bbox[2] - deltas[1], bbox[3] - deltas[1]]
                tile["bbox"] = bbox_new

    with open(out_file, 'w') as f:
        json.dump(data, f, indent=4)

def read_minxy_grep(tiles_spec_fname):
    cmd = "grep -A 5 \"bbox\" {}".format(tiles_spec_fname)
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

    # Output will be of the following format:
    """
        "bbox": [
            13262.416015625,
            16390.580476722345,
            10470.920522462373,
            13195.109375
        ],
--
        "bbox": [
            16273.984836771779,
            19402.12109375,
            10470.3515625,
            13194.508027928736
        ],
--
...
    """

    # Parse all bounding boxes in the given json file
    lines = p.stdout.readlines()
    min_x = np.min([float(line.decode('utf-8').strip(' ,\n')) for line in lines[1::7]])
    #max_x = np.max([float(line.decode('utf-8').strip(' ,\n')) for line in lines[2::7]])
    min_y = np.min([float(line.decode('utf-8').strip(' ,\n')) for line in lines[3::7]])
    #max_y = np.max([float(line.decode('utf-8').strip(' ,\n')) for line in lines[4::7]])
    #return np.array([min_x, max_x, min_y, max_y])
    return [min_x, min_y]

#     ret_val = None
#     cur_bbox_lines = []
#     for line in iter(p.stdout.readline, ''):
#         if line.startswith("--"):
#             cur_bbox = BoundingBox.fromStr(BoundingBox.parse_bbox_lines(cur_bbox_lines))
#             if ret_val is None:
#                 ret_val = cur_bbox
#             else:
#                 ret_val.extend(cur_bbox)
#             cur_bbox_lines = []
#         else:
#             cur_bbox_lines.append(line.strip(' \n'))
#     if len(cur_bbox_lines) > 0:
#         cur_bbox = BoundingBox.fromStr(BoundingBox.parse_bbox_lines(cur_bbox_lines))
#         if ret_val is None:
#             ret_val = cur_bbox
#         else:
#             ret_val.extend(cur_bbox)
#     return ret_val

def normalize_coordinates(tile_fnames_or_dir, output_dir, pool):
    # Get all the files that need to be normalized
    all_files = []

    logger.report_event("Reading {}".format(tile_fnames_or_dir), log_level=logging.INFO)
    for file_or_dir in tile_fnames_or_dir:
        if not os.path.exists(file_or_dir):
            logger.report_event("{0} does not exist (file/directory), skipping".format(file_or_dir), log_level=logging.WARN)
            continue

        if os.path.isdir(file_or_dir):
            actual_dir_files = glob.glob(os.path.join(file_or_dir, '*.json'))
            all_files.extend(actual_dir_files)
        else:
            all_files.append(file_or_dir)

    if len(all_files) == 0:
        logger.report_event("No files for normalization found.", log_level=logging.ERROR)
        raise Exception("No files for normalization found.")

    logger.report_event("Normalizing coordinates of {} files".format(len(all_files)), log_level=logging.INFO)

    # Retrieve the bounding box of these files
    entire_image_bbox = None

    # merge the bounding boxes to a single bbox
    if len(all_files) > 0:
        sections_min_xys = np.array(pool.map(read_minxy_grep, all_files))
#         entire_image_bbox = np.array([
#                                 np.min(sections_bboxes[:, 0]), np.max(sections_bboxes[:, 1]),
#                                 np.min(sections_bboxes[:, 2]), np.max(sections_bboxes[:, 3])
#                             ])

    # Set the translation transformation
    #deltaX = entire_image_bbox[0]
    #deltaY = entire_image_bbox[2]
    deltaX = np.min(sections_min_xys[:, 0])
    deltaY = np.min(sections_min_xys[:, 1])

    logger.report_event("Entire 3D image min x,y: {}, {}".format(deltaX, deltaY), log_level=logging.INFO)

    # TODO - use models to create the transformation
    transform = {
            "className" : "mpicbg.trakem2.transform.TranslationModel2D",
            "dataString" : "{0} {1}".format(-deltaX, -deltaY)
        }

    # Add the transformation to each tile in each tilespec
#     out_files = [os.path.join(output_dir, os.path.basename(in_file)) for in_file in all_files]
#     pool.map(lambda in_f, out_f: add_transformation(in_f, out_f, transform, [deltaX, deltaY]), zip(all_files, out_files))
    pool_results = []
    for in_file in all_files:
        out_file = os.path.join(output_dir, os.path.basename(in_file))
        res = pool.apply_async(add_transformation, (in_file, out_file, transform, [deltaX, deltaY]))
        pool_results.append(res)

    for res in pool_results:
        res.get()

if __name__ == '__main__':
    # Command line parser
    parser = argparse.ArgumentParser(description='Given a list of tilespec file names, normalizes all the tilepecs to a single coordinate system starting from (0,0).')
    parser.add_argument('tile_files_or_dirs', metavar='tile_files_or_dirs', type=str, nargs='+',
                        help='a list of json files that need to be normalized or a directories of json files')
    parser.add_argument('-o', '--output_dir', type=str,
                        help='an output directory (default: ./after_norm)',
                        default='./after_norm')
    parser.add_argument('-p', '--processes_num', type=int,
                        help='number of processes (default: 1)',
                        default=1)

    args = parser.parse_args()

    logger.start_process('normalize_coordinates', 'normalize_coordinates.py', [args.tile_files_or_dirs, args.output_dir])
    pool = mp.Pool(processes=args.processes_num)

    normalize_coordinates(args.tile_files_or_dirs, args.output_dir, pool)

    pool.close()
    pool.join()

    logger.end_process('normalize_coordinates', rh_logger.ExitCode(0))


from rest_framework import serializers

from .models import Location

# класс позволяющий преобразовать объект класса Location в объект JSON
class LocationSerializer(serializers.ModelSerializer):
    class Meta:
        # в качестве класса модели используется класс Location
        model = Location
        # поля: id, широта, долгота, время добавления
        fields = ('id', 'latitude', 'longitude', 'datetime')
        # данные заносятся через POST только в широту и долготу,
        # id и время выставляются автоматически (поэтому readonly)
        read_only_fields = ('id', 'datetime')

    def create(self, validated_data):
        # создание объекта Location
        # данные заносятся через пост только в широту и долготу,
        # id и время выставляются автоматически
        location = Location.objects.create(
            latitude=validated_data['latitude'],
            longitude=validated_data['longitude']
        )
        # сохранение объекта в БД
        location.save()
        # возврат созданного объекта
        return location


import os

import requests
import web3
from eth_account import Account
from flask import Flask, request, json, Response
from flask_jwt_extended import JWTManager, jwt_required, get_jwt_identity, get_jwt
from sqlalchemy import and_, or_
from sqlalchemy_utils import database_exists, create_database
from web3 import Web3, HTTPProvider

from role_check import roleCheck
from configuration import Configuration
from models import *

import io
import csv

def read_file(path):
	with open(path, 'r') as file:
		return file.read()

application = Flask ( __name__ )
application.config.from_object ( Configuration )
jwt = JWTManager ( application )

host_bc="localhost"
try:
    host_bc=os.environ['BC_HOST']
except:
    pass

web3 = Web3(HTTPProvider(f"http://{host_bc}:8545"))
admin_acc = web3.eth.accounts[0]
abifile=read_file("./contracts/Contract.abi")
bytefile=read_file("./contracts/Contract.bin")
contract = web3.eth.contract(abi=abifile,bytecode=bytefile)

def ret_msg(msg):
    return json.dumps({"message":msg})

@application.route("/update",methods=["POST"])
@jwt_required()
@roleCheck ( role = "admin" )
def update():
    file=None
    try:
        file=request.files["file"]
    except:
        return Response(ret_msg("Field file is missing."),status=400)
    content = file.stream.read ( ).decode ( "utf-8" )
    stream = io.StringIO ( content )
    reader = csv.reader ( stream )

    products = []
    product_categories=dict()
    categories = []
    c_appeared=[]
    rows=[]
    for row in reader:
        rows.append(row)

    for i in range(len(rows)):
        row=rows[i]
        if len(row)<3 or len(row[0])==0 or len(row[1])==0 or len(row[2])==0:
            return Response(ret_msg(f"Incorrect number of values on line {i}."),status=400)

    for i in range(len(rows)):
        row=rows[i]
        try:
            if float(row[2]) <= 0:
                return Response(ret_msg( f"Incorrect price on line {i}."), status=400)
        except:
            return Response(ret_msg(f"Incorrect price on line {i}."), status=400)
    for row in rows:
        if len(Product.query.filter(Product.name == row[1]).all()) > 0:
            return Response(ret_msg(f"Product {row[1]} already exists."), status=400)

    for row in rows:
        category_names = row[0].split('|')
        name=row[1]
        price=float(row[2])

        for category_name in category_names:
            category = Category.query.filter(Category.name == category_name).all()
            if len(category) > 0: continue
            if category_name in c_appeared: continue

            category = Category(name=category_name)
            categories.append(category)
            c_appeared.append(category_name)

        product = Product(name=name, price=price)
        products.append(product)

        product_categories[product] = category_names

    database.session.add_all(categories)
    database.session.add_all (products)
    database.session.commit()

    pcs=[]
    for p in product_categories.keys():
        for cat in product_categories[p]:
            c=Category.query.filter(Category.name==cat).first()
            pc=ProductCategory(idP=p.id, idC=c.id)
            pcs.append(pc)

    database.session.add_all (pcs)
    database.session.commit ()

    return Response(status=200)

@application.route("/product_statistics",methods=["GET"])
@jwt_required()
@roleCheck ( role = "admin" )
def product_statistics():
    IP="localhost"
    try:
        IP=os.environ["IP_ADDR"]
    except:
        pass
    resp = requests.get(f"http://{IP}:5003/product_statistics")
    return Response(resp.text.replace("'",'"'),status=200)

@application.route("/category_statistics",methods=["GET"])
@jwt_required()
@roleCheck ( role = "admin" )
def category_statistics():
    IP="localhost"
    try:
        IP=os.environ["IP_ADDR"]
    except:
        pass
    resp = requests.get(f"http://{IP}:5003/category_statistics")
    # return Response(json.dumps(resp.text),status=200)
    return Response(resp.text.replace("'",'"'),status=200)

@application.route("/search",methods=["GET"])
@jwt_required()
@roleCheck ( role = "customer" )
def search():
    category=""; product=""
    p=[]
    for k,v in request.args.items():
        if k=="name": product=v
        if k=="category": category=v

    query_c=Category.query.join(ProductCategory).join(Product)
    query_p=Product.query.join(ProductCategory).join(Category)

    if len(category)>0:
        query_c=query_c.filter(Category.name.like(f"%{category}%"))
        query_p=query_p.filter(Category.name.like(f"%{category}%"))
    if len(product)>0:
        query_c=query_c.filter(Product.name.like(f"%{product}%"))
        query_p=query_p.filter(Product.name.like(f"%{product}%"))

    categories=query_c.all()
    products=query_p.all()
    product_list=[]
    for p in products:
        obj={
            "categories":[c.name for c in p.categories],
            "id":p.id,
            "name":p.name,
            "price":p.price
        }
        product_list.append(obj)
    cat_list=[c.name for c in categories]

    return Response(json.dumps({
            "categories": cat_list,
            "products": product_list
        }),status=200)

@application.route("/order",methods=["POST"])
@jwt_required()
@roleCheck ( role = "customer" )
def order():
    content=request.json
    try:
        content=content['requests']
    except:
        return Response(ret_msg("Field requests is missing."),status=400)
    for i in range(len(content)):
        if content[i].get("id")==None:
            return Response(ret_msg(f"Product id is missing for request number {i}."),status=400)
    for i in range(len(content)):
        if content[i].get("quantity")==None:
            return Response(ret_msg(f"Product quantity is missing for request number {i}."), status=400)
    for i in range(len(content)):
        if not str(content[i]["id"]).isnumeric() or int(content[i]["id"])<=0:
            return Response(ret_msg(f"Invalid product id for request number {i}."), status=400)
    for i in range(len(content)):
        if not str(content[i]["quantity"]).isnumeric() or int(content[i]["quantity"])<=0:
            return Response(ret_msg(f"Invalid product quantity for request number {i}."), status=400)
    price=0
    for i in range(len(content)):
        try:
            p=Product.query.filter(Product.id==int(content[i]["id"])).all()
            if len(p)==0: return Response(ret_msg(f"Invalid product for request number {i}."), status=400)
            p=p[0]
            price+=p.price*int(content[i]["quantity"])
        except:
            return Response(ret_msg(f"Invalid product for request number {i}."), status=400)

    address=""
    try:
        address=request.json['address']
        if address=="": return Response(ret_msg("Field address is missing."),status=400)
    except:
        return Response(ret_msg("Field address is missing."),status=400)
    if not web3.is_address(address):
        return Response(ret_msg("Invalid address."),status=400)

    #make contract
    transaction_hash = contract.constructor(address).transact({"from": admin_acc})
    receipt = web3.eth.wait_for_transaction_receipt(transaction_hash)

    data = get_jwt()

    order=Order(price=price,status="CREATED",idU=data["email"],contract=receipt.contractAddress, address=address)
    database.session.add(order)
    database.session.commit()
    items=[]
    for r in content:
        item=InOrder(count=r["quantity"],idP=r["id"],idO=order.id)
        items.append(item)

    database.session.add_all(items)
    database.session.commit()

    return Response(json.dumps({"id":order.id}),status=200)

@application.route("/status",methods=["GET"])
@jwt_required()
@roleCheck ( role = "customer" )
def status():
    data = get_jwt()
    orders=Order.query.filter(Order.idU==data["email"])

    ret_list=[]
    for o in orders:
        p_list=[]
        for p in o.products:
            p_list.append({
                "categories":[c.name for c in p.categories],
                "name":p.name,
                "price":p.price,
                "quantity":InOrder.query.filter(and_(InOrder.idP==p.id,InOrder.idO==o.id)).first().count
            })
        ret_list.append({
            "products":p_list,
            "price":o.price,
            "status":o.status,
            "timestamp":o.timestamp
        })

    return Response(json.dumps({"orders":ret_list}),status=200)

@application.route("/delivered",methods=["POST"])
@jwt_required()
@roleCheck ( role = "customer" )
def delivered():
    id = request.json
    try:
        id = id['id']
    except:
        return Response(ret_msg("Missing order id."), status=400)
    try:
        id=int(id)
    except:
        return Response(ret_msg("Invalid order id."), status=400)
    if id<=0:
        return Response(ret_msg("Invalid order id."),status=400)

    order=None
    try:
        order=Order.query.filter(Order.id==int(id)).all()
        if len(order)==0: return Response(ret_msg("Invalid order id."), status=400)
        order=order[0]
    except:
        return Response(ret_msg("Invalid order id."), status=400)

    if order.status!="PENDING":
        return Response(ret_msg("Invalid order id."), status=400)

    keys=None; passphrase=""
    try:
        keys=request.json["keys"]
        if keys=="": return Response(ret_msg("Missing keys."),status=400)
    except:
        return Response(ret_msg("Missing keys."),status=400)
    try:
        passphrase=request.json["passphrase"]
        if passphrase=="": return Response(ret_msg("Missing passphrase."),status=400)
    except:
        return Response(ret_msg("Missing passphrase."),status=400)
    private_key="";address=""
    try:
        keys=keys.replace("'",'"')
        keys = json.loads(keys)
        private_key = Account.decrypt(keys, passphrase).hex()
        # address = web3.to_checksum_address(keys["address"])
        address=Account.from_key(private_key).address
    except Exception:
        return Response(ret_msg("Invalid credentials."),status=400)

    contract_new = web3.eth.contract(order.contract, abi=abifile, bytecode=bytefile)

    if order.address!=address:
        return Response(ret_msg("Invalid customer account."),status=400)

    if not contract_new.functions.is_paid().call():
        return Response(ret_msg("Transfer not complete."),status=400)
    if not contract_new.functions.in_transit().call():
        return Response(ret_msg("Delivery not complete."), status=400)

    transaction = contract_new.functions.deliver().build_transaction({
        "from": address,
        "gasPrice": 21000,
        "nonce": web3.eth.get_transaction_count(address)
    })

    signed_transaction = web3.eth.account.sign_transaction(transaction, private_key)
    transaction_hash = web3.eth.send_raw_transaction(signed_transaction.rawTransaction)
    receipt = web3.eth.wait_for_transaction_receipt(transaction_hash)

    order.status="COMPLETE"
    database.session.commit()

    return Response(status=200)

@application.route("/orders_to_deliver",methods=["GET"])
@jwt_required()
@roleCheck ( role = "courier" )
def orders_to_deliver():
    orders=Order.query.filter(Order.status=="CREATED")
    ret_list=[]
    for o in orders:
        ret_list.append({
            "id":o.id,
            "email":o.idU
        })
    return Response(json.dumps({"orders":ret_list}))

@application.route("/pick_up_order",methods=["POST"])
@jwt_required()
@roleCheck ( role = "courier" )
def pick_up_order():
    id = request.json
    try:
        id = id['id']
    except:
        return Response(ret_msg("Missing order id."), status=400)
    if not str(id).isnumeric() or int(id) <= 0:
        return Response(ret_msg("Invalid order id."), status=400)
    order = None
    try:
        order = Order.query.filter(Order.id == int(id)).all()
        if len(order)==0: return Response(ret_msg("Invalid order id."), status=400)
        order=order[0]
    except:
        return Response(ret_msg("Invalid order id."), status=400)
    if order.status != "CREATED":
        return Response(ret_msg("Invalid order id."), status=400)
    address=""
    try:
        address=request.json['address']
        if address=="": return Response(ret_msg("Missing address."),status=400)
    except:
        return Response(ret_msg("Missing address."),status=400)
    if not web3.is_address(address):
        return Response(ret_msg("Invalid address."),status=400)
    contract_new = web3.eth.contract(order.contract, abi=abifile, bytecode=bytefile)

    if not contract_new.functions.is_paid().call():
        return Response(ret_msg("Transfer not complete."), status=400)

    contract_new.functions.pick_up_order(address).transact({"from": admin_acc})

    order.status = "PENDING"
    database.session.commit()

    return Response(status=200)

@application.route("/pay",methods=["POST"])
@jwt_required()
@roleCheck ( role = "customer" )
def pay():
    id = request.json
    try:
        id = id['id']
    except:
        return Response(ret_msg("Missing order id."), status=400)
    if not str(id).isnumeric() or int(id) <= 0:
        return Response(ret_msg("Invalid order id."), status=400)
    order = None
    try:
        order = Order.query.filter(Order.id == int(id)).all()
        if len(order) == 0: return Response(ret_msg("Invalid order id."), status=400)
        order = order[0]
    except:
        return Response(ret_msg("Invalid order id."), status=400)

    keys = None;
    passphrase = ""
    try:
        keys = request.json["keys"]
        if keys=="": return Response(ret_msg("Missing keys."), status=400)
    except:
        return Response(ret_msg("Missing keys."), status=400)
    try:
        passphrase = request.json["passphrase"]
        if passphrase=="": return Response(ret_msg("Missing passphrase."), status=400)
    except:
        return Response(ret_msg("Missing passphrase."), status=400)
    private_key = "";
    address = ""
    try:
        keys = json.loads(keys.replace("'", '"'))
        private_key = Account.decrypt(keys, passphrase).hex()
        address = web3.to_checksum_address(keys["address"])
    except:
        return Response(ret_msg("Invalid credentials."), status=400)

    if web3.eth.get_balance(address) < order.price:
        return Response(ret_msg("Insufficient funds."),status=400)

    contract_new = web3.eth.contract(order.contract, abi=abifile, bytecode=bytefile)
    if contract_new.functions.is_paid().call():
        return Response(ret_msg("Transfer already complete."), status=400)

    transaction = contract_new.functions.pay().build_transaction({
        "from": address,
        "value": int(order.price),
        "gasPrice": 21000,
        "nonce": web3.eth.get_transaction_count(address)
    })

    signed_transaction = web3.eth.account.sign_transaction(transaction, private_key)
    transaction_hash = web3.eth.send_raw_transaction(signed_transaction.rawTransaction)
    receipt = web3.eth.wait_for_transaction_receipt(transaction_hash)

    return Response(status=200)

@application.route("/",methods=["GET"])
def index():
    return "Hello world"

if (__name__ == "__main__"):
    database.init_app(application)
    if (not database_exists(Configuration.SQLALCHEMY_DATABASE_URI)):
        create_database(Configuration.SQLALCHEMY_DATABASE_URI)
    application.run ( debug = True, host = "0.0.0.0")


import pandas as pd
import json
from collections import Counter
import nltk

class BigramFreq(object):
    """
    Calculates the frequency a word occurs within a given subreddit.
    WordFreq() deals with all time word freq, WordFreqByDay() gets top
    25 words used within a given day.
    """

    def __init__(self, comments, posts):
        # disable SettingWithCopyWarning
        # https://stackoverflow.com/a/20627316/4254021
        pd.options.mode.chained_assignment = None

        self.comments = comments
        self.posts = posts
        self.bigram_freq_df = None

    def get_bigram_freq(self, old_bigrams):
        """
        Creating a list of bigram frequencies from the gathered
        reddit comments and posts
        """
        # Copy of dataframes
        comments = self.comments.copy()
        posts = self.posts.copy()

        # Respective counters
        comment_counts = Counter()
        post_counts = Counter()

        # Loop through both dataframes, finding the most frequent bigrams
        for sent in comments["Text"]:
            words = nltk.word_tokenize(sent)
            comment_counts.update(nltk.bigrams(words))
        bc = pd.DataFrame(list(comment_counts.items()), columns=['bigram', 'n_comment'])

        for sent in posts["Text"]:
            words = nltk.word_tokenize(sent)
            post_counts.update(nltk.bigrams(words))
        bp = pd.DataFrame(list(post_counts.items()), columns=['bigram', 'n_post'])

        # Merge both dataframes and join both words to create bigram
        bigram = pd.merge(bc, bp, on='bigram', how='outer')
        bigram.fillna(0, inplace=True)
        bigram['n'] = bigram['n_comment'] + bigram['n_post']
        bigram["bigram"] = bigram["bigram"].apply(lambda x: ' '.join(x))

        self.bigram_freq_df = bigram
        # sort and limit to 500 bigrams
        bigram = bigram.sort_values('n', ascending=False).head(500)
        # If old bigram count in database, merge new with old and return
        # the updated counts
        if old_bigrams != None:
            old_bigrams = pd.DataFrame.from_records(data=old_bigrams)
            oldnew_merged = pd.concat([bigram,old_bigrams], sort=True)
            # updated counts
            bigram = oldnew_merged.groupby('bigram').sum().reset_index()

        bigram = bigram.to_json(orient='records', date_format=None)
        return json.loads(bigram)

    def get_bigram_freq_by_day(self, oldbigrams):
        # copy dataframes
        comments = self.comments.copy()
        posts = self.posts.copy()
        # Change date format from timestamp
        comments["Date"] = pd.to_datetime(comments['Date'], errors='coerce')
        posts["Date"] = pd.to_datetime(posts['Date'], errors='coerce')
        # Convert back to string date without hours
        comments["Date"] = comments["Date"].dt.strftime('%Y-%m-%d')
        posts["Date"] = posts["Date"].dt.strftime('%Y-%m-%d')
        #Merge dataframes
        merged = pd.concat([posts,comments], sort=True)
        # Group by date
        merged = merged.groupby('Date')
        bbd_list = []
        # loop through groups
        for name, group in merged:
            merged_counts = Counter()
            # Loop through text counting bigrams
            for sent in group["Text"]:
                words = nltk.word_tokenize(sent)
                merged_counts.update(nltk.bigrams(words))
            updated = []
            # join bigrams to one word
            for key, value in merged_counts.most_common(25):
                k = ' '.join(key)
                updated.append({
                    "bigram": k,
                    "n": value
                })
            # append grouped date and counted bigrams to list
            bbd_list.append([name, updated])

        # Create dataframe with counts
        bbd = pd.DataFrame(bbd_list, columns = ['Date','counts'])

        if oldbigrams != None:
            oldbigrams = pd.DataFrame.from_records(data=oldbigrams)
            oldnew_merged = pd.concat([bbd,oldbigrams], sort=True)

            flattened = []
            # loop through groups
            for date, group in oldnew_merged.groupby('Date'):
                for array in group["counts"]:
                    for obj in array:
                        flattened.append([date, obj["bigram"], obj["n"]])

            # Goal is to group similar dates and marge words together
            # keeping the 25 most popular for each date
            flattened = pd.DataFrame(flattened, columns = ['Date','bigram','n'])
            flattened = flattened.groupby(['Date', 'bigram'])['n'].sum().reset_index()

            flat = []
            for date, group in flattened.groupby('Date'):
                bigram = list(group["bigram"])
                n = list(group["n"])
                group_arr = []
                for i in range(len(n)-1):
                    if i > 25:
                        break
                    group_arr.append({
                        "bigram":bigram[i],
                        "n":n[i]
                    })
                flat.append([date, group_arr])
                group_arr = []

            bbd = pd.DataFrame(flat, columns = ['Date','counts'])

        bbd = bbd.to_json(orient='records', date_format=None)
        bbd = json.loads(bbd)
        return bbd


from __future__ import annotations

import time
from dataclasses import dataclass

from alpaca.utils import Visitor
from eisen.common.exceptionshandler import ExceptionsHandler
from eisen.validation.modulevisitor import ModuleVisitor
from eisen.validation.typechecker import TypeChecker
from eisen.validation.functionvisitor import FunctionVisitor
from eisen.validation.usagechecker import UsageChecker
from eisen.validation.declarationvisitor import DeclarationVisitor
from eisen.validation.finalizationvisitor import FinalizationVisitor, Finalization2
from eisen.validation.fnconverter import FnConverter
from eisen.validation.initalizer import Initializer
from eisen.validation.nilcheck import NilCheck
from eisen.validation.instancevisitor import InstanceVisitor
from eisen.validation.vectorvisitor import VectorVisitor
from eisen.trace.memoryvisitor import MemoryVisitor
from eisen.state.basestate import BaseState as State

# Notes:
# A module is a collection of structs/functions
# A context, properly, is a block with instances defined
# A context may have a parent as a module (as module level functions are available
# for use as instances)
# They are implemented the same.

# class used to debug
class Printast():
    def run(self, state: State):
        print(state.get_ast())
        return state

class Workflow():
    steps: list[Visitor] = [
        # initialize the .data attribute for all asts with empty NodeData instances
        Initializer,

        # create the module structure of the program.
        #   - the module of a node can be accessed by params.ast_get_mod()
        ModuleVisitor,

        # add proto types for struct/interfaces, which are the representation
        # of struct declaration without definition.
        DeclarationVisitor,

        # finalizes the proto types (which moves from declaration to definition)
        # TODO: for now, interfaces can only be implemented from the same module
        # in which they are defined.
        #
        # we must finalize interfaces first because structs depend on interfaces
        # as they implement interfaces
        FinalizationVisitor,
        Finalization2,

        # adds types for and constructs the functions. this also normalizes the
        # (def ...) and (create ...) asts so they have the same child structure,
        # which allows us to process them identically later in the
        # TypeClassFlowWrangler.
        FunctionVisitor,
        VectorVisitor,

        # this changes (ref ...) to (fn ...) if they refer to global functions
        FnConverter,

        # evaluate the flow of types through the program.
        #   - the type which is flowed through a node can be accessed by
        #     params.get_returned_type()
        TypeChecker,

        InstanceVisitor,

        # this handles restrictions based on let/var/val differences
        UsageChecker,

        NilCheck,
        MemoryVisitor,
        # MemCheck,
        # MoveVisitor
        # Printast,

        # note: def, create, fn, ref, ilet, :: need instances!!
    ]

    @staticmethod
    def _choose_steps(supplied_steps: list[Visitor]=None):
        if supplied_steps:
            return supplied_steps
        return Workflow.steps

    @staticmethod
    def _run_step_with_state(step: Visitor, state: State) -> State:
        return step().run(state)

    @staticmethod
    def execute(state: State, steps: list[Visitor]=None) -> tuple[bool, State]:
        for step in Workflow._choose_steps(steps):
            state = Workflow._run_step_with_state(step, state)
            ExceptionsHandler().apply(state)
            if Workflow.should_stop_execution(state):
                return False, state
        return True, state

    @staticmethod
    def _instrument_step_with_telemetry(step_times: list[int], step: Visitor) -> Visitor:
        class decorated_step():
            def run(self, state: State):
                start = time.perf_counter_ns()
                state = Workflow._run_step_with_state(step, state)
                end = time.perf_counter_ns()
                decorated_step.add_metric(start, end)
                return state

            @staticmethod
            def add_metric(start: int, end: int):
                step_times.append(PerformanceMetric(
                    step_name=step.__name__,
                    step_time=round((end-start)/1000000, 5)))

        return decorated_step

    @staticmethod
    def execute_with_benchmarks(state: State, steps: list[Visitor]=None) -> tuple[bool, State]:
        metrics: list[PerformanceMetric] = []
        steps_with_telemetry = [Workflow._instrument_step_with_telemetry(metrics, step)
            for step in Workflow._choose_steps(steps)]

        result, state = Workflow.execute(state, steps=steps_with_telemetry)
        PerformanceMetric.print_performance_metrics(metrics)
        return result, state

    @staticmethod
    def should_stop_execution(state: State) -> bool:
        return len(state.exceptions) > 0

@dataclass
class PerformanceMetric():
    step_name: str
    step_time: int

    @staticmethod
    def _print_performance_metric_line(metric: PerformanceMetric, block_size: int):
        print(" "*(block_size - len(metric.step_name)),
            metric.step_name, " ",
            metric.step_time)

    @staticmethod
    def print_performance_metrics(metrics: list[PerformanceMetric]):
        longest_name_size = max(len(metric.step_name) for metric in metrics)
        block_size = longest_name_size + 4
        for metric in metrics:
            PerformanceMetric._print_performance_metric_line(metric, block_size)

        print(" "*(block_size-len("Total")),
              "Total", " ",
              round(sum(metric.step_time for metric in metrics), 5))


import torch
import torch.nn as nn

class BaselineArch(nn.Module):
    def __init__(self, in_channels: int, channels: int, stride: int):
        super().__init__()
        self.out_channels = out_channels = channels
        self.relu = nn.ReLU(inplace=True)
        self.conv1 = nn.Conv2d(in_channels, out_channels, 3, padding=1,
                               stride=stride, bias=False)
        self.norm1 = nn.BatchNorm2d(out_channels)
        self.conv2 = nn.Conv2d(out_channels, out_channels, 3, padding=1,
                               stride=1, bias=False)
        self.norm2 = nn.BatchNorm2d(out_channels)

        if in_channels == out_channels and stride == 1:
            self.shortcut = nn.Sequential()
        else:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels, 1, padding=0,
                          stride=stride, bias=False),
                nn.BatchNorm2d(out_channels)
            )

    def forward(self, x):
        y = self.conv1(x)
        y = self.norm1(y)
        y = self.relu(y)
        y = self.conv2(y)
        y = self.norm2(y)
        y += self.shortcut(x)
        y = self.relu(y)
        return y

class BottleneckArch(nn.Module):
    def __init__(self, in_channels: int, channels: int, stride: int):
        super().__init__()
        self.out_channels = out_channels = channels * 4
        self.relu = nn.ReLU(inplace=True)
        self.conv1 = nn.Conv2d(in_channels, channels, 1, padding=0,
                               stride=stride, bias=False)
        self.norm1 = nn.BatchNorm2d(channels)
        self.conv2 = nn.Conv2d(channels, channels, 3, padding=1,
                               stride=1, bias=False)
        self.norm2 = nn.BatchNorm2d(channels)
        self.conv3 = nn.Conv2d(channels, out_channels, 1, padding=0,
                               stride=1, bias=False)
        self.norm3 = nn.BatchNorm2d(out_channels)

        if in_channels == out_channels and stride == 1:
            self.shortcut = nn.Sequential()
        else:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels, 1, padding=0,
                          stride=stride, bias=False),
                nn.BatchNorm2d(out_channels)
            )

    def forward(self, x):
        y = self.conv1(x)
        y = self.norm1(y)
        y = self.relu(y)
        y = self.conv2(y)
        y = self.norm2(y)
        y = self.relu(y)
        y = self.conv3(y)
        y = self.norm3(y)
        y += self.shortcut(x)
        y = self.relu(y)
        return y

class Layer(nn.Module):
    def __init__(self, Block, in_channels: int, channels: int, stride: int, count: int):
        super().__init__()
        blocks = []

        block = Block(in_channels, channels, stride)
        blocks.append(block)
        in_channels = block.out_channels

        for i in range(1, count):
            block = Block(in_channels, channels, 1)
            blocks.append(block)
            in_channels = block.out_channels

        self.blocks = nn.Sequential(*blocks)
        self.out_channels = in_channels

    def forward(self, x):
        return self.blocks(x)

class ResNet(nn.Module):
    def __init__(self, Block, counts: int, start_channels: int, classes: int):
        super().__init__()
        self.relu = nn.ReLU(inplace=True)
        self.conv0 = nn.Conv2d(3, start_channels, 7, padding=3,
                               stride=2, bias=False)
        self.norm0 = nn.BatchNorm2d(start_channels)
        self.pool0 = nn.MaxPool2d(3, stride=2, padding=1)
        self.layer1 = Layer(Block, start_channels,
                            start_channels, 1, counts[0])
        self.layer2 = Layer(Block, self.layer1.out_channels,
                            start_channels*2, 2, counts[1])
        self.layer3 = Layer(Block, self.layer2.out_channels,
                            start_channels*4, 2, counts[2])
        self.layer4 = Layer(Block, self.layer3.out_channels,
                            start_channels*8, 2, counts[3])
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.linear = nn.Linear(self.layer4.out_channels, classes)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode="fan_out",
                                        nonlinearity="relu")
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def forward(self, x):
        x = self.conv0(x)
        x = self.norm0(x)
        x = self.relu(x)
        x = self.pool0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.linear(x)
        return x

def resnet18(start_channels: int, classes: int):
    return ResNet(BaselineArch, [2, 2, 2, 2], start_channels, classes)

def resnet34(start_channels: int, classes: int):
    return ResNet(BaselineArch, [3, 4, 6, 3], start_channels, classes)

def resnet50(start_channels: int, classes: int):
    return ResNet(BottleneckArch, [3, 4, 6, 3], start_channels, classes)

def resnet101(start_channels: int, classes: int):
    return ResNet(BottleneckArch, [3, 4, 23, 3], start_channels, classes)

def resnet152(start_channels: int, classes: int):
    return ResNet(BottleneckArch, [3, 8, 36, 3], start_channels, classes)


import sys
sys.path.insert(0, "../../modules")

from posterior import *
from prior import *
from data import *
from pointsets import *
from gaussianprocesses import *
from data import *
np.random.seed(10)

toy_ip = ToyInverseProblem1d(variance = 1e-1)
true_param = toy_ip.locations[0,0]

posterior = Posterior(toy_ip, Prior.uniform)
posterior.compute_norm_const(100000)

num_plotpts = 250
locations = Mesh1d.construct(num_plotpts)

prior_density = posterior.prior_density(locations)
posterior_density = posterior.density(locations)

import matplotlib.pyplot as plt
plt.style.use("fivethirtyeight")
plt.plot(locations, posterior_density, label ="posterior")
plt.plot(locations, prior_density, label ="prior")
plt.vlines(toy_ip.locations, 0, posterior.density(toy_ip.locations), label = "True input value")
plt.legend()
plt.show()

print("\nPrior input/output:")
for i in range(num_plotpts):
    print("(", locations[i,0], ",", prior_density[i], ")")

print("\nPosterior input/output:")
for i in range(num_plotpts):
    print("(", locations[i,0], ",", posterior_density[i], ")")

print("\nTrue input/output:")
print("(", toy_ip.locations[0,0], ",", posterior.density(toy_ip.locations)[0], ")")


# vim: set tabstop=4 expandtab :
###############################################################################
#   Copyright (c) 2019-2021 ams AG
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
###############################################################################

# Authors:
# - Thomas Winkler, ams AG, thomas.winkler@ams.com

import binascii
import math
import struct
from enum import Enum
from typing import Union, Dict

from dottmi.dottexceptions import DottException
from dottmi.utils import log

ALIGN_DEFAULT = 4  # default alignment for memory allocation is 4 bytes

class TargetMemModel(Enum):
    """
    Enum for the different on-target memory allocation models supported by DOTT.
    """
    NOALLOC  = 0x0  # Don't use any of the built-in on-target memory allocation models.
    TESTHOOK = 0x1  # DOTT test-hook based memory allocation
    PRESTACK = 0x2  # memory allocation prior to stack memory ('stack stealing')

    @classmethod
    def get_keys(cls):
        return list(map(lambda c: c.name, cls))

# -------------------------------------------------------------------------------------------------
class TypedPtr:
    def __init__(self, target: 'Target', addr: int, var_type: str = None) -> None:
        """
        Constructor.

        Args:
            target: Target instance.
            addr: Address represented by the pointer.
            var_type: Type of the variable the pointer is pointing to.
        """
        self._target: 'Target' = target
        self._addr: int = addr
        self._var_type: str = var_type
        if self._var_type is None:
            self._var_type = 'void'

    @property
    def type(self) -> str:
        """
        This function returns the they of the variable the pointer is pointing to.

        Returns:
            Returns the type of the variable the pointer is pointing to.
        """
        return self._var_type

    @property
    def addr(self) -> int:
        """
        This function returns the raw address.

        Returns:
            Returns raw address as integer.
        """
        return self._addr

    @property
    def val(self) -> Union[int, float, bool, str, None]:
        """
        This function dereferences the pointer and returns the value. This might not be possible for all data types
        (e.g., composite types).

        Returns:
            Dereferences the pointer and returns the value.
        """
        return self._target.eval(f'*({self.__str__()})')

    def __getitem__(self, key) -> Union[int, float, bool, str, None]:
        """
        Allows access to items pointed to by TypedPtr in index notation. Example:
        p = dott().target.mem.alloc_type('uint32_t', cnt=16)
        p[8] = 0xdeadbeef
        print('0x%x' % p[8])

        Important: The implementation does not perform array index checking.

        Args:
            key: Index used for accessing the array data pointed to by the TypedPtr instance.

        Returns: The value at index key.
        """
        return self._target.eval(f'{self.__str__()}[{key}]')

    def __setitem__(self, key, value) -> None:
        """
        Allows access to items pointed to by TypedPtr in index notation. Example:
        p = dott().target.mem.alloc_type('uint32_t', cnt=16)
        p[8] = 0xdeadbeef
        print('0x%x' % p[8])

        Important: The implementation does not perform array index checking.

        Args:
            key: Index used for accessing the array data pointed to by the TypedPtr instance.

        Returns: The value at index key.
        """
        self._target.eval(f'{self.__str__()}[{key}] = {value}')

    def __str__(self) -> str:
        """
        This function returns a string containing address in hex (pre-fixed with 0x) together with the type of
        the pointer as a prefix. E.g., (uint32_t*)0x20000000

        Returns:
            Returns string representation of the pointer in hex format together with type prefix.
        """
        return f'(({self._var_type}*)0x{self._addr:x})'

# -------------------------------------------------------------------------------------------------
class TargetMem(object):
    def __init__(self, target: 'Target', target_mem_start_addr: int, target_mem_num_bytes: int, zero_mem: bool = True):
        """
        Constructor.

        Args:
            target: The target instance.
            target_mem_start_addr: The memory address where DOTT's on-target scratchpad memory starts.
            target_mem_num_bytes: The size in bytes of DOTT's on-target scratchpad memory.
            zero_mem: Zero out the on-target scratchpad memory when calling reset (default: True).
        """
        self._target: 'Target' = target
        self._sz_types: Dict = {}  # dict with target sizes (cache used by sizeof)
        self._heap_next_free_addr: int = target_mem_start_addr
        self._target_mem_base_addr: int = target_mem_start_addr
        self._target_mem_num_bytes: int = target_mem_num_bytes
        self._zero_mem = zero_mem
        self.reset()

    def _bytes_needed(self, n):
        if n == 0:
            return 1
        return int(math.log(n, 256)) + 1

    def _write_raw(self, dst_addr: Union[int, str, TypedPtr], values: bytes) -> None:
        content = binascii.hexlify(struct.pack('<%dB' % len(values), *values)).decode('utf8')
        self._target.exec(f'-data-write-memory-bytes {dst_addr} "{content}"')

    def sizeof(self, target_type: str) -> int:
        """
        This function returns the size in bytes of the given target data type.

        Args:
            target_type: Name of the target data type.

        Returns:
            The size in bytes of the data type on target.
        """
        if target_type not in self._sz_types.keys():
            try:
                sz = int(self._target.eval('sizeof(%s)' % target_type))
                self._sz_types[target_type] = sz
            except:
                log.error('Unable to determine size for target type %s' % target_type)

        return self._sz_types[target_type]

    def write(self, dst_addr: Union[int, str, TypedPtr], val: Union[int, bytes, str], cnt: int = 1) -> None:
        """
        This function writes the provided data to target memory at destination address. If cnt is other than one,
        the provided data is replicated cnt times.

        Args:
            dst_addr: The target's destination memory address to write to.
            val: Content to be written to the target.
            cnt: The number of times val shall be repeated when writing to the target.
        """
        if isinstance(val, int):
            # note: int.to_bytes raises exception if type_sz != val size
            bval = val.to_bytes(self._bytes_needed(val), byteorder=self._target.byte_order)
            self._write_raw(dst_addr, bval * cnt)
        elif isinstance(val, bytes):
            self._write_raw(dst_addr, val * cnt)
        elif isinstance(val, str):
            bval = bytes(val, encoding='ascii')
            self._write_raw(dst_addr, bval * cnt)
        else:
            raise ValueError('Only int, bytes or str (ascii) are supported as val types')

    def read(self, src_addr: Union[int, str, TypedPtr], num_bytes: int) -> bytes:
        """
        This function reads the requested number of bytes from the specified source address.

        Args:
            src_addr: The target's source memory address to read from.
            num_bytes: The number of bytes to read from target memory.

        Returns:
            Returns the bytes read from the target.
        """
        num_remaining: int = num_bytes
        if isinstance(src_addr, int):
            addr_to_read = src_addr
        elif isinstance(src_addr, str):
            addr_to_read = int(src_addr, 10)
        elif isinstance(src_addr, TypedPtr):
            addr_to_read = src_addr.addr
        else:
            raise ValueError('Illegal type for src_addr')
        content = ''

        while num_remaining > 0:
            bytes_to_read = 1024
            if num_remaining < 1024:
                bytes_to_read = num_remaining

            data = self._target.exec(f'-data-read-memory-bytes -o 0 {addr_to_read} {bytes_to_read}')
            buf = data['payload']['memory'][0]['contents']
            buf_len = len(buf) / 2
            content += buf

            num_remaining -= buf_len
            addr_to_read += buf_len

        return binascii.unhexlify(content)

    def reset(self) -> None:
        """
        This function resets the on-target memory. It sets the next_element pointer back to the first element and it
        zeros out the on-target memory region used for memory allocation.
        """
        self._heap_next_free_addr = self._target_mem_base_addr
        if self._zero_mem:
            # note: bytes(sz) creates zero-filled bytes object of length sz
            self.write(self._target_mem_base_addr, bytes(self._target_mem_num_bytes))

    def alloc(self, req_num_bytes: int, var_name: str = None, align: int = ALIGN_DEFAULT) -> TypedPtr:
        """
        This function allocates the requested number of bytes on the target device. The allocated memory is guaranteed
        to be word (32bit aligned). Optionally, a GDB convenience variable with the provided variable name is created
        which can be optionally used to reference the allocated memory in GDB commands.

        Args:
            req_num_bytes: Requested number of bytes.
            var_name: Optional GDB variable name used for the newly allocated memory.
            align: Alignment for allocated memory. Default is 4 bytes.

        Returns:
            Returns the address of the allocated on-target memory as a TypedPtr.
        """
        align_delta: int = 0
        if self._heap_next_free_addr % align != 0:
            align_delta = align - (self._heap_next_free_addr % align)

        # number of bytes still available on target for memory allocation
        avail_bytes: int = (self._target_mem_base_addr + self._target_mem_num_bytes) - (self._heap_next_free_addr + align_delta)

        assert(req_num_bytes <= avail_bytes), 'unable to allocate %d bytes of on-target memory' % req_num_bytes

        addr: int = self._heap_next_free_addr + align_delta
        self._heap_next_free_addr += align_delta + req_num_bytes

        if var_name is not None:
            self._target.cli_exec(f'set var {var_name} = (void*){addr}')

        return TypedPtr(self._target, addr, 'void')

    def alloc_type(self, var_type: str, val: Union[int, bytes, str] = None, cnt: int = 1, var_name: str = None, align: int = ALIGN_DEFAULT) -> TypedPtr:
        """
        This function allocates on-target memory for cnt number of variables of the specified type. The start of the
        allocated memory is guaranteed to be word (32bit aligned). If the memory is allocated for, e.g., for an array
        holding elements of a composite type with size not being a multiple of words, this function does not guarantee
        word alignment from the second array element onwards. In other word, this function simply allocates memory
        of size = sizeof(var_type) * cnt.
        Optionally, the allocate memory can be initialized with the provided value.
        Optionally, a GDB convenience variable with the provided variable name is created which can be used to reference
        the allocated memory in GDB commands.

        Args:
            var_type: Type of variable to be allocated on-target.
            val: Value for the newly assigned variable. Note that if cnt > 1 this value is set for all the elements.
            cnt: Number of elements to be allocated.
            var_name: Optional GDB variable name used for the newly allocated memory.
            align: Alignment for allocated memory. Default is 4 bytes.

        Returns:
            Returns the address of the allocated on-target memory as a TypedPtr.
        """
        type_sz: int = self.sizeof(var_type)
        p_var: TypedPtr = self.alloc(type_sz * cnt, var_name, align)
        if val is not None:
            self.write(p_var, val, cnt)

        # optionally create a gdb convenience variable and cast to concrete type as specified by var_type
        if var_name is not None:
            self._target.cli_exec('set var %s = (%s*)%s' % (var_name, var_type, var_name))
        return TypedPtr(self._target, p_var.addr, var_type)

    def get_num_alloc_bytes(self) -> int:
        """
        Returns: Number of bytes allocated since last call to TargetMem::reset.
        """
        return self._heap_next_free_addr - self._target_mem_base_addr

# -------------------------------------------------------------------------------------------------
class TargetMemTestHook(TargetMem):
    """
    This class implements a variation of the TargetMem class which is intended to be used if the DOTT_test_hook is
    used for on-target memory allocation. In addition to setting the correct memory address (in stack from of the
    test hook) this class also checks that allocation is only performed if the target is halted in the context of the
    test hook.
    """
    def __init__(self, target: 'Target'):
        start_addr = target.eval('dbg_mem_u32')
        num_bytes = int(target.eval('dbg_mem_u32_sz'))
        super().__init__(target, start_addr, num_bytes)

    def alloc(self, req_num_bytes: int, var_name: str = None, align: int = ALIGN_DEFAULT):
        try:
            self._target.eval('dbg_mem_u32')  # raises exception if symbol is not accessible in current context
        except:
            raise DottException('Test-hook based memory allocation is only possible '
                                'if program is halted in the test hook!') from None
        return super().alloc(req_num_bytes, var_name, align)

    def alloc_type(self, var_type: str, val: Union[int, bytes, str] = None, cnt: int = 1, var_name: str = None, align: int = ALIGN_DEFAULT):
        try:
            self._target.eval('dbg_mem_u32')  # raises exception if symbol is not accessible in current context
        except Exception:
           raise DottException('Test-hook based memory allocation is only possible '
                               'if program is halted in the test hook!') from None

        return super().alloc_type(var_type, val, cnt, var_name, align)

# -------------------------------------------------------------------------------------------------
class TargetMemNoAlloc(TargetMem):
    """
    This class implements a variation of the TargetMem class which actually does not allow on-target memory allocation.
    Target memory bulk read and write still are supported.
    """
    def __init__(self, target: 'Target'):
        start_addr = 0xffffffff
        num_bytes = 0
        super().__init__(target, start_addr, num_bytes)

    def alloc(self, req_num_bytes: int, var_name: str = None, align: int = ALIGN_DEFAULT) -> TypedPtr:
        raise Exception('On-target memory allocation not supported in by this memory model!')

    def alloc_type(self, var_type: str, val: Union[int, bytes, str] = None, cnt: int = 1, var_name: str = None, align: int = ALIGN_DEFAULT) -> TypedPtr:
        raise Exception('On-target memory allocation not supported in by this memory model!')

    def reset(self) -> None:
        return

# -------------------------------------------------------------------------------------------------
class TargetMemScoped(object):
    """
    This class implements a form of on-target, on-stack memory allocation with a limited scope. In contrast to the
    test hook approach, this allocation technique can be used at every location the program can be halted. At this
    halt location, it enables users to grab a chunk of memory which is located on the current stack as indicated by
    SP. If this is MSP or PSP is not checked.
    It returns a TargetMem instance which can be used to allocate memory blocks within this on-target memory chunk.
    These allocated memory regions can then be populated and used as arguments in on-target function calls using
    'eval(...)'. At the end of the 'with ... as' statement, the chunk of on target memory is 'returned' to the target
    by re-adjusting the stack pointer accordingly.

    Example:
        # - grab a chunk of 128 bytes of on-target (stack) memory
        # - within this chunk, allocate an array with 5 uint16_t variables
        # - fill the array, use it as function call parameter
        # - note: outside the scope of the 'with ... as', the m.alloc functions are unavailable as the chunk of
        #   on-target stack is already returned to the target system.

        with TargetMemScoped(dott().target, 128) as m:
            elements = [0, 1, 2, 65535, 99]
            addr = m.alloc_type('uint16_t', cnt=len(elements), val=0x0)
            for i in range(len(elements)):
                dott().target.eval(f'{addr}[{i}] = {elements[i]}')
            res = dott().target.eval(f'example_SumElements({addr}, {len(elements)})')
            assert (sum(elements) == res), f'expected: {sum(elements)}, is: {res}'
    """
    def __init__(self, target: 'Target', num_bytes: int, suppress_warnings: bool = False):
        """
        Constructor.

        Args:
            target: The target instance on which the memory shall be allocated.
            num_bytes: The number of bytes the chunk of memory shall be able to hold. Note: This value is internally
                     increased to the next value where num_bytes % 8 == 0 to ensure double-world alignment of the stack.
            suppress_warnings: Suppress warnings which would be issued if the SP and PC are not having the expected
                     values when leaving the 'with ... as' block.
        """
        self._target: 'Target' = target
        self._pc_init: int = 0x0  # the program counter upon entering the 'with' block
        self._sp_init: int = 0x0  # the stack pointer upon entering the 'with' block
        self._sp_init_dec: int = 0x0  # the decremented _sp_init
        self._mem: TargetMem = None
        # ensure that the requested number of bytes is a multiple of 8 (double-word alignment of stack; cp.
        # "Procedure Call Standard for the Arm® Architecture").
        if num_bytes % 8 != 0:
            num_bytes += 8 - (num_bytes % 8)
        self._num_bytes: int = num_bytes  # the size in bytes of the on-target memory chunk
        self._suppress_warnings: bool = suppress_warnings

    def __enter__(self) -> TargetMem:
        if self._target.is_running():
            raise DottException('Target must be halted when initializing scoped on-target memory.')

        # save current stack pointer and program counter
        self._sp_init = self._target.eval('$sp')
        self._pc_init = self._target.eval('$pc')

        # (1) set decremented SP equal to current SP
        self._sp_init_dec = self._sp_init

        # (2) check if SP is double-word aligned. if not, do so.
        if self._sp_init_dec % 8 != 0:
            # Arm 'Procedure Call Standard for the Arm Architecture' requires double-word alignment on
            # 'public interface' (i.e., when calling functions).
            log.warn('Current SP is not double-word aligned! Correcting alignment for allocated memory.')
            self._sp_init_dec += 8 - (self._sp_init_dec % 8)

        # (3) decrement SP
        # Note: Stack is "full-descending" (see 'Procedure Call Standard for the Arm Architecture').
        #       That means the SP points to the last used (full)  entry and the stack grows down (descends).
        self._sp_init_dec -= self._num_bytes  # decrement (descend) SP

        # Finally, adjust the SP to reserve the requested chunk of the stack. Create and return a memory manager for it.
        self._target.eval(f'$sp = {self._sp_init_dec}')
        self._mem = TargetMem(self._target, self._sp_init_dec, self._num_bytes)
        return self._mem

    def __reset_sp(self):
        pc_curr = self._target.eval('$pc')
        sp_curr = self._target.eval('$sp')

        sp_pc_ok: bool = True

        # check if SP and PC are as expected
        if sp_curr != self._sp_init_dec:
            sp_pc_ok = False
            if not self._suppress_warnings:
                log.warn(f'Stack pointer is not as expected (expected: 0x{self._sp_init_dec:x}, act: 0x{sp_curr:x}). '
                         f'You should not alter execution flow within "with" block os scoped memory by using DOTT '
                         f'functions such as "cont", "ret", "step", and "step_inst" etc.!')
        if pc_curr != self._pc_init:
            sp_pc_ok = False
            if not self._suppress_warnings:
                log.warn(f'Program counter is not as expected (expected: 0x{self._pc_init:x}, actual: 0x{pc_curr:x}). '
                         f'You should not alter execution flow within "with" block os scoped memory by using DOTT '
                         f'functions such as "cont", "ret", "step", and "step_inst" etc.!')

        if sp_pc_ok:
            self._target.eval(f'$sp = {self._sp_init}')
        else:
            if not self._suppress_warnings:
                log.warn('Not undoing stack memory allocation at end of "with" statement!')

    @staticmethod
    def __func_unavailable(*args) -> None:
        log.warn("The functions 'alloc', 'alloc_type', and 'reset' are no longer available after the 'with ... as' "
                 "block of TargetMemScoped!")

    def __exit__(self, exc_type, exc_val, exc_tb):
        # Note: We are not checking if there was an exception. We are always undoing (rolling back)
        # the on-stack memory allocation.

        if self._target.is_running():
            raise DottException('Target must be halted when leaving "with" block of scoped on-target memory.')

        # reset the internal state of the TargetMem instance with respect to the memory it managed.
        self._mem.reset()
        # reset the SP to the state it had before the 'with' block
        self.__reset_sp()
        # make alloc/reset functions of the TargetMem instance unusable after the 'with' block.
        self._mem.alloc = self.__func_unavailable
        self._mem.alloc_type = self.__func_unavailable
        self._mem.reset = self.__func_unavailable


from slam import process
from display import Display
from pointmap import PointMap
import cv2
import open3d as o3d
import depthai as dai

# Start defining a pipeline
pipeline = dai.Pipeline()

# Define a source - color camera
camRgb = pipeline.createColorCamera()
camRgb.setBoardSocket(dai.CameraBoardSocket.RGB)
# camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)  		#1080
# camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_12_MP)		#3040
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_4_K)			#2160
camRgb.setVideoSize(1000, 1000) ## CUSTOMIZE SIZE
camRgb.setInterleaved(False)
# camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)

# Create output
xoutRgb = pipeline.createXLinkOut()
xoutRgb.setStreamName("rgb")
camRgb.preview.link(xoutRgb.input)
xoutRgb1 = pipeline.createXLinkOut()
xoutRgb1.setStreamName("video")
xoutRgb1.input.setBlocking(False)
xoutRgb1.input.setQueueSize(1)
camRgb.video.link(xoutRgb1.input)

pmap = PointMap()
display = Display()

pcd = o3d.geometry.PointCloud()
visualizer = o3d.visualization.Visualizer()
visualizer.create_window(window_name="3D plot", width=960, height=540)

# Connect to the device
with dai.Device() as device:
	# Print out available cameras
	print('Connected cameras: ', device.getConnectedCameras())
	# Start pipeline
	device.startPipeline(pipeline)

	# Output queue will be used to get the rgb frames from the output defined above
	# qRgb = device.getOutputQueue(name="rgb", maxSize=4, blocking=False)
	qRgb = device.getOutputQueue(name="rgb", maxSize=30, blocking=False)
	out = device.getOutputQueue(name="video", maxSize=1, blocking=False)

	# while cap.isOpened():
	while True:
		frame = qRgb.get()
		# frame = frame.getCvFrame()
		output = out.get()
		frame = output.getCvFrame()

		img, tripoints, kpts, matches = process(frame)
		xyz = pmap.collect_points(tripoints)

		if kpts is not None or matches is not None:
			display.display_points2d(frame, kpts, matches)
		else:
			pass
		display.display_vid(frame)

		if xyz is not None:
			display.display_points3d(xyz, pcd, visualizer)
		else:
			pass
		if cv2.waitKey(1) == 27:
			break

	cv2.destroyAllWindows()


from keras.models import load_model
import os
from scipy import ndimage, misc
import re
from numpy.random import shuffle
from numpy import array
from matplotlib import pyplot as plt

images = []
for root, dirnames, filenames in os.walk("./flowers_to_predict"):
    for filename in filenames:
        if re.search("\.(jpg|jpeg|png|bmp|tiff|gif)$", filename):
            filepath = os.path.join(root, filename)
            image = ndimage.imread(filepath, mode="RGB")
            images.append(misc.imresize(image, (32, 32)))

shuffle(images)
images = array(images)

print ('loading model')
model = load_model('fine_tune-ep249-loss0.275-BEST_MODEL.h5')
print ('finished loading')

predictions = model.predict(images)

for i in range(len(images)):
    plt.imshow(images[i])
    plt.pause(0.05)
    print('{0} with prediction of {1}'.format(('Flower' if predictions[i] > 0.8 else 'Not flower'), predictions[i]))