#!/usr/bin/env python
"""
Module with channel estimation implementations based on the reference signals in
this package.
"""

from typing import Union

import numpy as np

from .dmrs import DmrsUeSequence
from .srs import SrsUeSequence, UeSequence

class CazacBasedChannelEstimator:
    """
    Estimated the (uplink) channel based on CAZAC (Constant Amplitude Zero
    AutoCorrelation) reference sequences sent by one user (either SRS or
    DMRS).

    The estimation is performed according to the paper [Bertrand2011]_,
    where the received signal in the FREQUENCY DOMAIN is used by the
    estimator.

    Note that for SRS sequences usually a comb pattern is employed such
    that only half of the subcarriers is used to send pilot
    symbols. Therefore, an FFT in the during the estimation will
    effectively interpolate for the other subcarriers. This is controlled
    by the `size_multiplier` argument (default is 2 to accommodate comb
    pattern). If all subcarriers are used to send pilot symbols then set
    `size_multiplier` to 1.

    Parameters
    ----------
    ue_ref_seq : SrsUeSequence | DmrsUeSequence | np.ndarray
        The reference signal sequence.
    size_multiplier : int, optional
        Multiplication factor for the FFT to get the actual channel size.
        When using the comb pattern for SRS this should be 2 (default value),
        but for DMRS, which does not employ the comb pattern, this should be
        set to 1.

    Notes
    -----

    .. [Bertrand2011] Bertrand, Pierre, "Channel Gain Estimation from
       Sounding Reference Signal in LTE," Conference: Proceedings of the
       73rd IEEE Vehicular Technology Conference.
    """
    def __init__(self,
                 ue_ref_seq: Union[SrsUeSequence, DmrsUeSequence, np.ndarray],
                 size_multiplier: int = 2) -> None:
        # If ue_ref_seq is not an instance of UeSequence (or a subclass)
        # assume it is a numpy array.
        if isinstance(ue_ref_seq, UeSequence):
            self._normalized_ref_seq = ue_ref_seq.normalized
            ue_ref_seq = ue_ref_seq.seq_array()
        else:
            self._normalized_ref_seq = False

        self._ue_ref_sequence = ue_ref_seq
        self._size_multiplier = size_multiplier

    @property
    def ue_ref_seq(self) -> np.ndarray:
        """Get the sequence of the UE."""
        return self._ue_ref_sequence

    def estimate_channel_freq_domain(self, received_signal: np.ndarray,
                                     num_taps_to_keep: int) -> np.ndarray:
        """
        Estimate the channel based on the received signal.

        Parameters
        ----------
        received_signal : np.ndarray
            The received reference signal after being transmitted through the
            channel (in the frequency domain). If this is a 2D numpy array
            the first dimensions is assumed to be "receive antennas" while
            the second dimension are the received sequence elements. The
            number of elements in the received signal (per antenna) is
            equal to the channel size (number of subcarriers) divided by
            `size_multiplier`.
        num_taps_to_keep : int
            Number of taps (in delay domain) to keep. All taps from 0 to
            `num_taps_to_keep`-1 will be kept and all other taps will be
            zeroed before applying the FFT to get the channel response in
            the frequency domain.

        Returns
        -------
        freq_response : np.ndarray
            The channel frequency response. Note that for SRS sequences
            this will have twice as many elements as the sent SRS signal,
            since the SRS signal is sent every other subcarrier.
        """
        # Reference signal sequence
        r = self.ue_ref_seq

        if received_signal.ndim == 1:
            # First we multiply (element-wise) the received signal by the
            # conjugate of the reference signal sequence
            y = np.fft.ifft(np.conj(r) * received_signal, r.size)

            # The channel impulse response consists of the first
            # `num_taps_to_keep` elements in `y`.
            tilde_h = y[0:num_taps_to_keep + 1]
        elif received_signal.ndim == 2:
            # Case with multiple receive antennas
            y = np.fft.ifft(
                np.conj(r)[np.newaxis, :] * received_signal, r.size)

            # The channel impulse response consists of the first
            # `num_taps_to_keep` elements in `y`.
            tilde_h = y[:, 0:num_taps_to_keep + 1]
        else:
            raise ValueError(  # pragma: no cover
                "received_signal must have either one dimension ("
                "one receive antenna) or two dimensions (first "
                "dimension being the receive antenna dimension).")

        # Now we can apply the FFT to get the frequency response.
        # The number of subcarriers is twice the number of elements in the
        # SRS sequence due to the comb pattern
        Nsc = r.size
        tilde_H = np.fft.fft(tilde_h, self._size_multiplier * Nsc)

        if self._normalized_ref_seq is True:
            tilde_H *= Nsc

        return tilde_H

class CazacBasedWithOCCChannelEstimator(CazacBasedChannelEstimator):
    """
    Estimated the (uplink) channel based on CAZAC (Constant Amplitude Zero
    AutoCorrelation) reference sequences sent by one user including the
    Orthogonal Cover Code (OCC).

    With OCC the user will send reference signal in multiple time slots, in
    each slot multiplied with the respective OCC sequence element.

    Parameters
    ----------
    ue_ref_seq : DmrsUeSequence
        The reference signal sequence.
    """
    def __init__(self, ue_ref_seq: DmrsUeSequence) -> None:
        cover_code = ue_ref_seq.cover_code
        ue_ref_seq_array = ue_ref_seq.seq_array()
        reference_seq = ue_ref_seq_array[0] * cover_code[0]

        super().__init__(reference_seq, size_multiplier=1)

        self._cover_code = cover_code
        self._normalized_ref_seq = ue_ref_seq.normalized

    @property
    def cover_code(self) -> np.ndarray:
        """Get the cover code of the UE."""
        return self._cover_code

    def estimate_channel_freq_domain(
            self,
            received_signal: np.ndarray,
            num_taps_to_keep: int,
            extra_dimension: bool = True) -> np.ndarray:
        """
        Estimate the channel based on the received signal with cover codes.

        Parameters
        ----------
        received_signal : np.ndarray
            The received reference signal after being transmitted through
            the channel (in the frequency domain).

            Dimension: Depend if there are multiple receive antennas and if
            `extra_dimension` is True or False. Let :math:`Nr` be the
            number of receive antennas, :math:`Ne` be the number of reference
            signal elements (reference signal size without cover code) and
            :math:`Nc` be the cover code size. The dimension of
            `received_signal` must match the table below.

            =================  =======================  ======================
                   /            extra_dimension: True   extra_dimension: False
            =================  =======================  ======================
            Single Antenna      Nc x Ne           (2D)   Ne * Nc          (1D)
            Multiple Antennas   Nr x Nc x Ne      (3D)   Nr x (Ne * Nc)   (2D)
            =================  =======================  ======================

        num_taps_to_keep : int
            Number of taps (in delay domain) to keep. All taps from 0 to
            `num_taps_to_keep`-1 will be kept and all other taps will be
            zeroed before applying the FFT to get the channel response in
            the frequency domain.
        extra_dimension : bool
            If True then the should be an extra dimension in
            `received_signal` corresponding to the cover code dimension. If
            False then the cover code is included in the dimension of the
            reference signal elements.

        Returns
        -------
        freq_response : np.ndarray
            The channel frequency response.
        """
        # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
        # xxxxx Add the extra dimension if it does not exist xxxxxxxxxxxxxx
        # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
        # Create a view for the received signals. If extra_dimension is
        # false we will reshape this view to add a dimension for the cover
        # code
        r = received_signal.view()
        if extra_dimension is False:
            # Let's reorganize the received signal so that we have the
            # extra dimension
            if received_signal.ndim == 1:
                # Case with a single antenna. Cover code dimension will be
                # the first dimension.
                r.shape = (self.cover_code.size, -1)

            elif received_signal.ndim == 2:
                # Case with multiple antennas. Cover code dimension will be
                # the second dimension.
                num_antennas = r.shape[0]
                r.shape = (num_antennas, self.cover_code.size, -1)
            else:
                raise RuntimeError(
                    'Invalid dimension for received_signal: {0}'.format(
                        r.ndim))

        # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
        # xxxxxxxxxxxxxxx Average over the cover code dimension xxxxxxxxxxx
        # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
        # Now we can consider the case with the extra cover_code dimension
        if r.ndim == 2:
            # Apply the cover code
            r_mean = np.mean(r * self.cover_code[:, np.newaxis], axis=0)
        elif r.ndim == 3:
            r_mean = np.mean(r * self.cover_code[np.newaxis, :, np.newaxis],
                             axis=1)

        else:
            raise RuntimeError(
                'Invalid dimension for received_signal: {0}'.format(r.ndim))

        # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
        # xxxxxxxxxxxxxxx Perform the estimation xxxxxxxxxxxxxxxxxxxxxxxxxx
        # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
        # Call the estimate_channel_freq_domain from the base class
        return super().estimate_channel_freq_domain(r_mean, num_taps_to_keep)


# a, b = 0, 1
# for i in range(1, 10):
#     print(a)
#     a, b = b, a + b

# x = [1, [1, ['a', 'b']]]
# print(x)

# def search_list(list_of_tuples, value):
#     for comb in list_of_tuples:

#         for x in comb:
#             if x == value:
#                 print(x)
#                 return comb
#     else:
#         return 0

# prices = [('AAPL', 96.43), ('IONS', 39.28), ('GS', 159.53)]
# ticker = 'GS'
# print(search_list(prices, ticker))
import datetime
date = '01-Apr-03'
date_object = datetime.datetime.strptime(date, '%d-%b-%y')
print((date_object).date())


import os
from tkinter.filedialog import askdirectory
import pygame
from tkinter import *

root = Tk()
root.minsize(400, 400)

listOfSongs = []
index = 0
labelstring = StringVar()

def playsong(event):
    pygame.mixer.music.play()

def stopsong(event):
    pygame.mixer.music.stop()

def nextsong(event):
    global index

    if index == len(listOfSongs) - 1:
        return
    index += 1
    pygame.mixer.music.load(listOfSongs[index])
    pygame.mixer.music.play()
    updatelable()

def prevsong(event):
    global index

    if index == 0:
        return
    index -= 1
    pygame.mixer.music.load(listOfSongs[index])
    pygame.mixer.music.play()
    updatelable()

def updatelable():
    global index
    labelstring.set(listOfSongs[index])

def directorychooser():

    directory = askdirectory()
    os.chdir(directory)

    for files in os.listdir(directory):
        if files.endswith(".mp3"):
            listOfSongs.append(files)

    pygame.mixer.init()
    labelstring.set(listOfSongs[index])
    pygame.mixer.music.load(listOfSongs[index])
    pygame.mixer.music.play()

directorychooser()

programlabel = Label(root, text='Media Player')
programlabel.pack()

listbox = Listbox(root)
listbox.pack()

songlable = Label(textvariable=labelstring, width=20)
songlable.pack()

i = 0
for song in listOfSongs:
    listbox.insert(i, song)
    i += 1

# Buttons Section
nextbutton = Button(root, text='Next Song')
previousbutton = Button(root, text='Previous Song')
playbutton = Button(root, text='Play')
stopbutton = Button(root, text='Stop')

playbutton.pack()
stopbutton.pack()
nextbutton.pack()
previousbutton.pack()

playbutton.bind("<Button-1>", playsong)
stopbutton.bind("<Button-1>", stopsong)
previousbutton.bind("<Button-1>", prevsong)
nextbutton.bind("<Button-1>", nextsong)

root.mainloop()


#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 26 17:12:48 2019

@author: ben
"""
import os
os.environ["MKL_NUM_THREADS"]="1"
os.environ["OPENBLAS_NUM_THREADS"]="1"

import numpy as np

from LSsurf.smooth_xytb_fit import smooth_xytb_fit
#from LSsurf.assign_firn_correction import assign_firn_correction
from surfaceChange.reread_data_from_fits import reread_data_from_fits
import pointCollection as pc
from CS2_fit.read_CS2_data import read_CS2_data
from pyTMD import compute_tide_corrections

import h5py
import sys
import glob
import re

def get_SRS_proj4(hemisphere):
    if hemisphere==1:
        return '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
    else:
        return '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'

def save_fit_to_file(S,  filename, sensor_dict=None, dzdt_lags=None):
    if os.path.isfile(filename):
        os.remove(filename)
    with h5py.File(filename,'w') as h5f:
        h5f.create_group('/data')
        for key in S['data'].fields:
            h5f.create_dataset('/data/'+key, data=getattr(S['data'], key))
        h5f.create_group('/meta')
        h5f.create_group('/meta/timing')
        for key in S['timing']:
            h5f['/meta/timing/'].attrs[key]=S['timing'][key]
        h5f.create_group('/RMS')
        for key in S['RMS']:
           h5f.create_dataset('/RMS/'+key, data=S['RMS'][key])
        h5f.create_group('E_RMS')
        for key in S['E_RMS']:
            h5f.create_dataset('E_RMS/'+key, data=S['E_RMS'][key])
        for key in S['m']['bias']:
            h5f.create_dataset('/bias/'+key, data=S['m']['bias'][key])
        if 'slope_bias' in S['m']:
            sensors=np.array(list(S['m']['slope_bias'].keys()))
            h5f.create_dataset('/slope_bias/sensors', data=sensors)
            x_slope=[S['m']['slope_bias'][key]['slope_x'] for key in sensors]
            y_slope=[S['m']['slope_bias'][key]['slope_y'] for key in sensors]
            h5f.create_dataset('/slope_bias/x_slope', data=np.array(x_slope))
            h5f.create_dataset('/slope_bias/y_slope', data=np.array(y_slope))
        if sensor_dict is not None:
            h5f.create_group('meta/sensors')
            for key in sensor_dict:
                h5f['/meta/sensors'].attrs['sensor_%d' % key]=sensor_dict[key]

    for key , ds in S['m'].items():
        if isinstance(ds, pc.grid.data):
            ds.to_h5(filename, group=key)
    return

def save_errors_to_file( S, filename, sensor_dict=None):
    with h5py.File(filename,'r+') as h5f:
        h5f.create_group('/dz/sigma/')
        for ii, name in enumerate(['y','x','t']):
            h5f.create_dataset('/dz/sigma/'+name, data=S['grids']['dz'].ctrs[ii])
        h5f.create_dataset('/dz/sigma/dz', data=S['E']['dz'])
        h5f.create_group('/z0/sigma')
        for ii, name in enumerate(['y','x']):
            h5f.create_dataset('/z0/sigma/'+name, data=S['grids']['z0'].ctrs[ii])
        h5f.create_dataset('/z0/sigma/z0', data=S['E']['z0'])
        for key in S['E']['bias']:
            h5f.create_dataset('/bias/sigma/'+key, data=S['E']['bias'][key])
        if 'slope_bias' in S['E']:
            sensors=np.array(list(S['E']['slope_bias'].keys()))
            h5f.create_dataset('/slope_bias/sigma/sensors', data=sensors)
            x_slope=[S['E']['slope_bias'][key]['slope_x'] for key in sensors]
            y_slope=[S['E']['slope_bias'][key]['slope_y'] for key in sensors]
            h5f.create_dataset('/slope_bias/sigma/x_slope', data=np.array(x_slope))
            h5f.create_dataset('/slope_bias//sigma/y_slope', data=np.array(y_slope))
    return

def fit_CS2(xy0, Wxy=4e4, E_RMS={}, t_span=[2003, 2020], spacing={'z0':2.5e2, 'dz':5.e2, 'dt':0.5},  \
            hemisphere=1, reference_epoch=None, reread_dirs=None, max_iterations=5, N_subset=None, Edit_only=False, \
            avg_scales=None, dzdt_lags=None,\
            sensor_dict={}, out_name=None, replace=False, DOPLOT=False, spring_only=False, \
            geoid_file=None, mask_file=None, calc_tide=False,\
            bias_nsigma_edit=None, bias_nsigma_iteration=3,\
            calc_error_file=None, data_file=None, restart_TSE=False,\
            DEM_file=None, DEM_tol=None):
    """
        Wrapper for smooth_xytb_fit that can find data and set the appropriate parameters
    """
    print("fit_CS2: working on %s" % out_name, flush=True)
    baseline_code={'C':2, 'D':3}
    # temporary:
    if hemisphere==-1:
        srs_proj4= '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
        #index_files={ 'D':{'POCA': glob.glob('/Volumes/ice3/ben/CS2_data/AA/retrack/2*/*/index_POCA.h5'),
        #                  'swath':glob.glob('/Volumes/ice3/ben/CS2_data/AA/retrack/2*/*/index_SW.h5')}}
        index_files={ 'D':{'POCA': glob.glob('/Volumes/ice3/ben/CS2_data/AA/tiles/2*/*/POCA/GeoIndex.h5'),
                          'swath':glob.glob('/Volumes/ice3/ben/CS2_data/AA/tiles/2*/*/sw/GeoIndex.h5')}}

    if hemisphere==1:
        srs_proj4= '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
        index_files={'C':{'POCA':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack_BC/2*/index_POCA.h5'),
                          'swath':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack_BC/2*/index_SW.h5')},
                    'D':{'POCA':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack/2*/index_POCA.h5'),
                          'swath':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack/2*/index_SW.h5')}
                    }

    compute_E=False
    # set defaults for E_RMS, then update with input parameters
    E_RMS0={'d2z0_dx2':200000./3000/3000, 'd3z_dx2dt':1000./3000/3000, 'd2z_dxdt':6000/3000, 'd2z_dt2':5000}
    E_RMS0.update(E_RMS)

    W={'x':Wxy, 'y':Wxy,'t':np.diff(t_span)}
    ctr={'x':xy0[0], 'y':xy0[1], 't':np.mean(t_span)}
    if out_name is not None:
        try:
            out_name=out_name %(xy0[0]/1000, xy0[1]/1000)
        except:
            pass

    if calc_error_file is not None:
        # get xy0 from the filename
        re_match=re.compile('E(.*)_N(.*).h5').search(calc_error_file)
        xy0=[float(re_match.group(ii))*1000 for ii in [1, 2]]
        data, sensor_dict=reread_data_from_fits(xy0, Wxy, [os.path.dirname(calc_error_file)])
        compute_E=True
        max_iterations=0
        N_subset=None
    elif data_file is not None:
        data=pc.data().from_h5(data_file, group='data')
        ###testing
        data.sigma_corr = 0.05 + 2*(data.swath>0.5)
    elif reread_dirs is None:
        data=[]
        for baseline in index_files.keys():
            data += [read_CS2_data(xy0, Wxy, index_files[baseline],
                                  apply_filters=True, DEM_file=DEM_file,
                                  dem_tol=50)]
            #'sigma':np.sqrt(0.2**2+0.5*(data.swath>0.5))
            ###testing
            data[-1].assign({'z':data[-1].h, \
                  'sigma_corr': 0.05 + 2*(data[-1].swath>0.5),
                  'baseline': baseline_code[baseline]+np.zeros_like(data[-1].swath)})
        if len(data) > 1:
            data[0].index(~np.in1d(data[0].abs_orbit, np.unique(data[1].abs_orbit)))
            data = pc.data().from_list(data)
        else:
            data=data[0]
        D_PC=data[data.swath==0]
        D_sw=data[data.swath==1]

        orb_dict=pc.bin_rows(np.c_[D_sw.abs_orbit])
        D_sw_sub=pc.data().from_list([D_sw[orb_dict[key]].blockmedian(400) for key in orb_dict])
        data=pc.data().from_list([D_PC, D_sw_sub])

        data.index((data.time>t_span[0]) & (data.time < t_span[1]))
        if calc_tide:
            if hemisphere==-1:
                # note: compute_tide_correction returns a masked array.
                # for now, we're going to convert it to an array by stripping the mask
                temp=compute_tide_corrections(\
                    data.x,data.y,(data.time-2000)*365.25*24*3600,
                    DIRECTORY='/Volumes/ice3/tyler/tide_models',MODEL='CATS2008',
                    EPOCH=(2000,1,1,0,0,0), TYPE='drift', TIME='utc')
                temp[temp.mask]=np.NaN
                data.assign({'tide':np.array(temp)})
                data.z[np.isfinite(data.tide)] -= data.tide[np.isfinite(data.tide)]
    else:
        data, sensor_dict = reread_data_from_fits(xy0, Wxy, reread_dirs, template='E%d_N%d.h5')
        N_subset=None

    if restart_TSE:
        if 'three_sigma_edit' in data.fields:
            data.three_sigma_edit=np.ones_like(data.x, dtype=bool)
        else:
            data.assign({'three_sigma_edit':np.ones_like(data.x, dtype=bool)})

    if reference_epoch is None:
        reference_epoch=np.ceil(len(np.arange(t_span[0], t_span[1], spacing['dt']))/2).astype(int)

    for field in data.fields:
        setattr(data, field, getattr(data, field).astype(np.float64))

    if DEM_file is not None:
        DEM=pc.grid.data().from_geotif(DEM_file, \
              bounds=[[xy0[0]-W['x']/1.8, xy0[0]+W['x']/1.8],\
                      [xy0[1]-W['y']/1.8, xy0[1]+W['y']/1.8]])
        data.assign({'DEM':DEM.interp(data.x, data.y)})

    # want less than 900K points
    if data.size > 9e5:
        D_sw=data[data.swath==1]
        D_poca=data[data.swath==0]
        D_sw.index(np.arange(0, D_sw.size, D_sw.size/(9e5-D_poca.size)).astype(int))
        data=pc.data().from_list([D_sw, D_poca])

    # run the fit
    KWargs={'data':data,
     'W': W,
     'ctr':ctr,
     'spacing':spacing,
     'srs_proj4':srs_proj4,
     'mask_file':mask_file,
     'E_RMS':E_RMS0,
     'E_RMS_d2x_PS_bias':1.e-7,
     'E_RMS_PS_bias':1,
     'compute_E':compute_E,
     'verbose': True,
     'mask_scale':{0:10, 1:1},
     'max_iterations':max_iterations,
     'N_subset':N_subset,
     'bias_params':['abs_orbit','swath'],
     'DEM_tol':DEM_tol,
     'avg_scales':avg_scales,
     'bias_filter': lambda D:D[D.swath>0.5],
     'bias_nsigma_edit':bias_nsigma_edit,
     'bias_nsigma_iteration':bias_nsigma_iteration
     }
    ### testing
    KWargs['bias_filter']=None
    S=smooth_xytb_fit(**KWargs)

    if out_name is not None:
        if calc_error_file is None:
            save_fit_to_file(S, out_name, sensor_dict, dzdt_lags=S['dzdt_lags'])
        else:
            save_errors_to_file(S, out_name, sensor_dict)
    print("fit_CS2: done with %s" % out_name, flush=True)
    return S, data, sensor_dict

def main(argv):
    # account for a bug in argparse that misinterprets negative agruents
    for i, arg in enumerate(argv):
        if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg

    import argparse
    parser=argparse.ArgumentParser(description="function to fit icebridge data with a smooth elevation-change model", \
                                   fromfile_prefix_chars="@")
    parser.add_argument('xy0', type=float, nargs=2, help="fit center location")
    parser.add_argument('--Width','-W',  type=float, help="fit width")
    parser.add_argument('--time_span','-t', type=str, help="time span, first year,last year AD (comma separated, no spaces)")
    parser.add_argument('--grid_spacing','-g', type=str, help='grid spacing:DEM (meters),dh maps xy (meters),dh_maps time (years): comma-separated, no spaces', default='250.,4000.,1.')
    parser.add_argument('--Hemisphere','-H', type=int, default=1, help='hemisphere: -1=Antarctica, 1=Greenland')
    parser.add_argument('--base_directory','-b', type=str, help='base directory')
    parser.add_argument('--out_name', '-o', type=str, help="output file name")
    parser.add_argument('--centers', action="store_true")
    parser.add_argument('--edges', action="store_true")
    parser.add_argument('--corners', action="store_true")
    parser.add_argument('--E_d2zdt2', type=float, default=5000)
    parser.add_argument('--E_d2z0dx2', type=float, default=0.02)
    parser.add_argument('--E_d3zdx2dt', type=float, default=0.0003)
    parser.add_argument('--data_gap_scale', type=float,  default=2500)
    parser.add_argument('--max_iterations', type=int, default=8)
    parser.add_argument('--mask_file', type=str)
    parser.add_argument('--geoid_file', type=str)
    parser.add_argument('--DEM_file', type=str)
    parser.add_argument('--DEM_tol', type=float, default=10)
    parser.add_argument('--calc_error_file','-c', type=str)
    parser.add_argument('--data_file', type=str)
    parser.add_argument('--restart_TSE', action='store_true', help='if reloading data, restart the three-sigma editing')
    parser.add_argument('--dzdt_lags', type=str, default='1,4', help='lags for which to calculate dz/dt, comma-separated list, no spaces')
    parser.add_argument('--avg_scales', type=str, help='scales at which to report average errors, comma-separated list, no spaces')
    parser.add_argument('--bias_nsigma_edit', type=int, default=6, help='edit points whose estimated bias is more than this value times the expected')
    parser.add_argument('--bias_nsigma_iteration', type=int, default=6, help='apply bias_nsigma_edit after this iteration')
    parser.add_argument('--calc_tide', action='store_true')

    args=parser.parse_args()

    if args.avg_scales is not None:
        args.avg_scales = [np.int(temp) for temp in args.avg_scales.split(',')]
    args.dzdt_lags = [np.int(temp) for temp in args.dzdt_lags.split(',')]

    args.grid_spacing = [np.float(temp) for temp in args.grid_spacing.split(',')]
    args.time_span = [np.float(temp) for temp in args.time_span.split(',')]

    spacing={'z0':args.grid_spacing[0], 'dz':args.grid_spacing[1], 'dt':args.grid_spacing[2]}
    E_RMS={'d2z0_dx2':args.E_d2z0dx2, 'd3z_dx2dt':args.E_d3zdx2dt, 'd2z_dxdt':args.E_d3zdx2dt*args.data_gap_scale,  'd2z_dt2':args.E_d2zdt2}

    reread_dirs=None
    if args.centers:
        dest_dir = args.base_directory+'/centers'
    if args.edges or args.corners:
        reread_dirs=[args.base_directory+'/centers']
        dest_dir = args.base_directory+'/edges'
    if args.corners:
        reread_dirs += [args.base_directory+'/edges']
        dest_dir = args.base_directory+'/corners'

    if args.calc_error_file is not None:
        dest_dir=os.path.dirname(args.calc_error_file)
        # get xy0 from the filename
        re_match=re.compile('E(.*)_N(.*).h5').search(args.calc_error_file)
        args.xy0=[float(re_match.group(ii))*1000 for ii in [1, 2]]

    if not os.path.isdir(args.base_directory):
        os.mkdir(args.base_directory)
    if not os.path.isdir(dest_dir):
        os.mkdir(dest_dir)

    if args.out_name is None:
        args.out_name=dest_dir + '/E%d_N%d.h5' % (args.xy0[0]/1e3, args.xy0[1]/1e3)

    if args.DEM_tol is not None and args.DEM_file is None:
        print("Must specify a DEM if DEM_tol is specified")
        sys.exit(1)

    fit_CS2(args.xy0, Wxy=args.Width, E_RMS=E_RMS, t_span=args.time_span, spacing=spacing, \
            hemisphere=args.Hemisphere, reread_dirs=reread_dirs, \
            out_name=args.out_name, DOPLOT=False, \
            mask_file=args.mask_file, geoid_file=args.geoid_file, \
                DEM_file=args.DEM_file, DEM_tol=args.DEM_tol, \
                calc_error_file=args.calc_error_file,
                data_file=args.data_file,
                restart_TSE=args.restart_TSE,
                max_iterations=args.max_iterations,
                avg_scales=args.avg_scales,
                dzdt_lags=args.dzdt_lags,
                bias_nsigma_edit=args.bias_nsigma_edit,
                bias_nsigma_iteration=args.bias_nsigma_iteration,
                calc_tide=args.calc_tide)

if __name__=='__main__':
    main(sys.argv)

# -440000. -560000. -W 40000 -t 2010,2019 -g 500,2000,1 -H -1 -b /Volumes/ice2/ben/CS2_fit/AA -o test1_halfdata.h5 --centers

# for Antarctica
#  -350000 -460000 --centers @/Volumes/ice2/ben/CS2_fit/AA/default_args_AA.txt

# for Greenland:
#  600000 -1240000 --centers  @/Volumes/ice2/ben/CS2_fit/GL/default_args_GL.txt


import pandas as pd
import numpy as np

if __name__ == '__main__':
    pd.set_option('display.max_columns', None)

    columns=["Sex", "Country"]
    X = [[np.nan, 'Germany'], ['Male', np.nan], ['Female', 'Poland'],['Female', 'Brasil'],[np.nan, 'Brasil']]
    df=pd.DataFrame(data=X,columns=columns)
    print(df.head())

    drop_columns=True
    for column in columns:
        labels=df[column].unique().tolist()
        labels = [str(x) for x in labels]  # converting nan to 'nan
        try:
            labels.remove('nan') #remove nan labels
        except:
            pass
        for label in labels:

            new_label=str(column)+'_'+str(label)
            print(f"New label: {new_label} Column: {column}, label: {label}")
            df[new_label] = np.where(df[column] == label, 1, 0)
            df.loc[df[column].isna(), new_label] = np.nan
    if drop_columns:
        df=df.drop(columns=columns) #drop encoded columns

    print(df.head())
    new_columns=df.columns.to_list()
    print(new_columns)

    # explicitly require this experimental feature
    from sklearn.experimental import enable_iterative_imputer  # noqa
    # now you can import normally from sklearn.impute
    from sklearn.impute import IterativeImputer

    imp_mean = IterativeImputer(min_value=0,
                                max_value=1,
                                random_state=0,
                                max_iter=100,
                                tol=1e-6,
                                verbose=2)

    df_imputed=imp_mean.fit_transform(df)
    df_imputed=pd.DataFrame(df_imputed,columns=new_columns)
    print(df_imputed.head())


from worker import Worker
import logging
import pickle
from multiprocessing import Process, Queue, Array, cpu_count
from time import sleep
from Env_example import Env
from baseline import LinearBaseline
import os
import numpy as np
import argparse
from tensorflow import nn
from tensorflow.python.client import device_lib

#load previously saved global initial parameters
def load_params(path):
    global_params = []
    with open(path,'rb') as f:
            gps = pickle.load(f)
    for gp in gps:
        global_params.append(Array('f', gp.flatten()))
    return global_params

def parse_args():
    parser = argparse.ArgumentParser(description='Implement multi-process TRPO\
                                     in tensorflow, solving zero-order \
                                     optimization problem')
    parser.add_argument('--init_params_path', type=str, default='global_vars.pickle',
                        help='the path to saved initial parameters')
    parser.add_argument('--hidden_sizes', type=str, default="128,64",
                        help='the size of hidden layer of the policy network\
                        format should be l1,l2, ... ,ln')
    parser.add_argument('--init_std', type=float, default=1.0,
                        help='the initial value of std dev used by gaussian\
                        sampling function')
    parser.add_argument('--hidden_activation', type=str, default="relu",
                        help='the activation used by hidden layer')
    parser.add_argument('--output_activation', type=str, default="tanh",
                        help='the activation used by output layer')
    parser.add_argument('--max_global_iters', type=int, default=501,
                        help='the max number of total iterations of all the\
                        worker processes')
    parser.add_argument('--batch_size', type=int, default=5000,
                        help='the batch size for each worker used to train')
    parser.add_argument('--max_path_length', type=int, default=10,
                        help='the maximum length of each path')
    parser.add_argument('--discount', type=float, default=0.0,
                        help='discount factor when calculate accumulative rewards')
    parser.add_argument('--log_file_name', type=str, default="tf_multiprocess.log",
                        help='the file name of the log recording each workers \
                        activity')
    parser.add_argument('--plot', type=int, default=0,
                        help='the flag indicate to save sampled path or not')
    parser.add_argument('--save_period', type=int, default=100,
                        help='the period of global iterations in which worker_0\
                        will save parameters. If plot is set to be true, a sampled\
                        path will be saved into a pickled binary file')
    parser.add_argument('--over_sampling_ratio', type=int, default=40,
                        help='the ratio over the batch_size that data sampler will sampled by')
    parser.add_argument('--alpha', type=float, default=1.35,
                        help='the parameter alpha of the exponential distribution\
                        that prioritized experience will use')
    parser.add_argument('--beta_zero', type=float, default=0.4,
                        help='the parameter beta of the importance sampling\
                        annealing that prioritized experience will use')
    parser.add_argument('--seeds', type=str, default="1018,1120,211,318",
                        help='the random seeds that each worker will recieve,\
                        format should be s1,s2,...,sn')
    parser.add_argument('--predict_size', type=int, default=10,
                        help='the batch_size that each worker will sampled when\
                        predicting the final minimum point')
    parser.add_argument('--log_std_tol', type=float, default=-5.0,
                        help='the tolerance of the log standard deviation that\
                        gaussian sampling will be parametrized. A too samll value\
                        can increase the convergence rate of the policy, leadint to\
                        a finer result. However, in this case, the training process\
                        will be controlled by max_global_iters. A too large value\
                        will lead to an inaccurate result')
    parser.add_argument('--verbose', type=int, default=0,
                        help='verbose level of logging. 0 only plot basic info, \
                        1 will plot specific training info, 2 will plot detailed \
                        policy convergence data')
    parser.add_argument('--num_gpu_devices', type=int, default=1,
                        help='The number of gpu devices in your mahcine. I have tried\
                        cpu implementation but it is way too slow.')
    parser.add_argument('--vanilla', type=int, default=0,
                        help='Use vanilla TRPO or prioritized experience TRPO. Vanilla TRPO usually has higher convergence stability but easily get stuck in local minimum.')
    return parser.parse_args()

def get_activation(act_type="relu"):
    if act_type == "relu":
        return nn.relu
    elif act_type == "tanh":
        return nn.tanh
    else:
        raise NotImplementedError

def getKey(path):
    return path[0]

if __name__ == "__main__":
    args = parse_args()

    #parsing some options
    init_path = args.init_params_path
    hidden_sizes_str = args.hidden_sizes.split(',')
    hidden_sizes = []
    for h in hidden_sizes_str:
        hidden_sizes.append(int(h))
    init_std = args.init_std
    hidden = args.hidden_activation
    output = args.output_activation
    hidden_activation = get_activation(hidden)
    output_activation = get_activation(output)
    max_global_iters = args.max_global_iters
    batch_size = args.batch_size
    max_path_length = args.max_path_length
    discount = args.discount
    plot = args.plot
    if plot is not 0:
        plot = 1
    save_period = args.save_period
    over_sampling_ratio = args.over_sampling_ratio
    alpha = args.alpha
    beta_zero = args.beta_zero
    seeds = []
    seeds_str = args.seeds.split(',')
    for s in seeds_str:
        seeds.append(int(s))
    predict_size = args.predict_size
    log_std_tol = args.log_std_tol
    verbose = args.verbose

    #logging setting
    log_file_name = args.log_file_name
    logging.basicConfig(level=logging.INFO,
                        format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                        datefmt='%d %b %H:%M:%S',
                        filename=log_file_name,
                        filemode='w')

    global_params = load_params(init_path)
    results = []
    num_gpu_devices = args.num_gpu_devices
    vanilla = args.vanilla
    worker_count = 0
    while(len(seeds) > 0):
        #object lists to record each worker
        workers = []
        envs = []
        baselines = []
        # Create worker classes
        for i in range(num_gpu_devices):
            if(i >= len(seeds)):
                break;
            envs.append(Env(seeds[i]))
            baselines.append(LinearBaseline())
            workers.append(Worker(
                                envs[-1],
                                worker_count,
                                i,
                                baselines[-1],
                                hidden_sizes = hidden_sizes,
                                init_std = init_std,
                                hidden_activation = hidden_activation,
                                output_activation = output_activation,
                                batch_size=batch_size,
                                max_path_length=max_path_length,
                                max_global_iters = max_global_iters,
                                discount=discount,
                                save_period = save_period,
                                plot = plot,
                                over_sampling_ratio = over_sampling_ratio,
                                alpha = alpha,
                                beta_zero = beta_zero,
                                seed = seeds[i],
                                predict_size = predict_size,
                                log_std_tol = log_std_tol,
                                verbose = verbose,
                                vanilla = vanilla
                            ))
            worker_count += 1
        #Start worker process using python multiprocessing wrapper
        worker_processes = []
        results_queue = Queue()
        for worker in workers:
            p = Process(target=worker.work, args=(global_params, results_queue))
            p.start()
            sleep(0.5)
            worker_processes.append(p)
        for process in worker_processes:
            process.join()
        for i in range(num_gpu_devices):
            results.append(results_queue.get())
        if len(seeds) > num_gpu_devices:
            seeds = seeds[num_gpu_devices:]
        else:
            seeds = []
    #Gathering and sorting results from each child process
    logging.info("The global minimum is:")
    logging.info(np.array2string(sorted(results,key=getKey)[0][1], max_line_width=10000))


# pommerman/cli/run_battle.py
# pommerman/agents/TensorFlowAgent/pit.py

import atexit
from datetime import datetime
import os
import random
import sys
import time

import argparse
import numpy as np

from pommerman import helpers, make
from TensorFlowAgent import TensorFlowAgent

from pommerman import utility

import tensorflow as tf

class Pit(object):

    def __init__(self, tfa, saver, game_nums=2):

        self.tfa = tfa
        self.saver = saver
        self.game_nums = game_nums

    def launch_games(self, sess, render=True):

        sess.run(tf.global_variables_initializer())
        self.tfa.restore_weigths(sess, self.saver)
        env = self.tfa.getEnv()

        reward_board = np.zeros((1, 4))

        for i in range(self.game_nums):

            curr_state = env.reset()

            while True:

                if render:
                    env.render()

                all_actions = env.act(curr_state)
                next_state, reward, terminal, _ = env.step(all_actions)

                if terminal:

                    reward_board += np.array(reward)

                    print("Game #{0}, rewards = {1}, reward agent = {2}".format(i, "".join(str(i) + " " for i in reward), reward[self.tfa.agent_id]))
                    break

def main(args):

    tf.reset_default_graph()

    with tf.Session() as sess:

        tfa = TensorFlowAgent(name="TFA", args=args, sess=sess)
        saver = tf.train.Saver(allow_empty=True)
        pit = Pit(tfa, saver, game_nums=2)

        pit.launch_games(sess)

if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("--environment", type=str, default="pommerman")
    parser.add_argument("--policy", type=str, default="MlpPolicy")
    parser.add_argument("--checkpoint_dir", type=str, default="./save_model")
    parser.add_argument("--a_learning_rate", type=float, default=0.0001)
    parser.add_argument("--c_learning_rate", type=float, default=0.0002)
    parser.add_argument('--summary_dir', type=str, default='./summary_log')
    parser.add_argument("--cliprange", type=float, default=0.2)
    parser.add_argument("--batch_size", type=int, default=32)
    parser.add_argument("--training_step", type=int, default=10)
    parser.add_argument("--gamma", type=float, default=0.9)
    parser.add_argument("--train", type=str, default="False", choices=["False"])
    parser.add_argument("--type", type=str, default="Simple", choices=["Simple, CNN"])

    args = parser.parse_args()

    main(args)


import networkx as nx
import random
import matplotlib.pyplot as plt

def main(plot=False, load=False, prnt_list=False, start_percent=5, infect_percent=15,
        days_infected=3, nodes=100, type="watts", draw="circular", vaccinate_percent=0, vaccine_affectiveness=0):
    num_days = list()
    num_healthy = list()
    for i in range(0, 10):
        if load:
            graphs = LoadFile()
        else:
            graphs = GetRandomGraph(nodes, type)
        print("Clustering Coefficient: " + str(nx.clustering(graphs[0])))
        G = StartInfection(graphs[0], start_percent, days_infected)
        G = Vaccinate(graphs[0], vaccinate_percent)
        color_map = MakeColorMap(G)

        statuses = ListStatus(G)
        if prnt_list:
            print("Day: 0")
            print(statuses)

        if plot:
            if draw == "circular":
                nx.draw_circular(G, node_color=color_map, with_labels=True, font_weight='bold')
            else:
                nx.draw(G, node_color=color_map, with_labels=True, font_weight='bold')
            plt.show()

        # for i in range(0, 10):
        i = 0
        while "Infected" in statuses:
            G = SpreadInfection(G, infect_percent,vaccine_affectiveness)
            # color_map = MakeColorMap(G)
            # nx.draw(G, node_color=color_map, with_labels=True, font_weight='bold')
            # plt.show()

            G = UpdateInfection(G, days_infected)
            color_map = MakeColorMap(G)

            i += 1
            statuses = ListStatus(G)
            if prnt_list:
                print("Day: " + str(i))
                print(statuses)

            if plot:
                if draw == "circular":
                    nx.draw_circular(G, node_color=color_map, with_labels=True, font_weight='bold')
                else:
                    nx.draw(G, node_color=color_map, with_labels=True, font_weight='bold')
                plt.show()
        num_days.append(i)
        num_healthy.append(len(statuses['Healthy']))
    print(num_days)
    print(num_healthy)

def Vaccinate(G, vaccinate_percent):
    for node_data in G.nodes.data():
        node = G.nodes[node_data[0]]
        if random.randint(0, 100) < vaccinate_percent:
            node['vaccinated'] = True
        else:
            node['vaccinated'] = False
    return G

def SpreadInfection(G, infect_percent, vaccine_affectiveness):
    for edges in G.edges:
        if (G.nodes[edges[0]]["status"] == "Healthy" and G.nodes[edges[1]]["status"] == "Infected") or (G.nodes[edges[0]]["status"] == "Infected" and G.nodes[edges[1]]["status"] == "Healthy"):
            node = edges[0]
            if G.nodes[edges[1]]["status"] == "Healthy":
                node = edges[1]

            if random.randint(0, 100) < infect_percent:
                if not G.nodes[node]["vaccinated"] or random.randint(0, 100) > vaccine_affectiveness:
                    G.nodes[node]["status"] = "Newly Infected"

        # print(edges[0] + ":" + G.nodes[edges[0]]["status"])
        # print(edges[1] + ":" + G.nodes[edges[1]]["status"])
        # print()
    return G

def UpdateInfection(G, days_infected):
    for node_data in G.nodes.data():
        node = G.nodes[node_data[0]]
        if node['status'] == "Infected":
            node['days_remaining'] -= 1
            if node['days_remaining'] == 0:
                node['status'] = "Recovered"
        elif node["status"] == "Newly Infected":
            node['status'] = "Infected"
            node['days_remaining'] = days_infected
    return G

def StartInfection(G, start_percent, days_infected):
    for node_data in G.nodes.data():
        node = G.nodes[node_data[0]]
        if random.randint(0, 100) < start_percent:
            node['status'] = "Infected"
            node['days_remaining'] = days_infected
        else:
            node['status'] = "Healthy"
    return G

def MakeColorMap(G):
    color_map = list()
    for node_data in G.nodes.data():
        node = G.nodes[node_data[0]]
        if node['status'] == "Infected":
            color_map.append("red")
        elif node['status'] == "Healthy":
            color_map.append("blue")
        elif node["status"] == "Newly Infected":
            color_map.append("yellow")
        elif node['status'] == "Recovered":
            color_map.append("green")
    return color_map

def ListStatus(G):
    statuses = dict()
    for node_data in G.nodes.data():
        node = G.nodes[node_data[0]]
        if node['status'] not in statuses:
            statuses[node['status']] = list()
        statuses[node['status']].append(node_data[0])
    return statuses

def GetRandomGraph(nodes, type):
    if (type == "random"):
        return [nx.gnm_random_graph(nodes, 350)]
    elif (type == "watts"):
        return [nx.connected_watts_strogatz_graph(nodes, 10, .3)]
    return [nx.connected_watts_strogatz_graph(nodes, 5, .3)]

def LoadFile():
    graph_list = list()
    with open('12_3_3.txt') as f:
        isGraph = True
        count = 0
        entry = 0
        for x in f:
            if x.startswith("Graph"):
                isGraph = True
                graph_list.append(dict())
                entry = 1
            elif x.startswith("Taillenweite"):
                isGraph = False
                count += 1
            elif isGraph and x.strip() != "":
                graph_list[count][entry] = x.split(":")[1].strip().split(" ")
                entry += 1
    count = 0
    graphs = list()
    for graph in graph_list:
        count += 1
        G = nx.Graph()
        for key in graph:
            for edge in graph[key]:
                G.add_edge(str(key), edge)
        graphs.append(G)

    return graphs


import os
import matplotlib.pyplot as plt

import torch

from population import Population
from genome import Genome
from optimizer import ADAMGene, SGDGene
from node import Node
from gene import KernelGene, PoolGene, DenseGene
from net import build_net_from_genome

def decode(line):
    """
    Reconstructs genome from str printed to console
    """
    alias = {"depth_change": "depth_size_change", "str": "stride", "pad": "padding", "d_mult": "depth_mult",
             "pool": "pooling", "mom": "momentum",
             "size": lambda x: [["width", "height"], x.split("*")] if not x == "None" else [[], []],
             "ID": lambda x: [["id"], [int(x)]] if "+" not in x else
                             [["id", "id_in", "id_out"], list(map(int, x.split("+")))]}
    # Stack of objects to be build. Form [class, attributes, Subobject in creation]
    object_stack = []
    for part in line.replace("|", ",").replace("->", "+").replace("(", "+").replace(")", ",").replace(">", ",>").\
            replace("[", "[,").replace("]", ",]").replace("\n", "").replace("False", "enabled=False").\
            replace("True", "enabled=True").split(","):
        if "<" in part or "[" in part:
            # Start genome
            if len(object_stack) == 0:
                object_stack.append([Genome, {"population": 1}, None])
            # New object attribute
            else:
                if "=" not in part:
                    name = None
                    strobj = part
                else:
                    name, strobj = part.split("=")
                obj = None
                for o, n in [(ADAMGene, "ADAMGene"), (SGDGene, "SGDGene"), (KernelGene, "KernelGene"),
                             (PoolGene, "PoolGene"), (DenseGene, "DenseGene"), (Node, "Node"), (list, "[")]:
                    if n in strobj:
                        obj = o
                        break
                if name is not None:
                    object_stack[-1][2] = name.replace(" ", "")
                # Add to Object Stack
                if obj == list:
                    object_stack.append([obj, [], None])
                else:
                    object_stack.append([obj, dict(), None])
        # List done
        elif part == "]":
            obj_raw = object_stack.pop(-1)
            object_stack[-1][1].update({object_stack[-1][2]: obj_raw[1]})
        # Object done
        elif part == ">":
            obj_raw = object_stack.pop(-1)
            obj = obj_raw[0](**obj_raw[1])
            if len(object_stack) == 0:
                # Genome constructed
                obj.nodes_by_id[0].role = 'input'
                obj.nodes_by_id[1].role = 'flatten'
                obj.nodes_by_id[2].role = 'output'
                return obj
            # Add the object attribute
            if isinstance(object_stack[-1][1], list):
                object_stack[-1][1] += [obj]
            else:
                object_stack[-1][1].update({object_stack[-1][2]: obj})
        elif "=" in part:
            # kwarg
            k, v = part.replace(" ", "").split("=")
            if callable(alias.get(k, k)):
                ks, vs = alias.get(k)(v)
            else:
                ks, vs = [alias.get(k, k)], [v]
            for i, v in enumerate(vs):
                try:
                    vs[i] = int(v)
                except (ValueError, TypeError):
                    try:
                        vs[i] = float(v)
                    except (ValueError, TypeError):
                        pass
            for k, v in zip(ks, vs):
                object_stack[-1][1].update({k: v})
        else:
            # arg (only in lists)
            v = part.replace(" ", "")
            try:
                v = int(v)
            except ValueError:
                pass
            object_stack[-1][1] += [v]

def from_human_readable(evaluate, input_size, output_size):
    while True:
        file = input("check genomes from file:")
        if not os.path.exists(file):
            continue
        with open(file, 'r') as f:
            for line in f.readlines():
                if "Genome" in line:
                    genome = decode(line)
                    fig, ax = plt.subplots()
                    genome.visualize(ax=ax, input_size=(1, 28, 28), dbug=True)
                    plt.show()
                    net, _, _ = build_net_from_genome(genome, input_size, output_size)
                    evaluate(net)
        return

def show_genomes(input_size, simultan=False):
    # What to show
    checkpoint = input("checkpoint name:")
    gens = input("generations ['all' / list separated by ' ']:")
    if gens == 'all':
        i = 1
        while os.path.exists(os.path.join("checkpoints", checkpoint, "%02d.cp" % i)):
            i += 1
        generations = list(range(1, i))
    else:
        generations = list(map(int, gens.split(" ")))

    for i in generations:
        p = Population(1, 1, 1, 1, 1, 1, 1, load=(checkpoint, i), load_params=False)
        show = 7
        fig, axs = plt.subplots(len(p.species), show, squeeze=False, figsize=(20, 10))
        fig.canvas.set_window_title('Generation %d' % i)
        for j, sp in enumerate(sorted(p.species.keys(), key=lambda x: p.history[-1][x][1] or 0, reverse=True)):
            for k, g in enumerate(sorted(p.species[sp], key=lambda x: x.acc or 0, reverse=True)[:show]):
                g.visualize(ax=axs[j, k], input_size=input_size)
                axs[j, k].title.set_text('%sacc: %s %%' % ("Species %d " % sp if k == 0 else "",
                                                           "%.2f" % (100 * g.acc) if g.acc is not None else "-"))
        if simultan:
            plt.pause(0.01)
        else:
            plt.show()
    plt.show()

"""
    # Plot Species representative ?
    for i in generations:
        p = Population(1, 1, 1, 1, 1, 1, 1, load=(checkpoint, i))
        show = 7
        fig, axs = plt.subplots(len(p.species), show, squeeze=False, figsize=(20, 10))
        fig.canvas.set_window_title('Generation %d' % i)
        for j, sp in enumerate(sorted(p.species.keys(), key=lambda x: p.history[-1][x][1] or 0, reverse=True)):
            p.species_repr[sp].visualize(ax=axs[j, 0], input_size=input_size)
            axs[j, k].title.set_text('species %d, mean_acc: %s %%' % (sp, p.history[j-1][sp][1] * 100))
            for k, g in enumerate(sorted(p.species[sp], key=lambda x: x.score or 0, reverse=True)[:show]):
                g.visualize(ax=axs[j, k], input_size=input_size)
                axs[j, k].title.set_text('acc: %.2f %%' % ((100 * g.score) if g.score is not None else "-"))
        plt.pause(0.01)
"""


#!/usr/bin/env python
from pathlib import Path
import pylivestream as pls
import pytest

R = Path(__file__).parent

inifn = R / 'test.ini'

@pytest.mark.parametrize('site', ['periscope', 'youtube', 'facebook'])
def test_disk(site):
    p = pls.SaveDisk(inifn, '')
    assert p.site == 'file'
    assert p.video_kbps == 3000

if __name__ == '__main__':
    pytest.main(['-x', __file__])


import sys
from collections import deque

N = int(sys.stdin.readline())

for i in range(N):
  S = sys.stdin.readline().rstrip()
  result = ''
  N_deque = deque()
  for j in range(0, len(S)):
    if S[j] == '(':
      N_deque.insert(j,'(')
      N_deque.insert(j+1,')')

  for k in N_deque:
    result += k

  if S == result:
    print('YES')
  else:
    print('NO')


from pathlib import Path
import pandas as pd
from zipfile import ZipFile
import streamlit as st
import urllib.response
import altair as alt
from sklearn.feature_selection import r_regression
import numpy as np

def line_break():
    st.text("\n")
    st.text("\n")
    st.text("\n")
    st.text("\n")
    st.text("\n")
    st.text("\n")
    st.text("\n")

    return

def year_range(start_year:int, end_year_exclusive:int):
    return [*range(start_year, end_year_exclusive)]

st.set_page_config(layout="wide")

st.sidebar.title("Sobre")
st.sidebar.info(
    """

    **Visualização Computacional** - SCC0252 - Trabalho de Recuperação

    **Autor**: Heitor Carvalho Pinheiro\n
    **Docente**: Maria Cristina Ferreira de Oliveira

    """
)

st.sidebar.title("Contato")
st.sidebar.info(
    """
    E-mail: heitor.c.pinheiro@usp.br
    [LinkedIn](https://www.linkedin.com/in/heitor-carvalho-pinheiro-4b58851b7/) | [Github](https://github.com/Heitorcp)
    """
)

st.write("# Emissões Globais de CO2 :factory:")

#filtering the years
start_year, end_year = st.select_slider(
    ":date: **Selecione os anos que deseja observar**",
    options = [*range(1750, 2023)], value = (1900, 2022))

def pivot_df(df, index:str, columns:str, values:str):

    pivot_df = pd.pivot(df, index=index, columns=columns, values=values)
    return pivot_df

@st.cache
def read_CO2_data():
    try:
        csv_path = Path("data/GCB2022v27_MtCO2_flat.csv")
    except:
        if not csv_path.is_file():
            Path("data").mkdir(parents=True, exist_ok=True)
            with ZipFile("data/archive.zip", 'r') as zObject:
                zObject.extractall(path="data")
    return pd.read_csv(Path("data/GCB2022v27_MtCO2_flat.csv"))

@st.cache
def data_with_coords():
    try:
        data = read_CO2_data().iloc[:, [0] + [*range(2,11)]]
        coords = pd.read_csv("./data/country-coord.csv", usecols=[0,3,4,5], dtype={'Numeric code':str})
        coords = coords.rename(columns={"Numeric code":"id"})

        df_with_coords = data.join(coords.set_index('Country'), on='Country')
        #filter years
        df_with_coords = df_with_coords.set_index('Year').loc[year_range(2000,2022)].reset_index()

    except BaseException as e:
        raise e

    return df_with_coords

def data_with_coords_r2_score():
    try:
        years = [*range(2000,2022)]
        data = data_with_coords().iloc[:,[0,1,2,10,11,12]]
        data = data.set_index("Year").loc[years].reset_index()
        data_groupped = data.groupby("Country")

        r2_dict = {}

        countries = data.Country.unique()

        for country in countries:

            df = data_groupped.get_group(country)

            X = df.iloc[:, [0]]
            y = df.iloc[:, 2]
            r2_score = r_regression(X,y)[0]
            r2_score = r2_score.round(2)

            r2_dict[country] = r2_score

        df_final = pd.DataFrame.from_dict(r2_dict, orient="index", columns=["r2_score"]).reset_index()
        df_final.columns = ["Country", "r2_score"]

        conditions = [

        df_final['r2_score'] < -0.8,
        (df_final['r2_score'] < -0.6) & (df_final['r2_score'] >= -0.8),
        (df_final['r2_score'] <= 0.6) & (df_final['r2_score'] >= 0.),
        (df_final['r2_score'] <= 0.)  & (df_final['r2_score'] >= -0.6),
        (df_final['r2_score'] <= 0.8) & (df_final['r2_score'] >= 0.6),
        df_final['r2_score'] > 0.8

                    ]

        choices = ["Group A", "Group B", "Group C", "Group C", "Group D", "Group E"]

        df_final["Group"] = np.select(conditions, choices)

        #merging coords with groups
        data = data.iloc[:, [1,3,4,5]]

        df_groups = pd.merge(data, df_final, on="Country")
        df_groups.drop_duplicates(inplace=True)

    except BaseException as e:
        raise e

    return df_groups

print(data_with_coords_r2_score())

def emission_per_category():

    df_per_category = read_CO2_data()[read_CO2_data().Country != "Global"]
    df_per_category = df_per_category.groupby(["Year","Coal","Oil","Gas","Cement","Flaring","Other"]).sum()
    df_per_category = df_per_category.reset_index()
    df_per_category = df_per_category.iloc[:,[0,1,2,3,4,5,6]]
    df_per_category = df_per_category.set_index("Year").loc[years]
    df_per_category = df_per_category.reset_index().melt(id_vars=["Year"], var_name = "Category", value_name="Total")
    df_per_category = df_per_category.astype({"Year":"string","Category":"string", "Total":"float64"})
    df_per_category = df_per_category.groupby(["Year","Category"])["Total"].sum().reset_index()

    return df_per_category

@st.cache
def rank_by_column(column_to_rank_by:str = ["Total", "Oil", "Coal", "Gas", "Flaring", "Cement", "Other", "Per Capita"]) -> pd.DataFrame():

    years = year_range(2000, 2022)

    df = read_CO2_data().loc[(read_CO2_data().Year.isin(years)) & (read_CO2_data().Country != "Global")]
    column_ranked = df[[column_to_rank_by, "Year", "Country"]].sort_values(by=["Year",column_to_rank_by], ascending=(True, False))

    ranks = column_ranked.groupby("Year")[column_to_rank_by].rank(method='first', ascending=False).reset_index()
    ranks.rename(columns={column_to_rank_by:"Rank"}, inplace=True)

    #joining on index
    ranked_df = column_ranked.join(ranks.set_index('index'), on=column_ranked.index)
    return ranked_df

try:
    df_raw = read_CO2_data()
    df = df_raw.iloc[:,[0,2,3]]
    df = df[df.Year.isin(range(start_year, end_year))]
    # print(df)

    countries = st.multiselect(
        ":earth_americas: **Selecione os países**", list(df.Country.unique()), ["USA", "China", "Russia",
        "Germany", "United Kingdom", "Japan", "India", "France","Canada"]
    )
    if not countries:
        st.error("Please select at least one country.")
    else:

        #df-wide format
        df = pivot_df(df, index='Country', columns='Year', values='Total')
        df = df.loc[countries]

        # st.write("Annually CO2 Emission", df.sort_index())

        #df long-format
        df =df.T.reset_index()
        df = df.astype({"Year":"str"})
        df = pd.melt(df, id_vars=["Year"]).rename(columns={
            "value":"Total (kt)"
        })

        area_chart = (
            alt.Chart(df).mark_area(opacity=0.4).encode(
                x="Year:T",
                y=alt.Y("Total (kt):Q", stack=None),
                color="Country:N"
            )
        )

        st.altair_chart(area_chart, use_container_width=True)
except urllib.error.URLError as e:
    st.error(
        """
        **This dashboard requires internet access.**
        Conection error: %s
        """
        % e.reason
    )

st.markdown("O gráfico acima, por *default*, representa os 9 países com as maiores emissões de CO2, por kilotoneladas, ao longo dos últimos 120 anos.\n\
    Fica evidente a tendência de crescimento da emissão ao longo dos anos.")

#LAYING OUT THE MIDDLE SECTION OF THE APP
row2_1, row_2_2 = st.columns((2,1))

with row2_1:
    st.subheader("Maiores emissores de CO2")
    st.write("**With the highest historical CO2 emissions**")

    try:
        total_by_country = df_raw.groupby('Country')['Total'].sum().reset_index()

        top_10_countries = total_by_country.sort_values(by="Total", ascending=False)[1:11]
        top_10_countries = top_10_countries.astype({"Total":"float64", "Country":"string"})

        bar_chart_top10 = alt.Chart(top_10_countries).mark_bar().encode(
                x=alt.X('Total:Q'),
                y = alt.Y('Country:N', sort="-x"),
                color=alt.Color(
                    "Country",
                    scale=alt.Scale(
                    domain = top_10_countries.Country.tolist(),
                    range = ['red']*3+['steelblue']*7),
                    legend=None
                )
            )

        st.altair_chart(bar_chart_top10, use_container_width=True)

    except BaseException as e:
        raise e

with row_2_2:
    line_break()
    st.markdown("Os três maiores países com as maiores taxas de emissões de CO2 históricas são os :red[países mais populosos]")

row_3_1, row_3_2 = st.columns((3,1))

with row_3_1:
    st.subheader("Menores emissores de CO2")
    st.write("**With the lowest historical CO2 emissions**")

    try:
        total_by_country = df_raw.groupby('Country')['Total'].sum().reset_index()

        top_10_low_countries = total_by_country.sort_values(by="Total", ascending=True)[:10]
        top_10_low_countries = top_10_low_countries.astype({"Total":"float64", "Country":"string"})

        bar_chart_top10_low = alt.Chart(top_10_low_countries).mark_bar().encode(
                x=alt.X('Total:Q'),
                y = alt.Y('Country:N', sort="x"),
                color=alt.Color(
                    "Country",
                    scale=alt.Scale(
                    domain = top_10_low_countries.Country.tolist(),
                    range = ['green']*3+['steelblue']*7),
                    legend=None
                )
            )

        st.altair_chart(bar_chart_top10_low, use_container_width=True)

    except BaseException as e:
        raise e

with row_3_2:
    line_break()
    st.markdown("Os três países com as menores taxas de emissões de CO2 históricas são os :green[países que apresentam as menores taxas populacionais]")

st.markdown("## Variação Anual para o período de 2000-2020")
st.markdown("### Emissão Total por ano")
st.markdown("Como observado no início, nos últimos 20 anos a tendência observada é de crescimento an emissão de CO2.\n \
    As quedas observadas em 2008 e 2019, referem-se, respectivamente, à crise imobiliária de 2008 e à pandemia do COVID-19")

years = year_range(2000, 2022)

try:
    df_last_22_years = df_raw[df_raw.Country != "Global"].groupby("Year")["Total"].sum().reset_index()
    df_last_22_years = df_last_22_years.set_index("Year")
    df_last_22_years = df_last_22_years.loc[years].reset_index()
    df_last_22_years = df_last_22_years.astype({"Year":"string", "Total":"float64"})

    annual_change_total_chart = alt.Chart(df_last_22_years).mark_line(point=True).encode(
        x=alt.X("Year:T", timeUnit="year", title="Year"),
        y="Total:Q"
    )

    st.altair_chart(annual_change_total_chart, use_container_width=True)

except BaseException as e:
    raise e

st.markdown("### Emissão total por ano por categoria")
st.markdown("A emissão de **carvão** (Coal), **gás** (Gas) e **óleo** (Oil), as três maiores fontes de emissão, diminuíram durante os anos de 2008 (crise financeira mundial) e 2019 (Covid-19) \
    Apesar disso, a tendência continua de crescimento nos últimos anos")

try:

    annual_emission_per_category = alt.Chart(emission_per_category()).mark_line(point=True).encode(
        x=alt.X("Year:T", timeUnit="year", title="Year"),
        y="Total:Q",
        color="Category"
    )

    st.altair_chart(annual_emission_per_category, use_container_width=True)

except BaseException as e:
    raise e

row_4_1, row_4_2 = st.columns([2.5,1])

with row_4_2:
    try:

        line_break()
        st.text("\n")
        st.text("\n")
        st.text("\n")

        emission_source = st.selectbox(
            ':pick: **Selecione uma fonte de emissão**',
            ("Total", "Coal", "Oil", "Gas", "Cement", "Flaring", "Other", "Per Capita")
        )

        st.write("Você selecionou:", emission_source)

        df_ranked = rank_by_column(column_to_rank_by=emission_source)
        df_ranked = df_ranked[~df_ranked.loc[:,emission_source].isnull()]
        df_ranked = df_ranked.astype({"Rank":"int64", emission_source:"float64", "Year":"string", "Country":"string"})

        countries_rank = st.multiselect(
        ":earth_americas: **Selecione os países**", list(read_CO2_data().Country.unique()), ["USA", "China", "Russia",
        "Germany", "United Kingdom", "Japan"])

        df_ranked = df_ranked[df_ranked.Country.isin(countries_rank)].sort_values(by=["Year", "Rank"], ascending=(True, False))

        if not countries_rank:
            st.error("Selecione um país, pelo menos.")

    except BaseException as e:
        raise e

with row_4_1:
    try:
        st.markdown("# Ranking de países")
        st.markdown("Selecione a **fonte de emissão** e o **país** no filtro ao lado para verificar o rank dos países durante os anos de 2000-2020.\n\
              \n**Emissões per capita** pode corrigir sua visão sobre o número de emissões por país.\n \
                Por exemplo, China lidera o número de emissões total há 15 anos, porém, em emissões per capita USA é o líder.")

        countries_rank_chart = alt.Chart(df_ranked).mark_line(point=True).encode(
            x=alt.X("Year:T", timeUnit="year", title="Year"),
            y=alt.Y("Rank:Q",
                    sort="descending",
                    scale=alt.Scale(domain=[1,df_ranked.Rank.max()]),
                    axis=alt.Axis(values=[i for i in range(1,df_ranked.Rank.max())], tickMinStep=1)),
            color="Country"
        ).properties(
                        title="Ranking dos países",
                        width=800,
                        height=400
                    )

        st.altair_chart(countries_rank_chart, use_container_width=True)

    except BaseException as e:
        raise e

st.markdown("# Grupos de Países")
st.markdown("Os 231 países foram divididos em 5 grupos de acordo com o coeficiente de determinação $R^{2}$ resultante do ajuste de um modelo \
    de Regressão Linear entre os anos e o total de emissões.")
st.markdown("Selecione ao lado o **Grupo de países** que deseja observar.")

row_5_1, row_5_2 = st.columns([4,1])

with row_5_2:

    group_box = st.selectbox(
    ':factory: **Grupo**',
    ("All groups", "Group A", "Group B", "Group C", "Group D", "Group E")
    )

    if group_box == "All groups":
        lookup_data = data_with_coords_r2_score()
    else:
        lookup_data = data_with_coords_r2_score().set_index("Group").loc[group_box].reset_index()

    st.markdown("#### Grupo A")
    st.markdown("Esse grupo apresentou uma tendência clara de redução de emissões ao longo dos anos de 2000-2021")
    st.markdown("#### Grupo B")
    st.markdown("Esse grupo apresentou uma tendência não muito expressiva de redução de emissões ao longo dos anos de 2000-2021")
    st.markdown("#### Grupo C")
    st.markdown("Esse grupo não apresentou uma tendência clara de redução ou aumento de emissões ao longo dos anos de 2000-2021")
    st.markdown("#### Grupo D")
    st.markdown("Esse grupo apresentou uma tendência de aumento não muito expressiva de emissões ao longo dos anos de 2000-2021")
    st.markdown("#### Grupo E")
    st.markdown("Esse grupo apresentou uma tendência clara de aumento de emissões ao longo dos anos de 2000-2021")

with row_5_1:

    from vega_datasets import data

    countries = alt.topo_feature(data.world_110m.url, "countries")

    background = alt.Chart(countries).mark_geoshape(fill='white')

    foreground = alt.Chart(countries).mark_geoshape(
        stroke='black'
    ).encode(
        color=alt.Color(
            "Group:O", scale=alt.Scale(scheme="lightgreyred"), legend=None,
        ),
        tooltip=[
            alt.Tooltip("Country:N", title="Country"),
            alt.Tooltip("r2_score:Q", title="R2 Score"),
            alt.Tooltip("Group:O", title="Group"),
        ],
    ).transform_lookup(
        lookup="id",
        from_=alt.LookupData(lookup_data, "id", ["Group", "Country", "r2_score"])
    )

    final_map = (
    (background + foreground)
    .configure_view(strokeWidth=0)
    .properties(width=1200, height=800)
    .project("naturalEarth1")

    )

    st.altair_chart(final_map, use_container_width=True)


from django.test import TestCase

from hc.api.models import Check

class StatusTestCase(TestCase):

    def test_it_works(self):
        check = Check()
        check.save()

        r = self.client.get("/status/%s/" % check.code)
        self.assertContains(r, "last_ping", status_code=200)

    def test_it_handles_bad_uuid(self):
        r = self.client.get("/status/not-uuid/")
        assert r.status_code == 400


import gym
import gym_prescan as gp

# env = gym.make('prescan-without-matlabengine-v0',verbose=True,host='172.21.217.140')
env_dict = {
    'id':       'prescan-without-matlabengine-v0',
    'verbose':  True,
    'host':     '172.21.217.140',
    # 'delay':    2
    'nget':       500
}
env = gym.make(**env_dict)
for _ in range(3):
    env.reset()
    for i in range(3000):
        env.render()
        # env.step(env.action_space.sample()) # take a random action
        n = 200
        a = 3
        if i%n == 0:
            if i%(2*n) == 0:
                a = 0
            else:
                a = 1
        s,r,done, _ = env.step(a)
        if done:
            env.reset()
    # env.close()

    print('+++++++++++++++++++++++++++++++++++++')
# env.reset()
# env.step(5)


import json
import os, os.path
import errno
from pathlib import Path

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def safe_open_w(path):
    mkdir_p(os.path.dirname(path))
    return open(path, 'w')

def write_json(path, content, sorting=False):
    path = get_main_path() + path
    with safe_open_w(path) as file:
        json.dump(content, file, indent=4, sort_keys=sorting)

def read_json(path):
    path = get_main_path() + path
    try:
        with open(path, "r") as file:
            content = json.load(file)
        return content
    except FileNotFoundError:
        print(f"File at the path {path} was not found")
        return False

def get_main_path():
    path = Path(__file__).parent
    while os.path.basename(os.path.normpath(path)) != "GoblinsCaverns":
        path = path.parent
    return str(path)


from difflib import SequenceMatcher
import pandas as pd
from itertools import combinations

#Dupicate Delete - Deletes similar strings in a list of strings.
#yenilistemiz = remove_duplicatish(list)
def remove_duplicatish(alist):
    list_remove = []
    alist = list(set(alist))
    listcomb = list(combinations(alist, 2))
    for birtuple in listcomb:
        item1 = birtuple[0]
        item2 = birtuple[1]
        if SequenceMatcher(None, item1.lower(), item2.lower()).ratio() > 0.6: #change this ratio
            list_remove.append(item2)
    list_difference = list(set(list_remove))
    list_final = [x for x in alist if x not in list_difference]
    return list_final

#Another duplicate deleter simple is better.
from difflib import SequenceMatcher
liste = ["PARALAR", "paralar", "kapıcı", "parasız", "kapı", "geldiğin yere geri dön", "geldiğin yere geri dönme", "Özdilek Park Gucci", "Özdilek Park 2.Kat Gucci"]
def eliminate_duplicates(liste, ratio = 0.8):
    newlist= []
    for item in liste:
        for itemcompare in newlist:
            if SequenceMatcher(None, item.lower(), itemcompare.lower()).ratio() > ratio:
                break
        else:
            newlist.append(item)
    return newlist
print(eliminate_duplicates(liste))


# importing libraries and functions
from fastapi import APIRouter, Request
import re
import json
import skops.io as sio
import pandas as pd

from utils import sklearn

# instantiating the router
router = APIRouter(prefix='/api')

# index test route
@router.get('/')
async def get_index():
    return 'OK'

# student analysis route
@router.post('/analyze_student')
async def post_analyze_student(request: Request):
    request_content = await request.json()
    X = pd.DataFrame.from_dict(request_content).T

    X_transformed = sklearn.PREPROCESSOR.transform(X)
    X_transformed = pd.DataFrame(X_transformed, columns=[re.sub(r'.+__', '', item) for item in sklearn.PREPROCESSOR.get_feature_names_out()], index=X.index)
    X_transformed = X_transformed.drop(columns=['PARTICIPANTE_TP_SEXO_M'])

    preds = {
        'knn_classifier': sklearn.KNN_CLASSIFIER.predict(X_transformed),
        'knn_regressor': sklearn.KNN_REGRESSOR.predict(X_transformed),
        'linear_regression': sklearn.LINEAR_REGRESSION.predict(X_transformed),
        'logistic_regression': sklearn.LOGISTIC_REGRESSION.predict(X_transformed),
        'naive_bayes': sklearn.NAIVE_BAYES.predict(X_transformed),
        'svm_classifier': sklearn.SVM_CLASSIFIER.predict(X_transformed),
        'svm_regressor': sklearn.SVM_REGRESSOR.predict(X_transformed),
        'randomforest_classifier': sklearn.RANDOMFOREST_CLASSIFIER.predict(X_transformed),
        'randomforest_regressor': sklearn.RANDOMFOREST_REGRESSOR.predict(X_transformed),
    }
    preds = pd.DataFrame(preds, index=X_transformed.index)

    classifiers = ['knn_classifier', 'logistic_regression', 'naive_bayes', 'svm_classifier', 'randomforest_classifier']
    regressors = ['knn_regressor', 'linear_regression', 'svm_regressor', 'randomforest_regressor']

    preds = preds.apply(lambda column: round(column, 2), axis=0)

    preds['classifiers_summary'] = preds.apply(lambda row: int(pd.Series([row[classifier] for classifier in classifiers]).mode()), axis=1)
    preds['regressors_summary'] = preds.apply(lambda row: round(pd.Series([row[regressor] for regressor in regressors]).mean(), 2), axis=1)

    return json.loads(preds.to_json(orient='index'))


from mxnet import gluon
from mxnet import ndarray as nd
from mxnet import autograd as ag
import datetime
#%matplotlib inline
import matplotlib as mpl
mpl.rcParams['figure.dpi'] = 120
import matplotlib.pyplot as plt
import numpy as np

def train(net, train_data, valid_data, ctx, num_epoches, softmax_cross_entropy, optimizer='adam',
          lr=0.01, lr_decay=0.1, lr_period=50, momentum=0.9, weight_decay=0,
          cost_period=10, print_cost=False):
    if optimizer == 'momentum':
        trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr,
                                                              'momentum': momentum,
                                                              'wd': weight_decay})
    elif optimizer == 'adam':
        trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': lr,
                                                               'wd': weight_decay})

    train_costs = []
    valid_costs = []
    v_loss_train = 0
    n_iter_train = 0
#     v_loss_valid = 0
#     n_iter_valid = 0
    for epoch in range(num_epoches):
        pre_time = datetime.datetime.now()
        train_acc = 0
#         train_loss = 0
        if (epoch+1) % lr_period == 0:
            trainer.set_learning_rate(trainer.learning_rate * lr_decay)
        for data, label in train_data:
            data = data.as_in_context(ctx)
            label = label.astype('float32').as_in_context(ctx)
            with ag.record():
                output = net(data)
                loss = softmax_cross_entropy(output, label)
            loss.backward()
            #trainer.step(batch_size)
            trainer.step(len(data))
            train_acc += nd.mean(output.argmax(axis=1) == label).asscalar()
#             train_loss += nd.mean(loss).asscalar()
            cur_loss = nd.mean(loss).asscalar()
            v_loss_train = 0.9 * v_loss_train + 0.1 * cur_loss
            n_iter_train += 1
            corr_loss_train = v_loss_train / (1 - pow(0.9, n_iter_train))

        cur_time = datetime.datetime.now()
        h, remainder = divmod((cur_time - pre_time).seconds, 3600)
        m, s = divmod(remainder, 60)
        time_str = 'Time %02d:%02d:%02d, ' % (h, m, s)

        if valid_data is not None:
            valid_acc = 0
            valid_loss = 0
            for data, label in valid_data:
                data = data.as_in_context(ctx)
                label = label.astype('float32').as_in_context(ctx)
                output = net(data)
                loss = softmax_cross_entropy(output, label)
                valid_acc += nd.mean(output.argmax(axis=1) == label).asscalar()
#                 cur_loss = nd.mean(loss).asscalar()
#                 v_loss_valid = 0.9 * v_loss_valid + 0.1 * cur_loss
#                 n_iter_valid += 1
#                 corr_loss_valid = v_loss_valid / (1 - pow(0.9, n_iter_valid))
                valid_loss += nd.mean(loss).asscalar()
            epoch_str = 'Epoch %d, Train_loss: %s, Train_acc: %s, Valid_acc: %s, ' % (epoch+1,
                                                                                corr_loss_train,
                                                                                train_acc/len(train_data),
                                                                                valid_acc/len(valid_data))
        else:
            epoch_str = 'Epoch %d, Train_loss: %s, Train_acc: %s, ' % (epoch+1,
                                                                     corr_loss_train,
                                                                     train_acc/len(train_data))
        if print_cost and (epoch+1) % cost_period == 0:
            train_costs.append(corr_loss_train)
#             train_costs.append(train_loss/len(train_data))
            valid_costs.append(valid_loss/len(valid_data))

        print(epoch_str + time_str + 'lr: %f' % trainer.learning_rate)

    if print_cost:
        x_axis = np.linspace(0, num_epoches, len(train_costs), endpoint=True)
        l1, = plt.semilogy(x_axis, train_costs)
        l2, = plt.semilogy(x_axis, valid_costs)
        plt.xlabel('epoch')
        plt.ylabel('loss')
        plt.legend([l1, l2], ['train', 'valid'], loc='upper right')
        plt.show()


import cv2
import numpy as np
import os
import matplotlib.pyplot as plt
import Perspective as Perspective
import MyFilter as Filter
import cluster_line as Cluster
import HoughLine as HoughLine
import find_endpoint as fe
from XML_Creater import XML
from Classifier_v04 import Classifier
import Classifier_v04
import time

################################################## 待调参数 #######################################################
# 数据集路径设置
# path = 'G:\JuntaWu\lane_detection\MyLane_20171103\image\dataSet'
path= 'G:\JuntaWu\lane_detection\MyLane_20171103\image\\test'
# 生成的 xml 文件保存路径
xml_path = 'G:\JuntaWu\lane_detection\MyLane_20171103\MyLaneTrack_v08.7\TSD-Lane-Result'

# canny 滤波参数
low_threshold = 50
high_threshold = 80

# hsv 颜色阈值空间，需与Classifier.py 颜色阈值参数一致
white_low_thres = Classifier_v04.white_low_thres
white_high_thres = Classifier_v04.white_high_thres
yellow_low_thres = Classifier_v04.yellow_low_thres
yellow_high_thres = Classifier_v04.yellow_high_thres

# 检测车道线数
lane_num = 4
###################################################################################################################

folders = os.listdir(path)
for folder in folders:
    # 提示信息与计时
    print('Processing ' + folder + ' :')
    start_time = time.time() # 计时

    image_path = os.path.join(path, folder)
    image_files = os.listdir(image_path)
    # 创建xml文件，存储车道线信息
    # xml_creater = XML('TSD-Lane-%05d-Result.xml' % data_index )
    xml_creater = XML(folder + '-Result.xml')
    # 当前图片帧号
    frameIndex = 0
    # 存储上一帧检测结果，初始化
    previous_extend_lane = np.array([[1194, 1024,  574,  295],  [1194, 900,  574,  250] ])
    for file in image_files:
        try:
            img = cv2.imread(image_path + '\\' + file)
            image = img.copy()
            #######################################################################################################################
            # 预处理：滤波+边缘检测
            median = cv2.medianBlur(img, 5)

            img_1 = (Filter.GaborFilter(median, 5, 0, 3, 5, 1, 0)).astype(np.uint8)  # 利用gabor滤波器提取方向边缘信息
            img_2 = (Filter.GaborFilter(median, 5, 25, 3, 5, 1, 0)).astype(np.uint8)
            img_3 = (Filter.GaborFilter(median, 5, 50, 3, 5, 1, 0)).astype(np.uint8)
            img_4 = (Filter.GaborFilter(median, 5, 90, 3, 5, 1, 0)).astype(np.uint8)
            gabor = np.abs((img_1 + +img_2 + img_3 + img_4)).astype(np.uint8)

            gabor_edges = Filter.CannyFilter(gabor, low_threshold, high_threshold)
            end_point = fe.find_endpoint(img, gabor_edges)
            # cv2.imshow("gabor_result", gabor_edges)

            ROI = np.array([[(0,end_point[1]),(img.shape[1],end_point[1]),(img.shape[1],img.shape[0]),(0,img.shape[0])]],dtype=np.int32)
            roi = Perspective.region_of_interest(gabor_edges, ROI)
            #########################################################################################################################

            ###############################################################################################
            # 获得颜色、虚实检测所需的canny_edges
            # 先进行一轮颜色滤波
            img_color = Filter.Color_Filter(image, (np.array(white_low_thres), np.array(white_high_thres)),
                                            (np.array(yellow_low_thres), np.array(yellow_high_thres)))
            img_gray = cv2.cvtColor(img_color, cv2.COLOR_BGR2GRAY)
            _, img_thres = cv2.threshold(img_gray, 127, 255, cv2.THRESH_BINARY)
            # cv2.imshow("thres", img_thres)
            ###############################################################################################

            ################################################################################################################################
            # 检测当前帧车道线，检测不到时异常处理
            # 车道检测：直线检测+斜率筛选+聚类+车道挑选
            hough_lines = HoughLine.HoughLineTransform(roi, 'p')
            lines = Cluster.slope_check(hough_lines, 0.2, end_point, img.shape[1], 0.2)
            polars = (Cluster.line_change(lines)).T
            labels, unique_labels = Cluster.dbscan(lines, polars, end_point, 0.1, 3, 280,2)  # epss=0.1, MinPts=3, epss2=350, MinPts2=2)
            lane = Cluster.select_lines(img, lines, polars, labels, unique_labels, end_point, cc=0.2, n_max = lane_num)

            hough = np.zeros_like(img)
            # for line in hough_lines:
            #     cv2.line(hough, (int(line[0, 0]), int(line[0, 1])), (int(line[0, 2]), int(line[0, 3])), [0, 0, 255], 1)
            # cv2.imshow('hough', hough)

            slope_check = np.zeros_like(img)
            # for line in lines:
            #     cv2.line(slope_check, (int(line[0, 0]), int(line[0, 1])), (int(line[0, 2]), int(line[0, 3])), [0, 0, 255], 1)
            # cv2.imshow('slope_check', slope_check)

            ########################################### 显示检测直线的效果 ##############################################
            colors2 = [[0, 0, 255], [0, 255, 0], [255, 0, 0], [0, 0, 0], [255, 255, 0], [255, 0, 255], [0, 255, 255],
                       [255, 255, 255]]  # 红、绿、蓝、黑、青、粉、黄、白   直线颜色按得分高到低从前往后排列
            y1 = img.shape[0]
            y2 = end_point[1]  # 取消失点下方的点
            extend_lane = []     # 延长后的车道
            for line, col in zip(lane, colors2):
                if line[0] == line[2]:
                    x1 = line[0]
                    x2 = line[0]
                else:
                    line_m = (line[1] - line[3]) / (line[0] - line[2])
                    line_b = line[3] - line_m * line[2]
                    x1 = int((y1 - line_b) / line_m)
                    x2 = int((y2 - line_b) / line_m)
                # cv2.line(img, (x1, y1), (x2, y2), col, 10)
                extend_lane.append([x1, y1, x2, y2])
            extend_lane = np.array(extend_lane)
            # cv2.imshow('img', img)
            #############################################################################################################

            # 更新上一帧检测到的车道线结果，以备当前帧检测不到车道线情况
            previous_extend_lane = extend_lane

            # 根据 x1 的数值大小，对检测到的车道线从左到右排序
            # 存储从左到右排好序的车道线端点
            lane_points= np.empty( shape=[0, extend_lane.shape[1] ] )  # lane_points：按x1从小到大排序的车道线
            iter_num = extend_lane.shape[0]
            for index in range( iter_num ):
                min_index = extend_lane[:, 0].argmin()
                lane_points = np.vstack( (lane_points, extend_lane[ min_index, :]) )
                extend_lane = np.delete(extend_lane, min_index, axis=0)
            # print(lane_points)

            # 对检测到的每条车道线进行颜色,类型判断,并将结果写入xml文件
            targetNumber = lane_points.shape[0]
            xml_creater.create_targetNumber(frameIndex, targetNumber)
            targetIndex = 0  # 目标车道线
            for i in range(targetNumber):
                # 检测不到时，异常如何处理 ？？？
                lane_position, color, type = Classifier(image, img_thres, img_thres, ( int(lane_points[i, 0]), int(lane_points[i, 1])), \
                                                        ( int(lane_points[i, 2]), int(lane_points[i, 3]) ), i+1, targetNumber )
                xml_creater.create_target(frameIndex, targetIndex, color, type, lane_position)
                targetIndex += 1

                ############################## 显示检测虚实、黄白的效果 ########################################
                # for point in lane_position:
                #     cv2.circle(image,(point[0], point[1] ), 3, [0, 0, 255], -1)
                if color == True:
                    if type == True:
                        col = [100,100,100] # 白虚线  灰色
                    else:
                        col = [0,0,0] # 白实线  黑色
                else:
                    if type == True:
                        col = [255,255,0] # 黄虚线 青色
                    else:
                        col = [0,255,0] # 黄实线  绿色
                cv2.line(image, ( int(lane_points[i, 0]), int(lane_points[i, 1])), \
                                                        ( int(lane_points[i, 2]), int(lane_points[i, 3]) ), col,7)
                #####################################################################################################
        except:
            pass
            # extend_lane = previous_extend_lane
            # # if(extend_lane)
            # lane_points = np.empty(shape=[0, extend_lane.shape[1]])  # lane_points：按x1从小到大排序的车道线
            # iter_num = extend_lane.shape[0]
            # for index in range(iter_num):
            #     min_index = extend_lane[:, 0].argmin()
            #     lane_points = np.vstack((lane_points, extend_lane[min_index, :]))
            #     extend_lane = np.delete(extend_lane, min_index, axis=0)
            # # print(lane_points)
            #
            # # 对检测到的每条车道线进行颜色,类型判断,并将结果写入xml文件
            # targetNumber = lane_points.shape[0]
            # xml_creater.create_targetNumber(frameIndex, targetNumber)
            # targetIndex = 0  # 目标车道线
            # for i in range(targetNumber):
            #     # 检测不到时，异常如何处理 ？？？
            #     lane_position, color, type = Classifier(image, img_thres, img_thres,
            #                                             (int(lane_points[i, 0]), int(lane_points[i, 1])), \
            #                                             (int(lane_points[i, 2]), int(lane_points[i, 3])), i + 1,
            #                                             targetNumber)
            #     xml_creater.create_target(frameIndex, targetIndex, color, type, lane_position)
            #     targetIndex += 1
            #
            #     ############################## 显示检测虚实、黄白的效果 ########################################
            #     # for point in lane_position:
            #     #     cv2.circle(image,(point[0], point[1] ), 3, [0, 0, 255], -1)
            #     if color == True:
            #         if type == True:
            #             col = [100, 100, 100]  # 白虚线  灰色
            #         else:
            #             col = [0, 0, 0]  # 白实线  黑色
            #     else:
            #         if type == True:
            #             col = [255, 255, 0]  # 黄虚线 青色
            #         else:
            #             col = [0, 255, 0]  # 黄实线  绿色
            #     cv2.line(image, (int(lane_points[i, 0]), int(lane_points[i, 1])), \
            #              (int(lane_points[i, 2]), int(lane_points[i, 3])), col, 7)
            #     #####################################################################################################
        ##############################################################################################################################
        frameIndex += 1

        # 显示检测效果
        # cv2.imshow("annotated_image", image)
        # cv2.waitKey(1)

    # 关闭xml文件
    xml_creater.close()

    ########################## 修改XML格式 #############################
    xml_new = open(xml_path + '\\' + folder + '-Result.xml', 'w', encoding='gb2312')
    with open( folder + '-Result.xml', 'r', encoding='utf-8') as f:
        xml_lines = f.readlines()
        count = 0
        for xml_line in xml_lines:
            if count == 0:
                count += 1
                xml_new.write(xml_line)
                continue
            if '.' in xml_line:
                new_line = ''
                fragments = xml_line.split('.')
                for fragment in fragments:
                    new_line += fragment
                xml_new.write(new_line)
                continue
            count += 1
            xml_new.write(xml_line)
    ########################################################################

    # 计时
    end_time = time.time()
    print(end_time - start_time)


import unittest
import matplotlib.pyplot as plt
import numpy as np
import cv2
import modulos.Datos as dt

class Plt(unittest.TestCase):
    def setUp(self) -> None:
        print('--> Iniciando test')
        datosx = [1, 2, 3, 4, 5, 6]
        datosy = [2, 4, 6, 7, 9, 2]
        self.imagen = cv2.imread(r'C:\Google Drive\Cursos\Python_OpenCV\Deskew\Docs\Texto.png', 1)
        self.datos = dt.Datos()

    def tearDown(self) -> None:
        print('--> Test terminado')

    def test_figura(self):
        fig = plt.Figure(figsize=(5, 5), dpi=100)
        plt_axes = fig.add_subplot(111)
        plt_axes.plot([1, 2, 3, 4, 5, 6], [2, 4, 6, 7, 9, 2])
        plt.show()
        y = np.random.rand(100)
        x = range(0, 100, 1)
        plt_axes.plot(x, y)
        plt.show()
        cv2.imshow('Ventana', self.imagen)
        cv2.waitKey()

    def test_matrices(self):
        self.datos.actualizar_figura_plt()

class FFT(unittest.TestCase):

    def setUp(self) -> None:
        print('--> Iniciando test')

    def tearDown(self) -> None:
        print('--> Test terminado')

    def test_fft(self):
        dti = 0.001
        t = np.arange(0, 1, dti)
        f = np.sin(2 * np.pi * 50 * t) + np.sin(2 * np.pi * 120 * t)
        f_clean = f
        f = f + 2.5 * np.random.randn(len(t))
        # plt.plot(t, f)
        # plt.plot(t, f_clean)
        # plt.show()

        n = len(t)
        fhat = np.fft.fft(f, n)
        PSD = fhat * np.conj(fhat) / n
        plt.plot(PSD[:n // 2])
        plt.show()

if __name__ == '__main__':
    unittest.main()


"""Chroma Auto-retrieval Pack."""

from typing import Any, Dict, List, Optional

from llama_index.indices.vector_store import VectorStoreIndex
from llama_index.indices.vector_store.retrievers import (
    VectorIndexAutoRetriever,
)
from llama_index.llama_pack.base import BaseLlamaPack
from llama_index.query_engine import RetrieverQueryEngine
from llama_index.schema import TextNode
from llama_index.storage.storage_context import StorageContext
from llama_index.vector_stores.chroma import ChromaVectorStore
from llama_index.vector_stores.types import VectorStoreInfo

class ChromaAutoretrievalPack(BaseLlamaPack):
    """Chroma auto-retrieval pack."""

    def __init__(
        self,
        collection_name: str,
        vector_store_info: VectorStoreInfo,
        nodes: Optional[List[TextNode]] = None,
        client: Optional[Any] = None,
        **kwargs: Any,
    ) -> None:
        """Init params."""
        import chromadb

        chroma_client = client or chromadb.EphemeralClient()
        chroma_collection = chroma_client.get_or_create_collection(collection_name)

        self._vector_store = ChromaVectorStore(chroma_collection=chroma_collection)

        if nodes is not None:
            self._storage_context = StorageContext.from_defaults(
                vector_store=self._vector_store
            )
            self._index = VectorStoreIndex(
                nodes, storage_context=self._storage_context, **kwargs
            )
        else:
            self._index = VectorStoreIndex.from_vector_store(
                self._vector_store, **kwargs
            )
            self._storage_context = self._index.storage_context

        self.retriever = VectorIndexAutoRetriever(
            self._index, vector_store_info=vector_store_info
        )
        self.query_engine = RetrieverQueryEngine(self.retriever)

    def get_modules(self) -> Dict[str, Any]:
        """Get modules."""
        return {
            "vector_store": self._vector_store,
            "storage_context": self._storage_context,
            "index": self._index,
            "retriever": self.retriever,
            "query_engine": self.query_engine,
        }

    def retrieve(self, query_str: str) -> Any:
        """Retrieve."""
        return self.retriever.retrieve(query_str)

    def run(self, *args: Any, **kwargs: Any) -> Any:
        """Run the pipeline."""
        return self.query_engine.query(*args, **kwargs)