import warnings
from collections import Iterable, OrderedDict
from functools import partial

import torch
import torchvision
from torch import nn

from neuralpredictors import regularizers
from neuralpredictors.layers.activations import AdaptiveELU
from neuralpredictors.layers.affine import Bias2DLayer, Scale2DLayer
from neuralpredictors.layers.attention import AttentionConv
from neuralpredictors.layers.conv import DepthSeparableConv2d

from .base import Core

# logger = logging.getLogger(__name__)

class Stacked2dCore(Core, nn.Module):
    """
    An instantiation of the Core base class. Made up of layers layers of nn.sequential modules.
    Allows for the flexible implementations of many different architectures, such as convolutional layers,
    or self-attention layers.
    """

    def __init__(
        self,
        input_channels,
        hidden_channels,
        input_kern,
        hidden_kern,
        layers=3,
        gamma_hidden=0,
        gamma_input=0.0,
        skip=0,
        stride=1,
        final_nonlinearity=True,
        scale=None,
        elu_shift=(0, 0),
        bias=True,
        momentum=0.1,
        pad_input=True,
        hidden_padding=None,
        batch_norm=True,
        batch_norm_scale=True,
        independent_bn_bias=True,
        hidden_dilation=1,
        laplace_padding=0,
        input_regularizer="LaplaceL2",
        stack=None,
        use_avg_reg=True,
        depth_separable=False,
        attention_conv=False,
        linear=False,
    ):
        """
        Args:
            input_channels:     Integer, number of input channels as in
            hidden_channels:    Number of hidden channels (i.e feature maps) in each hidden layer
            input_kern:     kernel size of the first layer (i.e. the input layer)
            hidden_kern:    kernel size of each hidden layer's kernel
            layers:         number of layers
            gamma_hidden:   regularizer factor for group sparsity
            gamma_input:    regularizer factor for the input weights (default: LaplaceL2, see neuralpredictors.regularizers)
            skip:           Adds a skip connection
            stride:         stride of the 2d conv layer.
            final_nonlinearity: Boolean, if true, appends an ELU layer after the last BatchNorm (if BN=True)
            elu_shift: a tuple to shift the elu in the following way: Elu(x - elu_xshift) + elu_yshift
            bias:           Adds a bias layer.
            momentum:       momentum in the batchnorm layer.
            pad_input:      Boolean, if True, applies zero padding to all convolutions
            hidden_padding: int or list of int. Padding for hidden layers. Note that this will apply to all the layers
                            except the first (input) layer.
            batch_norm:     Boolean, if True appends a BN layer after each convolutional layer
            batch_norm_scale: If True, a scaling factor after BN will be learned.
            independent_bn_bias:    If False, will allow for scaling the batch norm, so that batchnorm
                                    and bias can both be true. Defaults to True.
            hidden_dilation:    If set to > 1, will apply dilated convs for all hidden layers
            laplace_padding: Padding size for the laplace convolution. If padding = None, it defaults to half of
                the kernel size (recommended). Setting Padding to 0 is not recommended and leads to artefacts,
                zero is the default however to recreate backwards compatibility.
            input_regularizer:  String that must match one of the regularizers in ..regularizers
            stack:        Int or iterable. Selects which layers of the core should be stacked for the readout.
                            default value will stack all layers on top of each other.
                            Implemented as layers_to_stack = layers[stack:]. thus:
                                stack = -1 will only select the last layer as the readout layer.
                                stack of -2 will read out from the last two layers.
                                And stack of 1 will read out from layer 1 (0 indexed) until the last layer.
            use_avg_reg:    bool. Whether to use the averaged value of regularizer(s) or the summed.
            depth_separable: Boolean, if True, uses depth-separable convolutions in all layers after the first one.
            attention_conv: Boolean, if True, uses self-attention instead of convolution for all layers after the first one.
            linear:         Boolean, if True, removes all nonlinearities

            To enable learning batch_norms bias and scale independently, the arguments bias, batch_norm and batch_norm_scale
            work together: By default, all are true. In this case there won't be a bias learned in the convolutional layer, but
            batch_norm will learn both its bias and scale. If batch_norm is false, but bias true, a bias will be learned in the
            convolutional layer. If batch_norm and bias are true, but batch_norm_scale is false, batch_norm won't have learnable
            parameters and a BiasLayer will be added after the batch_norm layer.
        """

        if depth_separable and attention_conv:
            raise ValueError("depth_separable and attention_conv can not both be true")

        super().__init__()
        regularizer_config = (
            dict(padding=laplace_padding, kernel=input_kern)
            if input_regularizer == "GaussianLaplaceL2"
            else dict(padding=laplace_padding)
        )
        self._input_weights_regularizer = regularizers.__dict__[input_regularizer](**regularizer_config)
        self.num_layers = layers
        self.gamma_input = gamma_input
        self.gamma_hidden = gamma_hidden
        self.input_channels = input_channels
        self.hidden_channels = hidden_channels
        self.skip = skip
        self.stride = stride
        self.use_avg_reg = use_avg_reg
        if use_avg_reg:
            warnings.warn("The averaged value of regularizer will be used.", UserWarning)
        self.hidden_padding = hidden_padding
        self.input_kern = input_kern
        self.hidden_kern = hidden_kern
        self.laplace_padding = laplace_padding
        self.hidden_dilation = hidden_dilation
        self.final_nonlinearity = final_nonlinearity
        self.elu_xshift, self.elu_yshift = elu_shift
        self.bias = bias
        self.momentum = momentum
        self.pad_input = pad_input
        self.batch_norm = batch_norm
        self.batch_norm_scale = batch_norm_scale
        self.independent_bn_bias = independent_bn_bias
        if stack is None:
            self.stack = range(self.num_layers+1)
        else:
            self.stack = [*range(self.num_layers+1)[stack:]] if isinstance(stack, int) else stack
        self.linear = linear

        if depth_separable:
            self.conv_layer_name = "ds_conv"
            self.ConvLayer = DepthSeparableConv2d
            self.ignore_group_sparsity = True
        elif attention_conv:
            # TODO: check if name attention_conv is backwards compatible
            self.conv_layer_name = "attention_conv"
            self.ConvLayer = self.AttentionConvWrapper
            self.ignore_group_sparsity = True
        else:
            self.conv_layer_name = "conv"
            self.ConvLayer = nn.Conv2d
            self.ignore_group_sparsity = False

        if (self.ignore_group_sparsity) and (gamma_hidden > 0):
            warnings.warn(
                "group sparsity can not be calculated for the requested conv type. Hidden channels will not be regularized and gamma_hidden is ignored."
            )
        self.set_batchnorm_type()
        self.features = nn.Sequential()
        self.add_scale_layer()
        self.add_first_layer()
        self.add_subsequent_layers()
        self.initialize()

    def set_batchnorm_type(self):
        self.batchnorm_layer_cls = nn.BatchNorm2d
        self.bias_layer_cls = Bias2DLayer
        self.scale_layer_cls = Scale2DLayer

    def add_bn_layer(self, layer):
        if self.batch_norm:
            if self.independent_bn_bias:
                layer["norm"] = self.batchnorm_layer_cls(self.hidden_channels, momentum=self.momentum)
            else:
                layer["norm"] = self.batchnorm_layer_cls(
                    self.hidden_channels, momentum=self.momentum, affine=self.bias and self.batch_norm_scale
                )
                if self.bias:
                    if not self.batch_norm_scale:
                        layer["bias"] = self.bias_layer_cls(self.hidden_channels)
                elif self.batch_norm_scale:
                    layer["scale"] = self.scale_layer_cls(self.hidden_channels)

    def add_activation(self, layer):
        if self.linear:
            return
        if len(self.features) < self.num_layers - 1 or self.final_nonlinearity:
            layer["nonlin"] = AdaptiveELU(self.elu_xshift, self.elu_yshift)

    def add_first_layer(self):
        layer = OrderedDict()
        layer["conv"] = nn.Conv2d(
            self.hidden_channels//4,
            self.hidden_channels,
            self.input_kern,
            padding=self.input_kern // 2 if self.pad_input else 0,
            bias=self.bias and not self.batch_norm,
        )
        layer["norm"] = nn.BatchNorm2d(self.hidden_channels, momentum=self.momentum)
        self.add_activation(layer)
        self.features.add_module("layer0", nn.Sequential(layer))

    def add_subsequent_layers(self):
        if not isinstance(self.hidden_kern, Iterable):
            self.hidden_kern = [self.hidden_kern] * (self.num_layers - 1)

        for l in range(1, self.num_layers):
            layer = OrderedDict()
            if self.hidden_padding is None:
                self.hidden_padding = ((self.hidden_kern[l - 1] - 1) * self.hidden_dilation + 1) // 2
            layer[self.conv_layer_name] = self.ConvLayer(
                in_channels=self.hidden_channels*2 if l > 1 else self.hidden_channels,
                out_channels=self.hidden_channels*2,
                kernel_size=self.hidden_kern[l - 1],
                stride=self.stride,
                padding=self.hidden_padding,
                dilation=self.hidden_dilation,
                bias=self.bias,
            )
            layer["norm"] = nn.BatchNorm2d(self.hidden_channels*2, momentum=self.momentum)
            self.add_activation(layer)
            self.features.add_module("layer{}".format(l), nn.Sequential(layer))

    def add_scale_layer(self):
        layer = OrderedDict()
        layer["scale"] = nn.Conv2d(
            in_channels=self.input_channels,
            out_channels=self.hidden_channels//4,
            kernel_size=4,
            stride=4,
            bias=self.bias)
        layer["norm"] = nn.BatchNorm2d(self.hidden_channels//4, momentum=self.momentum)
        self.add_activation(layer)
        self.features.add_module("layer-1", nn.Sequential(layer))

    class AttentionConvWrapper(AttentionConv):
        def __init__(self, dilation=None, **kwargs):
            """
            Helper class to make an attention conv layer accept input args of a pytorch.nn.Conv2d layer.
            Args:
                dilation: catches this argument from the input args, and ignores it
                **kwargs:
            """
            super().__init__(**kwargs)

    def forward(self, input_):
        ret = []
        for l, feat in enumerate(self.features):
            # print("call:", feat)
            do_skip = l >= 1 and self.skip > 1
            input_ = feat(input_ if not do_skip else torch.cat(ret[-min(self.skip, l) :], dim=1))
            # print("out shape:", input_.size())
            ret.append(input_)
        # print([e.size() for e in ret])
        # print([ret[-1].size()])
        # print((torch.cat([ret[-1]], dim=1)).size())
        return torch.cat([ret[ind] for ind in self.stack], dim=1)

    def laplace(self):
        """
        Laplace regularization for the filters of the first conv2d layer.
        """
        reg_scale = self._input_weights_regularizer(self.features[0].scale.weight, avg=self.use_avg_reg)
        reg_first_conv = self._input_weights_regularizer(self.features[1].conv.weight, avg=self.use_avg_reg)
        return reg_scale

    def group_sparsity(self):
        """
        Sparsity regularization on the filters of all the conv2d layers except the first one.
        """
        ret = 0
        if self.ignore_group_sparsity:
            return ret

        for feature in self.features[1:]:
            ret = ret + feature.conv.weight.pow(2).sum(3, keepdim=True).sum(2, keepdim=True).sqrt().mean()
        return ret / ((self.num_layers - 1) if self.num_layers > 1 else 1)

    def regularizer(self):
        return self.group_sparsity() * self.gamma_hidden + self.gamma_input * self.laplace()

    @property
    def outchannels(self):
        return len(self.features) * self.hidden_channels


from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
from __future__ import print_function
import os
import csv
from scoreMatrices.matricesPuntajes import calculateScoreMatrix
import matplotlib.pylab as plt
import numpy as np
from filterbank.scoreFiltering import applyTs2Means, leakyIntegrator
from splitRecording.recordingSplitter import findApsPeriod
from scenarios import getScenery, lengthContour

def getTemplate(apsType, fsp):
    o = []

    # fsp: sampling frequency of the STFT
    if ('cuckoo' == apsType):
        T = lengthContour['cuckoo']  # s
        N = int(fsp * T)
        o = [0] * N
        for n in range(0, int(0.07 * fsp)):
            o[n] = 1100
        for n in range(int(0.270 * fsp), int(0.400 * fsp)):
            o[n] = 900
    elif ('highchirp' == apsType):
        T = lengthContour['highchirp']  # s
        N = int(fsp * T)
        o = np.linspace(3000, 2000, N)
    elif ('lowchirp' == apsType):
        T = lengthContour['lowchirp']  # s
        N = int(fsp * T)
        o = np.linspace(1750, 950, N)
    else:
        print('Error: unknown aps type')
        exit()

    return o

def computerContourFromFilterbank(filepath, NFFT, fRange, df, kernelType, nHarmonics, alphaLambda, T, thresholdMethod,
                                  verbose=False):
    [D, f0s, fsp, ss] = calculateScoreMatrix(filepath, NFFT, fRange, df, kernelType, nHarmonics)

    # get the contour signal based on the scores of the score matrix
    dim = 1
    iWin = np.argmax(ss, dim)  # get the f0s indexes
    sMax = np.max(ss, dim)
    f0s = np.asarray(f0s)
    if len(iWin) == len(f0s):
        print('Error: length of array containing indexes of the best scores should correspond to the highest dimension')
        exit()

    # create the contour
    c = f0s[iWin]  # get the fundamental frequencies

    # use the static or dynamic thresholding?
    if thresholdMethod == 'fixed':
        sp = [alphaLambda] * len(c)
        if verbose:
            print('Fixed threshold applied')
    elif thresholdMethod == 'ts2means':
        # use TS2Means
        dt = 1.0 / fsp
        wMin = T
        wMax = 2.0 * T
        sp = applyTs2Means(sMax, dt, wMin, wMax)
        if verbose:
            print('Ts2means threshold applied')
    elif thresholdMethod == 'leakyInt':
        sp = leakyIntegrator(sMax, alphaLambda)
        if verbose:
            print('LeakyInt threshold applied')
    else:
        if verbose:
            print('Error: "%s"  is unknown' % thresholdMethod)
        exit()

    # apply the selected threshold
    for n in range(0, len(c)):
        c[n] = 0 if sMax[n] < sp[n] else c[n]

    # calculate the time axis
    t = np.multiply(1.0 / fsp, np.arange(1, len(c) + 1))

    # return musical contour
    nf0s = np.size(f0s)
    return nf0s, fsp, t, c

def initializeNullPenalization(length):
    j = 0.06 + 2 * 2 * np.arange(0, length) * 0.01
    w = np.append(np.asarray([0]), np.cumsum(j))  # append cero at the beggining in case all the contour is zeros
    wp = 1.0 / w[-1] * w
    return wp

def modifiedEuclideanDistance(x, o, wp):
    # wp: penalization function
    # are there more zeros than non-zeros entries? If so, score=0
    nnz = np.count_nonzero(x)  # no. non-zeros
    nz = np.size(x) - nnz  # no. zeros
    if (nnz < nz):
        return 0
    else:
        xi = float(np.size(o))
        l2 = np.sqrt(np.sum(np.power(o - x, 2)))
        diff = max(o) - min(o) + 0.0
        den = np.sqrt(xi) * diff
        k = nz  # number of zeros found in the contour
        d = max(0, 1.0 - l2 / den - wp[k])

        return d

def alertFilteringByPeriodicity(a, T, fsp, Q):
    '''
    T: APS period in seconds
    fsp: sampling frequency of the STFT
    q: number of periods to look behind
    '''
    # 0<Q?
    if Q < 2:
        return a

    G = np.size(a)
    L = int(T * fsp)  # lag
    epsilon = int(0.025 * L)
    ap = np.zeros([G, 1])

    # add zeros to 'a' to pass the q filter through the entire signal
    NZBeginning = (Q - 1) * L + epsilon
    zerosBeginning = [0] * NZBeginning
    zerosEnd = [0] * epsilon
    extA = np.concatenate([zerosBeginning, a, zerosEnd])

    # iterate through the signal and calculate the scores.
    # Epsilon works like a shifting for the Q period/event looking,
    # the epsilon who gather more energy is the preserved one
    for g in range(0, G):  # iterate through alert signal entries
        cIndex = g + NZBeginning
        cScore = 1.0
        for q in range(0, Q):  # iterate through period/eventsd
            iMin = cIndex - q * L - epsilon
            iMax = cIndex - q * L + epsilon + 1
            cScore *= np.max(extA[iMin:iMax])

        # asign the highest score
        ap[g] = cScore

    return ap

def loadMatrixFromCsv(filepath):
    flatM = []
    with open(filepath, 'rb') as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        for row in reader:
            for value in row:
                flatM.append(float(value))
    N = int(np.sqrt(len(flatM)))
    return np.reshape(flatM, [N, N])

def mahalanobisDistance(contour, template, covarianceInverseMatrix):
    # mCovInv: matriz de covarianza inversa
    vDiff = np.subtract(contour, template)
    r2 = np.dot(np.dot(np.transpose(vDiff), covarianceInverseMatrix), vDiff)
    return np.sqrt(r2)

def calculateAlertSignalByApsType(sceneryID, apsType, NFFT, filepathOverride=None, alphaLambdaOverride=None,
                                  betaOverride=None, verbose=False, nHarmOverride=None):
    # get scenery
    escenario = getScenery(sceneryID)

    # get parameters
    filepath = escenario[apsType]['filepath'] if filepathOverride is None else filepathOverride
    tipoDistancia = escenario[apsType]['tipoDistancia']

    # override number of harmonics
    if nHarmOverride is None:
        nHarm = escenario[apsType]['nArmonicas']
    else:
        nHarm = nHarmOverride

    tipoNucleo = escenario[apsType]['tipoNucleo']
    fRange = escenario[apsType]['fRange']
    df = escenario[apsType]['df']
    thresholdMethod = escenario[apsType]['thresholdMethod']
    if thresholdMethod == 'fixed':
        # value is alpha
        alphaLambda = escenario[apsType]['alpha'] if alphaLambdaOverride is None else alphaLambdaOverride
    else:
        # value is lambda
        alphaLambda = escenario[apsType]['forgetFactor'] if alphaLambdaOverride is None else alphaLambdaOverride
    beta = escenario[apsType]['beta'] if betaOverride is None else betaOverride
    nEventos = escenario[apsType]['q']  # no. of periods to look behind in the alert signal
    invCovMatrix = loadInverseCovarianceMatrices(escenario[apsType]['covMatrixPath'])

    # call the function that does the work
    return calculateAlertSignalByApsTypeAux(filepath, apsType, NFFT, fRange, df, tipoNucleo,
                                            nHarm, tipoDistancia, alphaLambda, beta, nEventos, invCovMatrix,
                                            thresholdMethod, verbose)

def calculateAlertSignalByApsTypeAux(filepath, apsType, NFFT, fRange, df, kernelType, nHarmonics,
                                     distanceType, alphaLambda, beta, nEvents, inverseCovarianceMatrix,
                                     thresholdMethod, verbose):
    # get the contour signal based on the scores of the score matrix
    T = findApsPeriod(apsType)
    [nF0s, fsp, t, c] = computerContourFromFilterbank(filepath, NFFT, fRange, df,
                                                      kernelType, nHarmonics, alphaLambda, T, thresholdMethod, verbose)

    # get the template & zeros penalization
    o = getTemplate(apsType, fsp)  # get template
    wp = initializeNullPenalization(np.size(o))

    # apply a sliding euclidean modified distance or count the no. of correspondences?
    nC = len(c)
    nO = len(o)
    a = [0.0] * nC  # senial de alerta
    for n in range(0, nC - nO):
        cAct = np.asarray(c[n:n + nO])
        if (distanceType == 'l2mod'):
            a[n] = modifiedEuclideanDistance(cAct, o, wp)
        elif (distanceType == 'proporcion'):
            a[n] = 0
            for i in range(0, nO):
                a[n] += 1.0 if cAct[i] == o[i] and cAct[i] != 0 else 0.0
            a[n] /= np.count_nonzero(o)
        elif (distanceType == 'mahalanobis-real' or distanceType == 'mahalanobis-sin'):
            if inverseCovarianceMatrix is None:
                print('Error: no inverse covariance matrixes where specified')
                exit()
            a[n] = 1.0 / (mahalanobisDistance(cAct, o, inverseCovarianceMatrix) + 1)
        else:
            print('Error: unknown distance "%s"' % distanceType)
            exit()

    # post procesamiento de Ruiz que observa los picos de alerta anteriores antes de emitir una alerta
    T = findApsPeriod(apsType)
    ap = alertFilteringByPeriodicity(a, T, fsp, nEvents)

    # apply beta
    for n in range(0, len(ap)):
        ap[n] = 0 if ap[n] < beta else ap[n]

    # return
    return nF0s, t, c, a, ap

def loadInverseCovarianceMatrices(matrixPath):
    if matrixPath is None:
        return None
    S = loadMatrixFromCsv(matrixPath)
    return np.linalg.pinv(S)

def run(sceneryID, outImgFolderPath=None):
    # save current working directory and change to script's directory
    cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    NFFT = 256

    # iteramos a traves de los tipos de APS y generamos las seniales deseadas
    for apsType in ['cuckoo', 'highchirp', 'lowchirp']:

        # calculate the alert signal and apply beta after
        [_, t, c, _, ap] = calculateAlertSignalByApsType(sceneryID, apsType, NFFT)

        plt.figure()
        plt.subplot(211)
        plt.title('Escenario: %s, APS: %s' % (sceneryID, apsType))
        plt.plot(t, c)
        plt.ylabel('F0 (Hz)')
        plt.subplot(212)
        plt.plot(t, ap)
        plt.ylabel('Amplitud')
        plt.xlabel('Tiempo (s)')
        if outImgFolderPath is not None:
            plt.savefig('%s/alertSignalAndScenarios-%s-%s.pdf' % (outImgFolderPath, sceneryID, apsType))
        else:
            plt.show()

    os.chdir(cwd)  # restore directory

if __name__ == "__main__":
    run('scenery1')


from tkinter import *
from tkinter import ttk, messagebox, filedialog as fd
from tkinter.ttk import Combobox
import requests

window = Tk()
window.title("Шишкин Кирилл Сергеевич")
window.geometry('700x700')

tab_control = ttk.Notebook(window)

# Калькулятор

tab_calc = ttk.Frame(tab_control)
tab_control.add(tab_calc, text="Калькулятор")
tab_control.pack(expand=True, fill='both')

entry1 = Entry(tab_calc, text="Первое число")
entry1.pack()

sign_combo = Combobox(tab_calc)
sign_combo['values'] = ["+", "-", "*", "/"]
sign_combo.current(0)
sign_combo.pack()

entry2 = Entry(tab_calc, text="Второе число")
entry2.pack()

def calc():
    global sum_text
    a = int(entry1.get())
    b = int(entry2.get())

    sign = sign_combo.get()

    if sign == "+":
        sum_text.set(a + b)
    if sign == "-":
        sum_text.set(a - b)
    if sign == "*":
        sum_text.set(a * b)
    if sign == "/":
        sum_text.set(a / b)
calc_btn = Button(tab_calc, command=calc, text="Вычислить")
calc_btn.pack()

sum_text = StringVar()
entry_sum = Entry(tab_calc, textvariable=sum_text)
entry_sum.pack()

# Чекбоксы

tab_checkbox = ttk.Frame(tab_control)
tab_control.add(tab_checkbox, text="Чекбоксы")
tab_control.pack(expand=True, fill='both')

var = IntVar()
var.set(1)
rd1 = Radiobutton(tab_checkbox, text="Первый вариант", variable=var, value=1)
rd1.pack()
rd2 = Radiobutton(tab_checkbox, text="Второй вариант", variable=var, value=2)
rd2.pack()
rd3 = Radiobutton(tab_checkbox, text="Третий вариант", variable=var, value=3)
rd3.pack()

def rd_btn_click():
    messagebox.showinfo("Сообщение", f"Выбран вариант №{var.get()}")

rd_btn = Button(tab_checkbox, text="Сообщение", command=rd_btn_click)
rd_btn.pack()

# Текст

tab_text = ttk.Frame(tab_control)
tab_control.add(tab_text, text="Текст")
tab_control.pack(expand=True, fill='both')

def file_upload_clicked():
    global text_data
    filename = fd.askopenfilename()
    text_data.delete(1.0, END)
    with open(filename, "r") as f:
        text_data.insert(END, f.read())

menu = Menu(window)
window.config(menu=menu)
file_upload = Menu(menu)
file_upload.add_command(label='Открыть', command=file_upload_clicked)
menu.add_cascade(label='Файл', menu=file_upload)

text_data = Text(tab_text)
text_data.pack()

# Github

tab_github = ttk.Frame(tab_control)
tab_control.add(tab_github, text="Github")
tab_control.pack(expand=True, fill='both')

label = ttk.Label(tab_github, text="Вводить в формате автор/репозиторий\nНапример: kubernetes/kubernetes")
label.pack()

repo_name_entry = Entry(tab_github, text="Название репозитория")
repo_name_entry.pack()

def load_info():
    url = f"https://api.github.com/repos/{repo_name_entry.get()}"

    response = requests.get(url)

    if response.status_code == 200:
        data = response.json()
        keys = ['company', 'created_at', 'email', 'id', 'name', 'url']
        filtered_data = {k: data[k] if k in data else None for k in keys}

        with open("filtered_data.txt", "w") as f:
            f.write(str(filtered_data))

        messagebox.showinfo("Успешно", "Информация загружена")
    else:
        messagebox.showerror("Ошибка", f"Ошибка при загрузке информации: {response.reason}")

load_info_btn = ttk.Button(tab_github, command=load_info, text="Загрузить информацию")
load_info_btn.pack()

window.mainloop()


#import libraries
import os
import zipfile
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, confusion_matrix

# Define the name of the zip file and the folder where the data files will be extracted
DATA_ZIP = "EyeT.zip"
DATA_DIR = "EyeT"

# Preprocessing the data
# Define a function to preprocess the eye tracker data
def preprocess(data_dir):
    # Get a list of all CSV files in the directory
    csv_files = [os.path.join(data_dir, f) for f in os.listdir(data_dir) if f.endswith(".csv")]
    # Read the eyetracker data from files
    data_files = ['data1.csv', 'data2.csv', 'data3.csv']
    df_list = []
    for file in data_files:
        with open(file, 'r') as f:
          df_list.append(pd.read_csv(f))
    df = pd.concat(df_list)

   # Close the files
    for file in df_list:
     file.close()

    # Read the data from all CSV files into a single DataFrame
    df = pd.concat([pd.read_csv(f) for f in csv_files])

    # Calculate the saccades, pupil dilation, blink rate, gaze duration, and fixation
    # Columns in the data are: timestamp, x, y, pupil_dilation, saccade, blink, fixation
    df['duration'] = df.groupby('fixation')['timestamp'].diff().fillna(0)
    df['fixation_count'] = df.groupby('fixation')['timestamp'].transform('count')
    df['blink_rate'] = df['blink'] / df['duration']
    df['gaze_duration'] = df['duration'] / df['fixation_count']
    df = df[['pupil_dilation', 'saccade', 'blink_rate', 'gaze_duration', 'fixation']]

    return df

# Create the empathy measurement model
# The target variable is 'empathy', and it is based on the other features
X = df.drop('empathy', axis=1)
y = df['empathy']
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)
model = LogisticRegression()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
# Define a function to train the empathy model
def train(df):
    # Extract the relevant features from the DataFrame
    X = df[["saccades", "pupil_dilation", "blink_rate", "gaze_duration", "fixation"]]
    # Extract the empathy score from the DataFrame
    y = df["empathy_score"]

    # Train a linear regression model
    model = LinearRegression()
    model.fit(X, y)

    return model

# Function to evaluate the empathy model
def evaluate(model, test_data):
    # Extract relevant features from the test data
    X_test = test_data[["saccades", "pupil_dilation", "blink_rate", "gaze_duration", "fixation"]]
    # Extract the true empathy score from the EyeT data
    y_test = test_data["empathy_score"]

    # Use the trained model to predict the empathy score for the test data
    y_pred = model.predict(X_test)

    # Calculate the mean squared error of the predictions
    mse = ((y_pred - y_test) ** 2).mean()

    return mse

if __name__ == "__main__":
    # Extract the data files from the zip archive
    with zipfile.ZipFile(DATA_ZIP, "r") as zip_ref:
        zip_ref.extractall(DATA_DIR)

    # Preprocess the data
    df = preprocess(DATA_DIR)

    # Split the data into training and test sets
    train_data = df.sample(frac=0.8, random_state=42)
    test_data = df.drop(train_data.index)

    # Train the model
    model = train(train_data)

    # Evaluate the model on the test data
    mse = evaluate(model, test_data)

    # Print the mean squared error of the predictions
    print(f"Mean squared error: {mse}")

    # Remove the extracted data directory
    os.remove(DATA_ZIP)
    os.rmdir(DATA_DIR)


from collections import defaultdict

#inp = [l.strip() for l in open('dec22ex.txt')]
inp = [l.strip() for l in open('22.txt')]
#print(inp)

m = defaultdict(int)

def coord(x, y): return "{};{}".format(x, y)

CLEAN = 0
WEAKENED = 1
INFECTED = 2
FLAGGED = 3

for y in range(len(inp)):
    for x in range(len(inp[y])):
        if inp[y][x] == "#":
            m[coord(x, y)] = INFECTED

x = y = len(inp) // 2
print(x, y)
UP = [0,-1]; DOWN = [0,1]; LEFT = [-1, 0]; RIGHT = [1, 0]
directions = [UP, RIGHT, DOWN, LEFT]
d = infected = 0

turnmap = { CLEAN: 3, WEAKENED: 0, INFECTED: 1, FLAGGED: 2}
transitionmap = {CLEAN: WEAKENED, WEAKENED: INFECTED, INFECTED: FLAGGED, FLAGGED: CLEAN}

for i in range(10000000):
    if i % 10000 == 0: print(i)
    c = coord(x,y)
    d = (d + turnmap[m[c]]) % 4
    m[c] = transitionmap[m[c]]
    if m[c] == INFECTED:
        infected += 1

    x += directions[d][0]
    y += directions[d][1]

print(infected)

#Answer: 5259


from lxml import etree

FILE_NAME = 'besiktas-speed-40/osm.net.xml'
SPEED_SCALE = 2

root = etree.parse(FILE_NAME)
for elt in root.iter():
    if "speed" in elt.attrib:
        current_speed = float(elt.attrib["speed"])
        elt.attrib["speed"] = str(current_speed * SPEED_SCALE)
    if "limitTurnSpeed" in elt.attrib:
        current_speed = float(elt.attrib["limitTurnSpeed"])
        elt.attrib["limitTurnSpeed"] = str(current_speed * SPEED_SCALE)
    print(elt)
# getting the parent tag of
# the xml document
b_xml = etree.tostring(root, pretty_print=True)

with open(FILE_NAME, "wb") as f:
    f.write(b_xml)


import tensorflow as tf
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import csv
from sklearn.preprocessing import MinMaxScaler
from sklearn import metrics

NUM_MODEL_RUNS = 1 #abs(int(input("Enter the number of times to train model and store metrics: ")))
NUM_EPOCHS = 100 #abs(int(input("Enter the number of epochs to run: ")))
BATCH_SIZE = 32 #abs(int(input("Enter the desired batch size: ")))

LAYER_1_NEURONS = 512 #abs(int(input("Enter the number of neurons for layer 1: ")))
LAYER_2_NEURONS = 128 #abs(int(input("Enter the number of neurons for layer 2: ")))
LAYER_3_NEURONS = 64 #abs(int(input("Enter the number of neurons for layer 3: ")))

LOOK_BACK = 3 #abs(int(input("Enter the number of previous months to use for predictions: ")))
LOOK_AHEAD = 1 #abs(int(input("Enter the number of months in the future to predict a recession at: ")))

TRAIN_PERCENTAGE = 0.7 #abs(float(input("Enter the percentage of data to use in the training set: ")))
DROPOUT_RATE = 0.1 #abs(float(input("Enter a dropout rate for regularization of the recurrent layers. Value in range [0, 1]: ")))
CLASSIFIER_THRESHOLD = [0.04] #abs(float(input("Enter threshold for classifer: ")))
TO_PRINT = False #bool(input("Would you like to print the Train/Val curve and ROC every iteration? (0=False, 1=True): "))
WRITE_METRICS = False #bool(input("Would you like to write the output metrics to Excel .csv files? (0=False, 1=True): "))
DISPLAY_MODEL = False #bool(input("Would you like to display model summary? (0=False, 1=True): "))

NUM_FEATURES = 10
SCALER = MinMaxScaler()
TO_SCALE = True
OPTIMIZER = tf.keras.optimizers.Adam()
LOSS = tf.keras.losses.BinaryCrossentropy()
DENSE_ACTIVATION = "sigmoid"
#EARLY_STOP = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')
METRICS = [
      #tf.keras.metrics.TruePositives(name='tp'),
      #tf.keras.metrics.FalsePositives(name='fp'),
      #tf.keras.metrics.TrueNegatives(name='tn'),
      #tf.keras.metrics.FalseNegatives(name='fn'),
      tf.keras.metrics.BinaryAccuracy(name='accuracy'),
      #tf.keras.metrics.Precision(name='precision'),
      #tf.keras.metrics.Recall(name='recall'),
      #tf.keras.metrics.AUC(name='auc'),
]

def plot_training(model, name):

    plt.plot(model.history['loss'], label='train loss')
    plt.plot(model.history['accuracy'], label='train accuracy')

    plt.xlabel('Epoch')
    plt.ylabel('Value of Metric')
    plt.title('{} RecessionNET Training/Validation Loss and Accuracy vs Epoch'.format(name))
    plt.legend()
    plt.show()

    return(0)

def plot_predictions(actual, predictions,  name):

    plt.plot(actual, label='Actual')
    plt.plot(predictions, label='Predictions')

    plt.xlabel('Time')
    plt.ylabel('Probability of Recession')
    plt.title('{} RecessionNET Actual/Predicted vs Time'.format(name))
    plt.legend()
    plt.show()

    return(0)

def plot_out_of_sample_ROC(actual, predictions, name):

    fpr, tpr, thresholds = metrics.roc_curve(actual, predictions)
    auc = auc(fpr, tpr)

    plt.figure(1)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.plot(fpr, tpr, label='Area: {:.3f})'.format(auc))
    plt.xlabel('False positive rate')
    plt.ylabel('True positive rate')
    plt.title('{} RecessionNET ROC curve'.format(name))
    plt.legend(loc='best')
    plt.show()

    return(0)

def final_metrics(actual, predictions, threshold):

    binary_predictions = np.where(predictions > threshold, 1, 0)

    accuracy = metrics.accuracy_score(actual, binary_predictions)
    auc = metrics.roc_auc_score(actual, binary_predictions)
    f1 = metrics.f1_score(actual, binary_predictions)
    recall = metrics.recall_score(actual, binary_predictions)

    if TO_PRINT:
        print("Out-of-sample Accuracy: {}".format(accuracy))
        print("Out-of-sample Area Under Receiver Operating Curve: {}".format(auc))
        print("Out-of-sample F1-Score: {}".format(f1))
        print("Out-of-sample Recall: {}\n".format(recall))

    return [accuracy, auc, f1, recall]

def make_dataset(features, responses, look_back, look_ahead):
    print("\nGenerating dataset...")
    print("Using the past {} months of data to predict a recession {} month(s) away.".format(look_back, look_ahead))

    features = np.array(features)
    responses = np.array(responses)

    # Normalize Sets Here
    if TO_SCALE:
        features = SCALER.fit_transform(features)
        responses = SCALER.fit_transform(responses)

    feature_set, response_set = [], []
    t = 0
    max_index = len(features)-1
    for t in range(max_index-look_back-look_ahead+2):
        feature_sample = features[t:t+look_back, ]
        response_sample = responses[t+look_back+look_ahead-1, ]
        feature_set.append(feature_sample)
        response_set.append(response_sample)

    print("Dataset has been made.\n")

    return np.array(feature_set), np.array(response_set)

def split_dataset(features, responses, split_ratio):
    print("\nSplitting data...\n")
    n = len(features)
    print("Length of data set {}".format(n))
    print("Length of train set {}".format(int(n*split_ratio)))
    print("Length of test set {}".format(int(n-(n*split_ratio)+1)))
    X_train = features[0:int(n * split_ratio)]
    Y_train = responses[0:int(n * split_ratio)]
    X_test = features[int(n * split_ratio)+1:]
    Y_test = responses[int(n * split_ratio)+1:]
    print("Dataset has been split into test-train sets.\n")
    return X_train, Y_train, X_test, Y_test

print("\nImporting data...\n")
macrovars = pd.read_csv("macrovars.csv")
recession = pd.read_csv("recession_state.csv")
macrovars = macrovars.rename(columns={"SP500 (Percent change from a year ago)": "SP5001YDelta",
                                      "10Y Rate - 3M TBill ": "10YR3MDelta",
                                      "10Y Rate (Percent change from a year ago": "10YR1YDelta",
                                      "3M TBill (Percent Change from a year ago)": "3M1YDelta",
                                      "3M TBill - FF ": "3MFFDelta",
                                      "6M TBill - FF": "6MFFDelta",
                                      "1Y Rate - FF": "1YRFFDelta",
                                      "5Y Rate - FF": "5YRFFDelta",
                                      "10Y Rate - FF": "10YRFFDelta",
                                      "Composite Leading Indicator": "CLI"})

macrovars = macrovars.drop(["Date"], axis = 1)
recession = recession.rename(columns={"Recession State": "Recession"})

print("Features:")
print(macrovars.head())

print("\nResponse:")
print(recession.head())

macrovars = macrovars.to_numpy()
recession = recession.to_numpy()

X, Y = make_dataset(features = macrovars, responses = recession, look_back = LOOK_BACK, look_ahead = LOOK_AHEAD)
X_train, Y_train, X_test, Y_test = split_dataset(features = X, responses = Y, split_ratio = TRAIN_PERCENTAGE)

print("X_train shape: {}".format(X_train.shape))
print("Y_train shape: {}".format(Y_train.shape))
print("\n")

###################################################################################################################################################

for j in range(len(CLASSIFIER_THRESHOLD)):

    LSTM_HISTORY = []
    GRU_HISTORY = []

    for i in range(NUM_MODEL_RUNS):

        inputs = tf.keras.Input(shape=(LOOK_BACK, NUM_FEATURES))
        inputs.shape
        lstm1 = tf.keras.layers.LSTM(units = LAYER_1_NEURONS, input_shape = (LOOK_BACK, NUM_FEATURES), return_sequences = True, dropout=DROPOUT_RATE)
        lstm2 = tf.keras.layers.LSTM(units = LAYER_2_NEURONS, input_shape = (LOOK_BACK, NUM_FEATURES), return_sequences = True, dropout=DROPOUT_RATE)
        lstm3 = tf.keras.layers.LSTM(units = LAYER_3_NEURONS, input_shape = (LOOK_BACK, NUM_FEATURES), return_sequences = False, dropout=DROPOUT_RATE)

        x = lstm1(inputs)
        x = lstm2(x)
        x = lstm3(x)
        outputs = tf.keras.layers.Dense(units = 1, activation=DENSE_ACTIVATION)(x)
        lstm_model = tf.keras.Model(inputs = inputs, outputs = outputs)

        if DISPLAY_MODEL:
            lstm_model.summary()
            tf.keras.utils.plot_model(lstm_model, show_shapes=True, to_file = "lstm_model.png")
        lstm_model.compile(loss=LOSS,
                        optimizer=OPTIMIZER,
                        metrics=METRICS)

        lstm_model_history = lstm_model.fit(x = X_train, y = Y_train, epochs = NUM_EPOCHS, verbose = 0) #validation_data = (X_test, Y_test), callbacks = [EARLY_STOP])
        lstm_model_preds = lstm_model.predict(x = X_test)

        # plot training history
        if TO_PRINT:
            plot_training(lstm_model_history, "LSTM")

        Y_predicted = lstm_model.predict(x = X_test).ravel()
        plot_predictions(Y_test, Y_predicted, "LSTM")
        oos_metrics = final_metrics(Y_test, Y_predicted, CLASSIFIER_THRESHOLD[j])
        LSTM_HISTORY.append(oos_metrics)

        if TO_PRINT:
            plot_out_of_sample_ROC(Y_test, Y_predicted, "LSTM")

        #########################################################################################################################################################################

        inputs = tf.keras.Input(shape=(LOOK_BACK, NUM_FEATURES))
        inputs.shape
        gru1 = tf.keras.layers.GRU(units = LAYER_1_NEURONS, input_shape = (LOOK_BACK, NUM_FEATURES), return_sequences = True, dropout=DROPOUT_RATE)
        gru2 = tf.keras.layers.GRU(units = LAYER_2_NEURONS, input_shape = (LOOK_BACK, NUM_FEATURES), return_sequences = True, dropout=DROPOUT_RATE)
        gru3 = tf.keras.layers.GRU(units = LAYER_3_NEURONS, input_shape = (LOOK_BACK, NUM_FEATURES), return_sequences = False, dropout=DROPOUT_RATE)

        x = gru1(inputs)
        x = gru2(x)
        x = gru3(x)
        outputs = tf.keras.layers.Dense(units = 1, activation=DENSE_ACTIVATION)(x)
        gru_model = tf.keras.Model(inputs = inputs, outputs = outputs)

        if DISPLAY_MODEL:
            gru_model.summary()
            tf.keras.utils.plot_model(gru_model, show_shapes=True, to_file = "gru_model.png")

        gru_model.compile(loss=LOSS,
                        optimizer=OPTIMIZER,
                        metrics=METRICS)

        gru_model_history = gru_model.fit(x = X_train, y = Y_train, epochs = NUM_EPOCHS, verbose = 0) #validation_data = (X_test, Y_test), callbacks = [EARLY_STOP])
        gru_model_preds = gru_model.predict(x = X_test)

        if TO_PRINT:
            plot_training(gru_model_history, "GRU")

        Y_predicted = gru_model.predict(x = X_test).ravel()
        plot_predictions(Y_test, Y_predicted, "GRU")
        oos_metrics = final_metrics(Y_test, Y_predicted, CLASSIFIER_THRESHOLD[j])
        GRU_HISTORY.append(oos_metrics)

        if TO_PRINT:
            plot_out_of_sample_ROC(Y_test, Y_predicted, "GRU")

        tf.keras.backend.clear_session()

    if WRITE_METRICS:

        LSTM_HISTORY_df = pd.DataFrame(LSTM_HISTORY, columns = ["Accuracy", "AUC", "F1-Score", "Recall"])
        GRU_HISTORY_df = pd.DataFrame(GRU_HISTORY, columns = ["Accuracy", "AUC", "F1-Score", "Recall"])

        LSTM_HISTORY_df.to_csv("LSTM_HISTORY_{}.csv".format(str(CLASSIFIER_THRESHOLD[j])))
        GRU_HISTORY_df.to_csv("GRU_HISTORY_{}.csv".format(str(CLASSIFIER_THRESHOLD[j])))

print("RecessionNET has completed running.")
print("All output metrics have been written into the current working directory.")


from setuptools import setup, Extension
from setuptools.command.install import install
from codecs import open
import os, os.path, struct, re, platform

import numpy

try:
    from setuptools._distutils import _msvccompiler
    _msvccompiler.PLAT_TO_VCVARS['win-amd64'] = 'amd64'
except:
    pass

if os.environ.get('TOMOTOPY_CPU_ARCH'):
    tomotopy_cpu_arch = os.environ['TOMOTOPY_CPU_ARCH']
else:
    try:
        import platform
        tomotopy_cpu_arch = platform.machine()
    except:
        tomotopy_cpu_arch = 'x86_64'

from sysconfig import get_platform
fd = get_platform().split('-')
if fd[0] == 'macosx':
    if os.environ.get('MACOSX_DEPLOYMENT_TARGET'):
        from distutils import sysconfig
        cfg_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') or ''
        if cfg_target.startswith('10.'):
            cfg_target = list(map(int, cfg_target.split('.')))
            cur_target = list(map(int, os.environ['MACOSX_DEPLOYMENT_TARGET'].split('.')))
            target = max(cfg_target, cur_target)
            if target != cur_target:
                print(f"MACOSX_DEPLOYMENT_TARGET={'.'.join(map(str, cur_target))} is not supported. MACOSX_DEPLOYMENT_TARGET={'.'.join(map(str, target))} is used instead.")
            os.environ['MACOSX_DEPLOYMENT_TARGET'] = '.'.join(map(str, target))
        fd[1] = os.environ['MACOSX_DEPLOYMENT_TARGET']
    os.environ['_PYTHON_HOST_PLATFORM'] = '-'.join(fd[:-1] + [tomotopy_cpu_arch])

exec(open('tomotopy/_version.py').read())

here = os.path.abspath(os.path.dirname(__file__))

long_description = ''
for line in open(os.path.join(here, 'tomotopy/documentation.rst'), encoding='utf-8'):
    long_description += re.sub(r'^<.+>\s*$', '', line)

sources = []
for f in os.listdir(os.path.join(here, 'src/python')):
    if f.endswith('.cpp') and not f.endswith('py_rt.cpp'): sources.append('src/python/' + f)
for f in os.listdir(os.path.join(here, 'src/TopicModel')):
    if f.endswith('.cpp'): sources.append('src/TopicModel/' + f)
for f in os.listdir(os.path.join(here, 'src/Labeling')):
    if f.endswith('.cpp'): sources.append('src/Labeling/' + f)

largs = []
arch_levels = {'':'', 'sse2':'-msse2', 'avx':'-mavx', 'avx2':'-mavx2 -mfma'}
if platform.system() == 'Windows':
    cargs = ['/O2', '/MT', '/Gy', '/D__SSE2__']
    arch_levels = {'':'', 'sse2':'/arch:SSE2', 'avx':'/arch:AVX', 'avx2':'/arch:AVX2'}
elif platform.system() == 'Darwin':
    cargs = ['-std=c++1y', '-O3', '-fpermissive', '-stdlib=libc++', '-Wno-unused-variable', '-Wno-switch']
    largs += ['-stdlib=libc++']
    if 'many' in os.environ.get('AUDITWHEEL_PLAT', ''): cargs.append('-g0')
    if tomotopy_cpu_arch == 'arm64':
        arch_levels = {'': ''}
        cargs += ['-DTOMOTOPY_ISA=arm64', '-arch', 'arm64']
        largs += ['-arch', 'arm64']
    else:
        if 'many' not in os.environ.get('AUDITWHEEL_PLAT', ''): arch_levels = {'':'-msse2'}
        cargs += ['-arch', 'x86_64']
        largs += ['-arch', 'x86_64']
elif 'many' in os.environ.get('AUDITWHEEL_PLAT', ''):
    cargs = ['-std=c++1y', '-O3', '-fpermissive', '-g0', '-Wno-unused-variable', '-Wno-switch']
    if tomotopy_cpu_arch in ('arm64', 'aarch64'):
        arch_levels = {'': ''}
        cargs += ['-DTOMOTOPY_ISA=arm64']
else:
    cargs = ['-std=c++1y', '-O3', '-fpermissive', '-Wno-unused-variable', '-Wno-switch']
    arch_levels = {'': '-march=native'}

if struct.calcsize('P') < 8: arch_levels = {k:v for k, v in arch_levels.items() if k in ('', 'sse2')}
else: arch_levels = {k:v for k, v in arch_levels.items() if k not in ('sse2',)}

lang_macro = []
if os.environ.get('TOMOTOPY_LANG') == 'kr': lang_macro = [('DOC_KO', '1')]

modules = []
if len(arch_levels) > 1:
    modules.append(Extension('_tomotopy',
        libraries=[],
        include_dirs=['include', numpy.get_include()],
        sources=['src/python/py_rt.cpp'],
        extra_compile_args=cargs, extra_link_args=largs))

for arch, aopt in arch_levels.items():
    if len(arch_levels) > 1:
        module_name = '_tomotopy_' + (arch or 'none')
    else:
        module_name = '_tomotopy'
    modules.append(Extension(module_name,
                    libraries=[],
                    include_dirs=['include', numpy.get_include()],
                    sources=sources,
                    define_macros=[('MODULE_NAME', 'PyInit_' + module_name)] + lang_macro,
                    extra_compile_args=cargs + (aopt.split(' ') if aopt else []), extra_link_args=largs))

setup(
    name='tomotopy',

    version=__version__,

    description='Tomoto, Topic Modeling Tool for Python',
    long_description=long_description,

    url='https://github.com/bab2min/tomotopy',

    author='bab2min',
    author_email='bab2min@gmail.com',

    license='MIT License',

    classifiers=[
        'Development Status :: 3 - Alpha',

        "Intended Audience :: Information Technology",
        "Intended Audience :: Science/Research",
        "Topic :: Software Development :: Libraries",
        "Topic :: Text Processing :: Linguistic",
        "Topic :: Scientific/Engineering :: Information Analysis",

        "License :: OSI Approved :: MIT License",

        'Programming Language :: Python :: 3',
        'Programming Language :: C++',
        "Operating System :: Microsoft :: Windows :: Windows Vista",
        "Operating System :: Microsoft :: Windows :: Windows 7",
        "Operating System :: Microsoft :: Windows :: Windows 8",
        "Operating System :: Microsoft :: Windows :: Windows 8.1",
        "Operating System :: Microsoft :: Windows :: Windows 10",
        "Operating System :: POSIX",
        "Operating System :: MacOS"
    ],
    install_requires=['numpy>=1.11.0'],
    keywords='NLP,Topic Model',

    packages=['tomotopy'],
    include_package_data=True,
    ext_modules=modules
)


#! /usr/bin/python

import pandas as pd
import numpy as np
from itertools import repeat
import matplotlib.pyplot as plt
import seaborn as sns
import itertools

# load guppy data
df = {}
barcode_list = ['01','02','03','04','05','06','08','09','10','11']
for barcode_number in barcode_list:
    df[barcode_number] = pd.DataFrame()
    f1 = open("barcode{}/guppy.5mC.cpg.bam".format(barcode_number),'r')
    chromosome_list = list()
    start_position_list = list()
    end_position_list = list()
    modified_base_percentage_list = list()
    unmodified_base_count_list = list()
    modified_base_count_list = list()

    lines = f1.readlines()
    # strand = ['+','-']

    for line in lines:

        #split the line into a list by whitespace
        splitLine = line.rstrip().split()

        # change the condition here for data in either positive or negative strand
        if splitLine[5] == '+':

            chromosome = float(splitLine[0])
            start_position = float(splitLine[1])
            end_position = float(splitLine[2])
            modified_base_percentage = float(splitLine[10])
            unmodified_base_count = float(splitLine[11])
            modified_base_count = float(splitLine[12])

            chromosome_list.append(chromosome)
            start_position_list.append(start_position)
            end_position_list.append(end_position)
            modified_base_percentage_list.append(modified_base_percentage)
            unmodified_base_count_list.append(unmodified_base_count)
            modified_base_count_list.append(modified_base_count)

    f1.close()

    df[barcode_number]['chromosome'] = chromosome_list
    df[barcode_number] = df[barcode_number].astype({'chromosome':'int'})
    df[barcode_number]['start'] = start_position_list
    df[barcode_number] = df[barcode_number].astype({'start':'int'})
    df[barcode_number]['end'] = end_position_list
    df[barcode_number]['modified_base_percentage'] = modified_base_percentage_list
    df[barcode_number]['unmodified_base_count'] = unmodified_base_count_list
    df[barcode_number]['modified_base_count'] = modified_base_count_list
    df[barcode_number]['coordinates'] = mgc1(df[barcode_number]['chromosome'],df[barcode_number]['start']).rename('new')
    df[barcode_number] = df[barcode_number].rename(columns={'modified_base_percentage':'barcode{}_methylated_frequency'.format(barcode_number)})

# scatter plot of region in chromosome 7
SMALL_SIZE = 20
MEDIUM_SIZE = 20
BIGGER_SIZE = 30

sns.set_style('darkgrid')
plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=40)     # fontsize of the axes title
plt.rc('axes', labelsize=BIGGER_SIZE)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
plt.rc('figure', titlesize=50)  # fontsize of the figure title

plt.figure()
plt.figure(figsize=(35, 15), dpi=80)

barcode_list = ['08','09','10','11','01','02','03','04','05','06']
legend_list = ['30hpi_BR1','36hpi_BR1','30hpi_BR2','36hpi_BR2','30hpi_BR3','36hpi_BR3','30hpi_BR4','36hpi_BR4',
               '30hpi_BR5','36hpi_BR5']
for count, barcode_number in enumerate(barcode_list):
    data =  df[barcode_number][ ( df[barcode_number]['chromosome']==7) & ( df[barcode_number]['start']>=500000) & (df[barcode_number]['end']<=600000)]
    if barcode_number in ['01','03','05','08','10']:
        plt.plot(data['start'],data['modified_base_percentage'],'o',label='{}'.format(legend_list[count]),color='blue')

    if barcode_number in ['02','04','06','09','11']:
        plt.plot(data['start'],data['modified_base_percentage'],'o',label='{}'.format(legend_list[count]),color='green')

plt.xlabel("Chr7:500000-600000",labelpad=15,fontsize=40)
plt.ylabel("modified base percentage(%)",labelpad=15,fontsize=40)
plt.ylim(0,100)
plt.title("Positive Strand",fontsize=60)
plt.legend()
file_name = 'chr7:500000_600000_positive_strand.png'
plt.savefig(file_name,dpi=500, bbox_inches='tight')
plt.show()


'''
utils.py

Utility functions used by multiple scripts.
'''

import os
import gzip
import json
import logging
import csv
import itertools
import numpy as np
import string
from operator import itemgetter

def gen_shares(filepath):
    with open(filepath, 'r') as f:
        shares = json.loads(f.read())
        for uid in shares:
            for tid in shares[uid]:
                yield uid, tid, shares[uid][tid]

def domain(url):
    """
    >>> domain('http://https://test.com')
    'https'
    >>> domain('http://test.com?site=https://other.com')
    'test.com'
    >>> domain('http://test.com/index?test=1')
    'test.com'
    >>> domain('http://indiana.facebook.com/dir1/page.html')
    'indiana.facebook.com'
    >>> domain('http://facebook.com')
    'facebook.com'
    >>> domain('http://www.facebook.com:80/')
    'facebook.com'
    """
    url = (''.join(filter(lambda c: c in string.printable, url))).lower()

    # remove protocol
    idx = url.find("://")
    if idx != -1:
        url = url[idx + 3:]

    # remove port
    idx = url.find(":")
    if idx != -1:
        url = url[:idx]

    # remove trailing / and everything that follows
    idx = url.find("/")
    if idx != -1:
        url = url[:idx]

    # remove ? and anything that follows if it wasn't in previous steps
    idx = url.find("?")
    if idx != -1:
        url = url[:idx]

    url_is_clean = False
    while not url_is_clean:
        if url.startswith('www.'):
            url = url[4:]
        elif url.startswith('www2.') or url.startswith('www3.'):
            url = url[5:]
        else:
            url_is_clean = True
    return url

def readcol(csv_filepath, col=0, delim='\t', skip_rows=0, apply_fn=None):
    if csv_filepath is None or not os.path.exists(csv_filepath):
        raise ValueError('Invalid filepath: {}.'.format(csv_filepath))

    items = []
    with open(csv_filepath, 'r') as f:
        reader = csv.reader(f, delimiter=delim)

        # skip header lines
        for _ in range(skip_rows):
            try:
                next(reader)
            except StopIteration:
                raise ValueError('Invalid skip_rows: {}, {}'.format(skip_rows, csv_filepath))

        for row in reader:
            if col >= len(row):
                raise ValueError('Invalid col: {}, {}'.format(col, csv_filepath))

            if apply_fn is not None:
                val = apply_fn(row[col])
            else:
                val = row[col]

            items.append(val)

    return items

def write_csv(filepath, items, headers=None, delimiter='\t'):
    with open(filepath, 'w') as f:
        w = csv.writer(f, delimiter=delimiter)
        if headers is not None:
            rows = [headers]
            rows.extend(items)
        else:
            rows = items
        w.writerows(rows)

if __name__ == "__main__":
    logging.info('Testing...')
    import doctest
    doctest.testmod()


import sys
from collections import Counter
# +++your code here+++
# Define print_words(filename) and print_top(filename) functions.
# You could write a helper utility function that reads a file
# and builds and returns a word/count dict for it.
# Then print_words() and print_top() can just call the utility function.

def print_words(filename):
  words_list=[]
  f=open(filename,'rU')
  for line in f:
    line_words=line.split()
    line_words=[ x.lower() for x in line_words ]
    words_list+=line_words
  words_count_map=Counter(words_list)
  unique_word_list=list(words_count_map.keys())
  unique_word_list_sorted=sorted(unique_word_list)
  for word in unique_word_list_sorted:
    print(word,words_count_map[word])

if __name__ == '__main__':
  print_words('small.txt')


from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from .spiders import DocScraperSpider

def crawl(domains,
          start_urls=[],
          directory='./output',
          extensions=['.pdf', '.docx', '.doc', '.txt'],
          robotstxt_obey=True,
          download_delay=0,
          time_range=None):
    """ Run the crawler from a Python script.

    :param domains: the domains allowed to be crawled
    :type domains: list
    :param directory: the output directory for downloaded files
    :type directory: str
    :param extensions: the document extension types to download
    :type extensions: list
    :param robotstxt_obey: whether to obey robots.txt instructions
    :type robotstxt_obey: bool
    :param wayback_machine_time_range: dates for crawling the wayback machine
    :type wayback_machine_time_range: tuple

    """
    settings = get_project_settings()

    settings['ROBOTSTXT_OBEY'] = robotstxt_obey
    settings['DOWNLOAD_DELAY'] = download_delay

    # Set directory path for file downloads
    settings['FILES_STORE'] = directory

    settings['FEEDS'] = {
        '{}/file-listing.csv'.format(directory): {'format': 'csv'}
    }
    settings['FEED_EXPORT_FIELDS'] = ['domain', 'url', 'relative_path',
                                      'checksum', 'status', 'url_date',
                                      'filetype', 'filename']

    if time_range is not None:
        settings['DOWNLOADER_MIDDLEWARES'] = {
            'scrapy_wayback_machine.WaybackMachineMiddleware': 5
        }
        settings['WAYBACK_MACHINE_TIME_RANGE'] = time_range

    process = CrawlerProcess(settings)
    process.crawl(DocScraperSpider,
                  domains,
                  start_urls,
                  directory=directory,
                  extensions=extensions,
                  time_range=time_range
                  )

    process.start()

    return


#!/usr/bin/env python
# coding: utf-8

# In[168]:

import matplotlib.pyplot as plt
import numpy as np
import random
from cat import Cat

# In[169]:

MAXROW = 20
MAXCOL = 30

# In[170]:

def processline(linelist):
    newlist = [(int(l.split(",")[0]),int(l.split(",")[1])) for l in linelist[1:]]
    return newlist

def read_data(filename):
    file = open(filename,'r')
    data = file.readlines()
    lists = []
    for line in data:
        new_list = processline(line.split(":"))
        lists.append(new_list)
    foodlist = lists [0]
    waterlist = lists [1]
    bedlist = lists [2]
    boxlist = lists [3]
    return foodlist, waterlist, bedlist, boxlist

def move_em(current, moves):
    nextgrid = np.zeros(current.shape, dtype="int16")

    for row in range(MAXROW):
        for col in range(MAXCOL):
            for g in range(current[row,col]):
                nextrow = row + random.choice(moves)
                nextcol = col + random.choice(moves)
                if nextrow < 0:
                    nextrow = 0
                if nextcol < 0:
                    nextcol = 0
                if nextrow >= MAXROW:
                    nextrow = MAXROW - 1
                if nextcol >= MAXCOL:
                    nextcol = MAXCOL - 1
                nextgrid[nextrow, nextcol] += 1
    return nextgrid

def rule1(pop, datalist, marksList):
    for row in range(MAXROW):
        for col in range(MAXCOL):
            if (row, col) in datalist:
                for g in range(pop[row,col]):
                    pop[row, col] -= 1
                    marksList.append(-5)
                    print("CAT KILLED")

def rule2(pop, datalist, marksList):
    for row in range(MAXROW):
        for col in range(MAXCOL):
            if (row, col) in datalist:
                for g in range(pop[row,col]):
                    marksList.append(10)
                    print("CAT GET ENERGY")

def rule3(array1, array2, marksList):
    for row in range(MAXROW):
        for col in range(MAXCOL):
            if array1[row,col]>0 and array2[row,col]>0:
                for k in range(array1[row,col]):
                    array1[row, col] += 1
                    array2[row,col] -= 1
                    print("CAT PERSONALITY CHANGE")

def make_feature_scatter_boost(itemlist, colour, plotData):
    xlist = []
    ylist = []
    for r,c in itemlist:
        ylist.append(MAXROW - r - 1)
        xlist.append(c)
    plotData.append([ str(xlist), str(ylist), '^'])
    plt.scatter(xlist,ylist,color=colour, marker='^')

def make_feature_scatter_barriers(itemlist, colour, plotData):
    xlist = []
    ylist = []
    for r,c in itemlist:
        ylist.append(MAXROW - r - 1)
        xlist.append(c)
    plotData.append([ str(xlist), str(ylist), 'X'])
    plt.scatter(xlist,ylist,color=colour, marker='X')

def make_my_scatter(pop, colour, plotData):
    xlist = []
    ylist = []
    slist = []
    for row in range(MAXROW):
        for col in range(MAXCOL):
            if pop[row,col] > 0:
                ylist.append(MAXROW - row - 1)
                xlist.append(col)
                slist.append(pop[row,col]*20)
    plotData.append([ str(xlist), str(ylist), 'o'])
    plt.scatter(xlist,ylist,s=slist,color=colour, marker='o')

# In[171]:

def ask_questions(questions):
    FriendlyCats = []
    AggressiveCats = []
    answerList=[]

    try:
        for question in questions:
            answer = input(question)
            answerList.append(answer)

        q1 = int(answerList[0])
        q2 = answerList[1]
        q3 = int(answerList[2])
        q4 = int(answerList[3])

        for a in range(q3):
            if(q1==1):
                FriendlyCats.append(Cat("Male", q2, "Friendly"))
            elif(q1==2):
                FriendlyCats.append(Cat("Female", q2, "Friendly"))
            else:
                print("OUT OF RANGE")

        for b in range(q4):
            if(q1==1):
                AggressiveCats.append(Cat("Male", q2, "Aggressive"))
            elif(q1==2):
                AggressiveCats.append(Cat("Female", q2, "Aggressive"))
            else:
                print("OUT OF RANGE")

        return FriendlyCats, AggressiveCats

    except:
        print("An exception occurred")

# In[172]:

def main():
    print("\n**********WELCOME TO GAME OF CATS (VILLAGE)**********\n ")

    questions = [
     "Cats gender?\n(1) Male \n(2) Female\n",
     "Cats age group?\n(Kitten, Junior, Prime, Mature, Senior)\n",
     "Friendly Cats Count?\n",
     "Aggressive Cats Count?\n"
    ]

    friendlyCartList, aggressiveCartList = ask_questions(questions)

    pointsList = []
    plotData = []

    friendlyArray = np.zeros((MAXROW,MAXCOL), dtype="int16")
    aggressiveArray = np.zeros((MAXROW,MAXCOL), dtype="int16")

    food, water, bed, boxes = read_data("cat.txt")

    moves  = [-1,0,1]
#     moves  = [-2,-1,0,1,2]
    print("--- CAT VILLAGE ---")
    for friendlyCart in friendlyCartList:
        print(friendlyCart)
        friendlyArray[random.randint(0,MAXROW-1),random.randint(0,MAXCOL-1)] += 1

    for aggressiveCart in aggressiveCartList:
        print(aggressiveCart)
        aggressiveArray[random.randint(0,MAXROW-1),random.randint(0,MAXCOL-1)] += 1

    # Simulation
    for t in range(10): # Simulation Time 10s
        try:
            friendlynext = move_em(friendlyArray, moves)
            aggressivenext = move_em(aggressiveArray, moves)

            rule1(friendlynext, bed, pointsList)
            rule1(friendlynext, boxes, pointsList)
            rule2(friendlynext, food, pointsList)
            rule2(friendlynext, water, pointsList)
            rule3(friendlynext, aggressivenext, pointsList)

            rule1(aggressivenext, bed, pointsList)
            rule1(aggressivenext, boxes, pointsList)
            rule2(aggressivenext, food, pointsList)
            rule2(aggressivenext, water, pointsList)
            rule3(aggressivenext, friendlynext, pointsList)

            friendlyArray = friendlynext
            aggressiveArray = aggressivenext

            make_my_scatter(friendlyArray, "b", plotData)
            make_my_scatter(aggressiveArray, "r", plotData)

            make_feature_scatter_boost(food, "#f0ad4e", plotData)
            make_feature_scatter_boost(water, "#5bc0de", plotData)

            make_feature_scatter_barriers(bed, "#d9534f", plotData)
            make_feature_scatter_barriers(boxes, "#f7f7f7", plotData)

            plt.title("Cat Simulation (time = " + str(t) + ")")
            plt.rcParams['axes.facecolor']='black'
            plt.xlabel("Columns")
            plt.ylabel("Rows")
            plt.xlim(-1,MAXCOL)
            plt.ylim(-1,MAXROW)
            plt.savefig('Cat_Simulation.png')
            plt.pause(1)
            plt.clf()

            print("Score {}".format(sum(pointsList)))
        except:
              print("An exception occurred")

    print("Friendly Cats Count {}".format(len(friendlyArray)))
    print("Aggressive Cats Count {}".format(len(aggressiveArray)))

    plotToCSV = np.array(plotData)
    np.savetxt('plotData.csv', plotToCSV, fmt="%s", delimiter=',')

    if(sum(pointsList)>0):
        print("GAME WON")
    elif(sum(pointsList)==0):
        print("GAME DRAW")
    else:
        print("GAME LOSE")

# In[173]:

if __name__ == "__main__":
    main()

# In[ ]:


from typing import List

def minimumOperations(nums: List[int]) -> int:
    cont = 0

    while sum(nums) != 0:
        print(nums)
        min_number = min(elem for elem in nums if elem != 0)
        nums = list(map(lambda x: x - min_number if x != 0 else 0, nums))

        cont += 1

    return cont

if __name__ == "__main__":
    print(minimumOperations([0]))


# %%
from typing import Optional, Union
import pandas as pd
import numpy as np
import requests, re, os

from concurrent.futures import ProcessPoolExecutor

class Decrypter:
    """Simple MCMC decryption algorithm using random walk"""
    def __init__(self, text_url: str,
                       allowable_chars: str = r"[a-zA-Z0-9\s]",
                       allow_uppercase:bool = False,
                       max_workers: Optional[int] = None,
                       **kwargs) -> None:

        self.reference_text_url = text_url
        self.allowable_chars    = allowable_chars
        self.allow_uppercase    = allow_uppercase
        self.max_workers        = max_workers if max_workers is not None else os.cpu_count()
        self.kwargs             = kwargs

        self.reference_text     = self.get_text()
        self.reference_text     = self.clean_text(self.reference_text)

    def get_text(self) -> str:
        """Retrieves text from given url

        Returns:
            str: text from url
        """
        r = requests.get(self.reference_text_url)
        return r.text

    def clean_text(self, text:str) -> str:
        """Removes non-allowable characters from text

        Args:
            text (str): text to clean

        Returns:
            str: cleaned text
        """

        #Find the substring that marks the end of the text
        start_substring = re.search(r"\*\*\* START OF THE PROJECT GUTENBERG EBOOK ", text).end()
        end_substring   = re.search(r"\*\*\* END OF THE PROJECT GUTENBERG EBOOK ", text).start()
        text            = text[start_substring:end_substring]

        #If uppercase are not allowed, turn them into lowercase
        if not self.allow_uppercase:
            text = text.lower()

        #Remove non-allowable characters
        text = re.sub(self.invert_regex_pattern(), "", text)

        #Convert all whitespace to single space
        text = re.sub(r"\s+", " ", text)

        return text

    def invert_regex_pattern(self, pattern: Optional[str] = None) -> str:
        """Inverts the given regex pattern. Only works with simple regex patterns

        Args:
            pattern (Optional[str], optional): Simple regex pattern to invert. Defaults to None.

        Returns:
            str: inverted regex pattern
        """
        if pattern is None:
            pattern = self.allowable_chars

        pattern = re.sub(r"\[", r"[^", pattern)
        return pattern

    def generate_transition_matrix(self, text: str, initialized_dataframe:pd.DataFrame) -> pd.DataFrame:
        """Generates transition matrix from text

        Args:
            text (str): text to generate transition matrix from

        Returns:
            pd.DataFrame: transition matrix
        """

        #Iterate through each character in the text.
        for i in range(len(text) - 1):
            initialized_dataframe.loc[text[i], text[i+1]] += 1

        return initialized_dataframe

    def get_charset(self, text:str, sorted:bool = False) -> Union[set[str], list[str]]:
        """Gets the charset of the given text

        Args:
            text (str): text to get charset for

        Returns:
            set: charset
        """
        return_set = set(text)
        if sorted:
            return_set = list(return_set)
            return_set.sort()

        return return_set

    def generate_transition_matrix_from_text(self) -> None:
        """Generates transition matrix from text

        Returns:
            pd.DataFrame: transition matrix
        """
        #Check if the transition matrix has already been generated
        if hasattr(self, "transition_matrix"):
            return

        #Generate a set of all characters in the text then sort them
        self.charset    = self.get_charset(self.reference_text, sorted=True)

        #Create a dataframe with the characters as the columns and the characters as the rows
        transition_matrix = pd.DataFrame(0, index=self.charset, columns=self.charset)

        #Begin preparations for multiprocessing to speed up the process

        #Create a list of the text to be split up into chunks
        chunk_length = int(np.ceil(len(self.reference_text) / self.max_workers))

        text_chunks = [
            self.reference_text[i * chunk_length : (i + 1)  * chunk_length]
            for i in range(self.max_workers)
        ]

        #Create a list of the dataframes to be used for multiprocessing
        dataframes = [transition_matrix.copy() for _ in range(self.max_workers)]

        #Start multiprocessing
        with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
            dataframe_list = executor.map(self.generate_transition_matrix, text_chunks, dataframes)

        #Combine the dataframes
        for dataframe in dataframe_list:
            transition_matrix += dataframe

        #Add the transitions between the chunks
        for i in range(self.max_workers - 1):
            transition_matrix.loc[text_chunks[i][-1], text_chunks[i+1][0]] += 1

        self.transition_matrix = transition_matrix

        #Normalize the transition matrix so that each row sums to 1 and replace nan values with 0
        self.transition_matrix = self.transition_matrix.div(self.transition_matrix.sum(axis=1), axis=0).fillna(0)

        #Take the log of the transition matrix with a hard bottom of the lowest non-zero value in the matrix minus 20
        transition_matrix       = self.transition_matrix.to_numpy()
        nonzero_values          = transition_matrix[transition_matrix > 0]
        min_value               = np.log(np.min(nonzero_values))
        self.transition_matrix  = np.log(self.transition_matrix).clip(min_value - 20)

        return

    def find_cypher(self, text:str, max_iterations:int = 10_000) -> dict:
        """Finds the cypher for the given text

        Args:
            text (str): text to find cypher for
            max_iterations (int, optional): Maximum number of iterations to run. Defaults to 100.
            max_plausibility (float, optional): Maximum plausibility of the cypher. Defaults to 0.99.

        Returns:
            dict: cypher
        """
        self.text = text
        #Generate the transition matrix, protecting against multiprocess loops in Windows
        if __name__ == "__main__":
            self.generate_transition_matrix_from_text()

        text_charset = self.get_charset(text)

        #Generate a cypher, then calculate its plausibility
        cypher              = self.generate_map(text_charset)
        cypher.plausibility = self.calculate_plausibility(cypher)

        #Keep track of the best cypher found so far as per the plausibility. This will be the cypher that is returned
        self.best_cypher = cypher

        for i in range(max_iterations):
            #Generate the cypher based on the current cypher and calculate its plausibility
            new_cypher              = self.generate_map(text_charset, cypher.copy())
            new_cypher.plausibility = self.calculate_plausibility(new_cypher)

            #If the new cypher is better than the current best cypher, replace it
            if new_cypher.plausibility >= cypher.plausibility:
                cypher = new_cypher
                #If it's also better than the best cypher found so far, replace that too
                if cypher.plausibility >= self.best_cypher.plausibility:
                    self.best_cypher = cypher
            else:
                #If the new cypher is worse than the current cypher, flip a coin with acceptance P(new_cypher) / P(current_cypher)
                #If heads, replace the current cypher with the new one even if it's worse than the current cypher. This will
                #help the algorithm to break out of local minima
                uniform_value = np.random.uniform(0, 1)
                div = np.exp(new_cypher.plausibility - cypher.plausibility)
                if uniform_value < div:
                    cypher = new_cypher

            #Every 1000 iterations, print out the current best cypher
            if i % 1000 == 0:
                print(f"Iteration {i} - Plausibility: {self.best_cypher.plausibility}")
                print(self.best_cypher.decode(text))

        return self.best_cypher, self.best_cypher.decode(text)

    def calculate_plausibility(self, cypher_map:'Cypher') -> float:
        """Test the plausibility of the given map

        Args:
            map (dict): map to test

        Returns:
            float: plausibility of the map
        """
        #Calculate the plausibility of the cypher, starting as if the first character is a space
        plausability = 0
        text = " " + self.text

        for i, char in enumerate(text[:-1]):
            first_char      = cypher_map[char]
            second_char     = cypher_map[text[i+1]]
            plausability   += self.transition_matrix.loc[first_char, second_char]

        return plausability

    def generate_map(self, text_charset:set[str], cypher_map:Optional['Cypher'] = None) -> 'Cypher':
        """Generates or updates the map

        Args:
            map (Optional[dict], optional): map to update. Defaults to None.

        Returns:
            dict: map
        """
        #If the map is not given, generate a random map assigning each character to a random character without replacement
        if cypher_map is None:
            cypher_map = {}
            # #The initial charset should primarily focus on the text charset, but should also include the allowable charset
            charset = [char for char in text_charset] + [char for char in self.charset if char not in text_charset]
            charset_copy = [x for x in charset]
            for char in charset_copy:
                random_char = np.random.choice(charset)
                cypher_map[char] = random_char
                charset.remove(random_char)
            cypher_map = Cypher(cypher_map)
        else:
            #Otherwise make one swap by picking the first character from the charset and the second character from the cypher map
            char1           = np.random.choice(list(text_charset))
            cypher_keys     = [char for char in cypher_map.keys() if char != char1]
            char2           = np.random.choice(cypher_keys)

            cypher_map.swap_chars(char1, char2)

        return cypher_map

class Cypher:
    def __init__(self, cypher_map:dict[str, str], invert_map:bool = False) -> None:
        if invert_map:
            self.cypher_map = self.invert_map(cypher_map)
        else:
            self.cypher_map     = cypher_map
        self.plausibility   = None

    @staticmethod
    def invert_map(dictionary: dict) -> dict:
        return {v: k for k, v in dictionary.items()}

    def decode(self, text:str) -> str:
        """Encodes the given text using the cypher map

        Args:
            text (str): text to encode

        Returns:
            str: encoded text
        """
        encoded_text = ""
        for char in text:
            encoded_text += self.cypher_map[char]
        return encoded_text

    def encode(self, text:str) -> str:
        """Decodes the given text using the cypher map

        Args:
            text (str): text to decode

        Returns:
            str: decoded text
        """
        decoded_text = ""
        char_map = self.invert_map(self.cypher_map)
        for char in text:
            decoded_text += char_map[char]
        return decoded_text

    def __getitem__(self, char:str) -> str:
        return self.cypher_map[char]

    def __setitem__(self, char:str, value:str) -> None:
        self.cypher_map[char] = value

    def swap_chars(self, char1:str, char2:str) -> None:
        """Swaps the characters in the cypher map

        Args:
            char1 (str): first character
            char2 (str): second character
        """
        self.cypher_map[char1], self.cypher_map[char2] = self.cypher_map[char2], self.cypher_map[char1]

    def __eq__(self, other:'Cypher') -> bool:
        return self.cypher_map == other.cypher_map

    def copy(self) -> 'Cypher':
        """Returns a copy of the cypher

        Returns:
            Cypher: copy of the cypher
        """
        new_cypher = Cypher(self.cypher_map.copy())
        new_cypher.plausibility = self.plausibility
        return new_cypher

    def keys(self) -> list[str]:
        """Returns the keys of the cypher map

        Returns:
            list[str]: keys of the cypher map
        """
        return list(self.cypher_map.keys())

# %%
if __name__ == "__main__":
    dc = Decrypter(r"https://www.gutenberg.org/files/2600/2600-0.txt")
    encoded_text = 'g afpa ham gag afpag1qga7wag1paz9pwg7 mac1pg1phag7wam fnpha7mag1pa27mdag aw900phag1pawn7mxwaqmdaqhh cwa 0a 9ghqxp 9wa0 hg9mpa hag agqepaqh2waqxq7mwgaqawpqa 0agh 9fnpw' #Harder test text
    # encoded_text = "mg zt gk fgm mg zt miam ol mit jwtlmogf citmitk mol fgzstk of mit dofr mg lwyytk mit lsoful afr akkgcl gy gwmkautgwl ygkmwft gk mg maqt akdl auaoflm a lta gy mkgwzstl afr zb ghhglofu tfr mitd mg rot mg lstth" #Easy test text
    cypher, decoded_text = dc.find_cypher(encoded_text, max_iterations = 100_000, )
# %%
# dc.find_cypher(encoded_text)


import subprocess
import os
import re
import abc
import datetime

class BaseBackend(object):
    __metaclass__=abc.ABCMeta

    @abc.abstractmethod
    def create_tmp_file(self, version_dir):
        return

    @abc.abstractmethod
    def get_parsed_commit(self):
        return

class SVNBackend(BaseBackend):
    """
    svn log |grep [0-9] | trac
    """
    TEMP_FILENAME = 'svn_log_tmp.txt'
    DIR = '/tmp'

    def create_tmp_file(self, version_dir):
        os.chdir(version_dir)
        result = subprocess.\
            call("svn log | grep '^\\r[0-9]' | tac | awk '{print $3,$5}' > %s/%s"\
                % (self.DIR, self. TEMP_FILENAME), shell=True)

    def get_parsed_commit(self):
        """
        file format: 'dev  %Y-%m-%d'
        """
        f = open('%s/%s' % (self.DIR, self.TEMP_FILENAME), 'r')
        for line in f:
            info = line.split(' ')
            yield (info[0], datetime.datetime.strptime(re.sub(r'\n',
                                                      '', info[1]),
                                                      '%Y-%m-%d'))
        f.close()

class GitBackend(BaseBackend):
    """
    git log | egrep 'Author|Date' | sed '$!N;s/\n/ /
    """
    TEMP_FILENAME = 'git_log_tmp.txt'
    DIR = '/tmp'

    def create_tmp_file(self, version_dir):
        "git log | egrep 'Author|Date' | sed '$!N;s/\n/ /' | awk '{print $2,$6,$7,$9}'"
        os.chdir(version_dir)
        subprocess.\
            call("git log | egrep 'Author|Date'"
                    "| sed '$!N;s/\\n/ /' | "
                    "tac |"
                    "awk '{print $2,$6,$7,$9}' > %s/%s"
                    % (self.DIR, self.TEMP_FILENAME),
                    shell=True)

    def get_parsed_commit(self):
        f = open('%s/%s' % (self.DIR, self.TEMP_FILENAME), 'r')
        for line in f:
            info = line.split(' ')
            yield (info[0], datetime.datetime.strptime(
                re.sub('\n', '', ' '.join(info[1:])), '%b %d %Y'))
        f.close()


from django.urls import path
from .views import *

app_name = 'expenses'

urlpatterns = [

	# Admin Site URLs

    # Client Site URLs

    # Api Urls
    path('expense/list/', ExpenseListAPIView.as_view(), name="expenses"),
    path('expense/<int:id>/', ExpenseDetailAPIView.as_view(), name="expense"),

]


from django.conf import settings
from django.conf.urls import patterns, include, url
from django.conf.urls.static import static

from filebrowser.sites import site

from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns(
    '',
    url(r'^$', 'pmsal.core.views.home', name='home'),
    # url(r'^institucional/', 'canaa.core.views.institutional',
    #     name='institutional'),

    url(r'^a-prefeitura/', 'pmsal.core.views.city', name='city'),
    url(r'^contato/', 'pmsal.core.views.contact', name='contact'),
    url(r'^transparencia/', 'pmsal.core.views.transparency',
        name='transparency'),

    url(r'^prefeitura/(?P<slug>[-\w]+)/$', 'pmsal.core.views.content',
        name='prefeitura'),
    url(r'^imprensa/(?P<slug>[-\w]+)/$', 'pmsal.core.views.content',
        name='imprensa'),
    url(r'^secretaria/(?P<slug>[-\w]+)/$', 'pmsal.core.views.content',
        name='secretaria'),

    url(r'^agenda/', include('pmsal.event.urls', namespace='event')),
    url(r'^noticias/', include('pmsal.blog.urls', namespace='blog')),
    url(r'^licitacoes/', include('pmsal.bid.urls', namespace='bid')),

    url(r'^admin/filebrowser/', include(site.urls)),
    url(r'^grappelli/', include('grappelli.urls')),
    url(r'^admin/', include(admin.site.urls)),

    # tinymce
    url(r'^tinymce/', include('tinymce.urls')),
) + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
# )


import dataclasses
from typing import Any, Dict, List

from acme import core
from acme import types

from acme.multiagent import types as ma_types

LearnerState = Any

@dataclasses.dataclass
class SynchronousDecentralizedLearnerSetState:
  """State of a SynchronousDecentralizedLearnerSet."""
  # States of the learners keyed by their names.
  learner_states: Dict[ma_types.AgentID, LearnerState]

class SynchronousDecentralizedLearnerSet(core.Learner):
  """Creates a composed learner which wraps a set of local agent learners."""

  def __init__(self,
               learners: Dict[ma_types.AgentID, core.Learner],
               separator: str = '-'):
    """Initializer.

    Args:
      learners: a dict specifying the learners for all sub-agents.
      separator: separator character used to disambiguate sub-learner variables.
    """
    self._learners = learners
    self._separator = separator

  def step(self):
    for learner in self._learners.values():
      learner.step()

  def get_variables(self, names: List[str]) -> List[types.NestedArray]:
    """Return the named variables as a collection of (nested) numpy arrays.

    The variable names should be prefixed with the name of the child learners
    using the separator specified in the constructor, e.g. learner1/var.

    Args:
      names: args where each name is a string identifying a predefined subset of
        the variables. The variables names should be prefixed with the name of
        the learners using the separator specified in the constructor, e.g.
        learner-var if the separator is -.

    Returns:
      A list of (nested) numpy arrays `variables` such that `variables[i]`
      corresponds to the collection named by `names[i]`.
    """
    variables = []
    for name in names:
      # Note: if separator is not found, learner_name=name, which is OK.
      learner_id, _, variable_name = name.partition(self._separator)
      learner = self._learners[learner_id]
      variables.extend(learner.get_variables([variable_name]))
    return variables

  def save(self) -> SynchronousDecentralizedLearnerSetState:
    return SynchronousDecentralizedLearnerSetState(learner_states={
        name: learner.save() for name, learner in self._learners.items()
    })

  def restore(self, state: SynchronousDecentralizedLearnerSetState):
    for name, learner in self._learners.items():
      learner.restore(state.learner_states[name])


from django.test import TestCase
from django.utils.translation import override

from modeltrans.manager import transform_translatable_fields
from modeltrans.utils import (
    build_localized_fieldname,
    get_instance_field_value,
    get_language,
    get_model_field,
    split_translated_fieldname,
)

from .app.models import Blog, Category

class UtilsTest(TestCase):
    def test_get_language(self):
        self.assertEqual(get_language(), "en")

        with override("nl"):
            self.assertEqual(get_language(), "nl")

        with override("id"):
            self.assertEqual(get_language(), "en")

    def test_split_translated_fieldname(self):
        self.assertEqual(split_translated_fieldname("title_nl"), ("title", "nl"))

        self.assertEqual(split_translated_fieldname("full_name_nl"), ("full_name", "nl"))

    def test_transform_translatable_fields(self):
        self.assertEqual(
            transform_translatable_fields(Blog, {"title": "bar", "title_nl": "foo"}),
            {"i18n": {"title_nl": "foo"}, "title": "bar"},
        )

    def test_transform_translatable_fields_without_translations(self):
        self.assertEqual(
            transform_translatable_fields(Blog, {"title": "bar", "title_nl": "foo", "i18n": None}),
            {"i18n": {"title_nl": "foo"}, "title": "bar"},
        )

    def test_transform_translatable_fields_keep_translations(self):
        self.assertEqual(
            transform_translatable_fields(
                Blog, {"title": "bar", "title_de": "das foo", "i18n": {"title_nl": "foo"}}
            ),
            {"i18n": {"title_nl": "foo", "title_de": "das foo"}, "title": "bar"},
        )

    def test_build_localized_fieldname(self):
        self.assertEqual(build_localized_fieldname("title", "nl"), "title_nl")
        self.assertEqual(build_localized_fieldname("category__name", "nl"), "category__name_nl")
        self.assertEqual(build_localized_fieldname("title", "id"), "title_ind")
        self.assertEqual(build_localized_fieldname("title", "en-US"), "title_en_US")

    def test_get_model_field(self):
        with self.assertRaises(ValueError):
            get_model_field(object(), "name")
        self.assertEqual(get_model_field(Category, "name"), Category._meta.get_field("name"))
        self.assertEqual(get_model_field(Category, "color"), None)
        self.assertEqual(get_model_field(Blog, "category__name"), Category._meta.get_field("name"))
        self.assertEqual(get_model_field(Blog, "category__color"), None)

    def test_get_instance_field_value(self):
        test = Category(name="test")
        blog = Blog(category=test, title="Python")

        self.assertEqual(get_instance_field_value(Category(), "content"), None)
        self.assertEqual(get_instance_field_value(test, "name"), "test")
        self.assertEqual(get_instance_field_value(blog, "category__name"), "test")
        self.assertEqual(get_instance_field_value(blog, "category__color"), None)


from copy import deepcopy
from pathlib import Path

class FileFamily():
    def __init__(self):
        r'''A file family collects multiple asset names sharing the same path.
        '''
        self.path = b""
        self.assets = []

    def set_path(self, path):
        self.path = path

    def add_asset(self, fname):
        if fname not in self.assets:
            self.assets.append(fname)

    def remove_asset(self, fname):
        try:
            self.assets.remove(fname)
            return 1
        except:
            return 0

class FileFamilyManager():
    def __init__(self, data, parent_path=None):
        r'''A file family manager initialises and manages a group of file families.
        Pass in a list of Path objects and a parent Path object
        A parent path is a shared path of all file families if there is one
        '''
        self.file_families = []
        self.parent_path = parent_path

        template = FileFamily()

        for path in data:
            try:
                item = Path(path).relative_to(self.parent_path) #everything after parent path
            except:
                item = Path(path)
            is_path = False
            for ff in self.file_families:
                if item.parent == ff.path:
                    ff.add_asset(item.name)
                    is_path = True
            if not is_path:
                ffobj = deepcopy(template)
                ffobj.set_path(item.parent)
                ffobj.add_asset(item.name)
                self.file_families.append(ffobj)

    def get_num_assets(self):
        numassets = 0
        for family in self.file_families:
            for asset in family.assets:
                numassets += 1
        return numassets

    def remove_asset(self, fpath):
        try:
            item = fpath.relative_to(self.parent_path)
        except:
            item = fpath

        for ff in self.file_families:
            if item.parents[0] == ff.path:
                ff.remove_asset(item.name)


from django.contrib.auth import get_user_model
from django.shortcuts import get_object_or_404
from rest_framework import serializers
from .models import Referral
from .referral import attach_referral

#------------------------------------------------------------------------------

class ReferralSerializer(serializers.ModelSerializer):
    referree = serializers.CharField(source='referree.username', read_only=True)
    referrer = serializers.CharField(source='referrer.username')
    ip_address = serializers.CharField(source='referree_ip_address', required=False)

    class Meta:
        model = Referral
        fields = [
            'id',
            'referrer',
            'referree',
            'ip_address'
        ]
        read_only_fields = [
            'id',
        ]

        def create(self, validated_data, *args, **kwargs):
            user = None
            request = self.context.get('request', None)
            if request and hasattr(request, 'user'):
                user = request.user
            else:
                # serializer validation error
                raise Exception("Could not identify the user in request.")

            referrer_username = validated_data.pop('referrer')['username']
            referrer = get_object_or_404(get_user_model(), username=referrer_username)

            # create instance
            instance = attach_referral(referrer=referrer, referree=user, **validated_data)

            return instance

        def update(self, instance, validated_data, *args, **kwargs):
            # serviceunavailableerror
            raise Exception("Cannot edit referrals")


import tensorflow as tf
import numpy as np
from baselines.common.mpi_adam_optimizer import MpiAdamOptimizer
from baselines.common.tf_util import get_session, save_variables, load_variables

from mpi4py import MPI
from baselines.common.tf_util import initialize
from baselines.common.mpi_util import sync_from_root

# Helper function to build convnets
def conv2d(inputs, filters, kernel_size, strides, padding):
        return tf.layers.conv2d(inputs = inputs,
                           filters = filters,
                            kernel_size = (kernel_size, kernel_size),
                            strides = strides,
                            padding = padding)

class ICM(object):
    def __init__(self, ob_space, ac_space, max_grad_norm, beta, icm_lr_scale):

        sess = get_session()

        #TODO find a better way
        input_shape = [ob_space.shape[0], ob_space.shape[1], ob_space.shape[2]]
        self.action_shape = 36

        # Placeholders
        self.state_ = phi_state = tf.placeholder(tf.float32, [None, *input_shape], name="icm_state")
        self.next_state_ = phi_next_state =  tf.placeholder(tf.float32, [None, *input_shape], name="icm_next_state")
        self.action_ = action = tf.placeholder(tf.float32, [None], name="icm_action")

        with tf.variable_scope('icm_model'):
            # Feature encoding
            # Aka pass state and next_state to create phi(state), phi(next_state)
            # state --> phi(state)
            phi_state = self.feature_encoding(self.state_)

            with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
                # next_state to phi(next_state)
                phi_next_state = self.feature_encoding(self.next_state_)

            # INVERSE MODEL
            pred_actions_logits, pred_actions_prob = self.inverse_model(phi_state, phi_next_state)

            # FORWARD MODEL
            pred_phi_next_state = self.forward_model(action, phi_state)

        # CALCULATE THE ICM LOSS
        # Inverse Loss LI
        # We calculate the cross entropy between our ât and at
        # Squeeze the labels (required)
        labels = tf.cast(action, tf.int32)

        self.inv_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred_actions_logits, labels=labels), name="inverse_loss")

        # Foward Loss
        # LF = 1/2 || pred_phi_next_state - phi_next_state ||
        # TODO 0.5 * ?
        self.forw_loss = tf.reduce_mean(tf.square(tf.subtract(pred_phi_next_state, phi_next_state)), name="forward_loss")

        # Todo predictor lr scale ?
        # ICM_LOSS = [(1 - beta) * LI + beta * LF ] * Predictor_Lr_scale
        self.icm_loss = ((1-beta) * self.inv_loss + beta * self.forw_loss) * icm_lr_scale

        # UPDATE THE PARAMETERS USING LOSS
        # 1. Get the model parameters
        icm_params = tf.trainable_variables('icm_model')
        # 2. Build our trainer
        icm_trainer = MpiAdamOptimizer(MPI.COMM_WORLD, learning_rate=1e-4, epsilon=1e-5)
        # 3. Calculate the gradients
        icm_grads_and_var = icm_trainer.compute_gradients(self.icm_loss, icm_params)
        icm_grads, icm_var = zip(*icm_grads_and_var)

        if max_grad_norm is not None:
            # Clip the gradients (normalize)
            icm_grads, icm__grad_norm = tf.clip_by_global_norm(icm_grads, max_grad_norm)
        icm_grads_and_var = list(zip(icm_grads, icm_var))
        # zip aggregate each gradient with parameters associated
        # For instance zip(ABCD, xyza) => Ax, By, Cz, Da

        _icm_train = icm_trainer.apply_gradients(icm_grads_and_var)

        if MPI.COMM_WORLD.Get_rank() == 0:
            print("Initialize")
            initialize()
        global_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="")
        print("GLOBAL VARIABLES", global_variables)
        sync_from_root(sess, global_variables) #pylint: disable=E1101

    # We use batch normalization to do feature normalization as explained in the paper
    def feature_encoding(self, x):
        x = tf.nn.elu(tf.layers.batch_normalization(conv2d(x, 8, 5, 4, "valid")))
        x = tf.nn.elu(tf.layers.batch_normalization(conv2d(x, 16, 3, 2, "valid")))
        x = tf.nn.elu(tf.layers.batch_normalization(conv2d(x, 32, 3, 2, "valid")))
        x = tf.nn.elu(tf.layers.batch_normalization(conv2d(x, 64, 3, 2, "valid")))
        x = tf.layers.flatten(x)
        x = tf.nn.elu(tf.contrib.layers.fully_connected(x, 256))

        return x

    # Inverse Model
    # Given phi(state) and phi(next_state) returns the predicted action ât
    """
    Parameters
    __________

    action:   The real action taken by our agent
    phi_state: The feature representation of our state generated by our feature_encoding function.
    phi_next_state: The feature representation of our next_state generated by our feature_encoding function.

    returns pred_actions_logits: the logits and pred_actions_prob: the probability distribution of our actions
    """
    def inverse_model(self, phi_state, phi_next_state):
        # Concatenate phi(st) and phi(st+1)
        icm_inv_concatenate = tf.concat([phi_state, phi_next_state], 1)
        icm_inv_fc1 = tf.nn.relu(tf.layers.dense(icm_inv_concatenate, 256))
        pred_actions_logits = tf.layers.dense(icm_inv_fc1, self.action_shape)
        pred_actions_prob = tf.nn.softmax(pred_actions_logits, dim=-1)

        return pred_actions_logits, pred_actions_prob

    # Foward Model
    # Given action and phi(st) must find pred_phi(st+1)
    """
    Parameters
    __________

    action:   The action taken by our agent
    phi_state: The feature representation of our state generated by our feature_encoding function.
    phi_next_state: The feature representation of our next_state generated by our feature_encoding function.

    returns pred_phi_next_state: The feature representation prediction of our next_state.
    """
    def forward_model(self, action, phi_state):
        # Concatenate phi_state and action
        action = tf.expand_dims(action, axis=1) # Expand dimension to be able to concatenate

        icm_forw_concatenate = tf.concat(axis=1, values=[phi_state, action])

        # FC
        icm_forw_fc1 = tf.layers.dense(icm_forw_concatenate, 256)

        # FC (size of phi_state [1] aka the width)
        icm_forw_pred_next_state = tf.layers.dense(icm_forw_fc1, phi_state.get_shape()[1].value)

        return icm_forw_pred_next_state

    # Calculate intrinsic reward
    """
    Parameters
    __________

    phi_next_state: The feature representation of our next_state generated by our feature_encoding function.
    pred_phi_next_state:   The feature representation prediction of our next_state.

    returns intrinsic_reward: The intrinsic reward
    """
    def calculate_intrinsic_reward(self, state, next_state, action):

        sess = tf.get_default_session()

        error = sess.run(self.forw_loss, {self.state_: state, self.next_state_: next_state, self.action_: action})

        error = error * 0.5

        # Return intrinsic reward
        return error

"""

   Need implement train function

"""