#vaemodel
import copy
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.autograd as autograd
from torch.utils import data
from data_loader import DATA_LOADER as dataloader
from dataloader import Flickr30k as dataLoader
import final_classifier as  classifier
import models
from torchvision import models as torchModels
from sklearn.neighbors import NearestNeighbors
from sklearn.manifold import TSNE
from scipy.spatial import distance
import numpy as np

import matplotlib.pyplot as plt

class LINEAR_LOGSOFTMAX(nn.Module):
    def __init__(self, input_dim, nclass):
        super(LINEAR_LOGSOFTMAX, self).__init__()
        self.fc = nn.Linear(input_dim,nclass)
        self.logic = nn.LogSoftmax(dim=1)
        self.lossfunction =  nn.NLLLoss()

    def forward(self, x):
        o = self.logic(self.fc(x))
        return o

class Model(nn.Module):

    def __init__(self,hyperparameters):
        super(Model,self).__init__()

        self.device = hyperparameters['device']
        self.auxiliary_data_source = hyperparameters['auxiliary_data_source']
        self.attr = hyperparameters['attr']
        self.all_data_sources  = ['resnet_features', 'attributes']
        self.DATASET = hyperparameters['dataset']
        self.num_shots = hyperparameters['num_shots']
        self.latent_size = hyperparameters['latent_size']
        self.batch_size = hyperparameters['batch_size']
        self.hidden_size_rule = hyperparameters['hidden_size_rule']
        self.warmup = hyperparameters['model_specifics']['warmup']
        self.generalized = hyperparameters['generalized']
        self.classifier_batch_size = 32
        #self.img_seen_samples   = hyperparameters['samples_per_class'][self.DATASET][0]
        #self.att_seen_samples   = hyperparameters['samples_per_class'][self.DATASET][1]
        #self.att_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][2]
       # self.img_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][3]
        self.reco_loss_function = hyperparameters['loss']
        self.margin = hyperparameters['margin_loss']
        self.nepoch = hyperparameters['epochs']
        self.lr_cls = hyperparameters['lr_cls']
        self.cross_reconstruction = hyperparameters['model_specifics']['cross_reconstruction']
        self.cls_train_epochs = hyperparameters['cls_train_steps']
        #self.dataset = dataloader(self.DATASET, copy.deepcopy(self.auxiliary_data_source) , device= 'cuda')
        self.dataset = dataLoader(copy.deepcopy(self.auxiliary_data_source) , device= 'cuda', attr = self.attr)
        if self.DATASET=='CUB':
            self.num_classes=200
            self.num_novel_classes = 50
        elif self.DATASET=='SUN':
            self.num_classes=717
            self.num_novel_classes = 72
        elif self.DATASET=='AWA1' or self.DATASET=='AWA2':
            self.num_classes=50
            self.num_novel_classes = 10

        if self.attr == 'attributes':
            feature_dimensions = [2048, self.dataset.K]
        elif self.attr == 'bert':
            feature_dimensions = [2048, 768] #2048, 768

        # Here, the encoders and decoders for all modalities are created and put into dict

        self.fc_ft = nn.Linear(2048,2048)
        self.fc_ft.to(self.device)

        self.ft_bn = nn.BatchNorm1d(2048).to(self.device)

        self.fc_at = nn.Linear(self.dataset.K, self.dataset.K)
        self.fc_at.to(self.device)
        self.at_bn = nn.BatchNorm1d(self.dataset.K).to(self.device)

        self.encoder = {}

        for datatype, dim in zip(self.all_data_sources,feature_dimensions):

            self.encoder[datatype] = models.encoder_template(dim,self.latent_size,self.hidden_size_rule[datatype],self.device)

            print(str(datatype) + ' ' + str(dim))

        self.decoder = {}
        for datatype, dim in zip(self.all_data_sources,feature_dimensions):
            self.decoder[datatype] = models.decoder_template(self.latent_size,dim,self.hidden_size_rule[datatype],self.device)

        # An optimizer for all encoders and decoders is defined here
        parameters_to_optimize = list(self.parameters())
        for datatype in self.all_data_sources:
            parameters_to_optimize +=  list(self.encoder[datatype].parameters())
            parameters_to_optimize +=  list(self.decoder[datatype].parameters())
        parameters_to_optimize += list(self.fc_ft.parameters())
        parameters_to_optimize += list(self.fc_at.parameters())
        parameters_to_optimize += list(self.ft_bn.parameters())
        parameters_to_optimize += list(self.at_bn.parameters())

        self.optimizer  = optim.Adam( parameters_to_optimize ,lr=hyperparameters['lr_gen_model'], betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True)

        if self.reco_loss_function=='l2':
            self.reconstruction_criterion = nn.MSELoss(size_average=False)

        elif self.reco_loss_function=='l1':
            self.reconstruction_criterion = nn.L1Loss(size_average=False)

    def reparameterize(self, mu, logvar):
        if self.reparameterize_with_noise:
            sigma = torch.exp(logvar)
            eps = torch.cuda.FloatTensor(logvar.size()[0],1).normal_(0,1)
            eps  = eps.expand(sigma.size())
            return mu + sigma*eps
        else:
            return mu

    def forward(self):
        pass

    def map_label(self,label, classes):
        mapped_label = torch.LongTensor(label.size()).to(self.device)
        for i in range(classes.size(0)):
            mapped_label[label==classes[i]] = i

        return mapped_label

    def trainstep(self, img, att):

        ##############################################
        # Encode image features and additional
        # features
        ##############################################

        img_in = F.normalize(img, p=2, dim=1)
        img_in = self.ft_bn(img_in)

        img_in = self.fc_ft(img_in)

        att_in = F.normalize(att, p=2, dim=1)
        att_in = self.at_bn(att_in)

        att_in = self.fc_at(att_in)

        #Add non-linearity?

        mu_img, logvar_img = self.encoder['resnet_features'](img_in)
        z_from_img = self.reparameterize(mu_img, logvar_img)

        mu_att, logvar_att = self.encoder[self.auxiliary_data_source](att_in)
        z_from_att = self.reparameterize(mu_att, logvar_att)

        ##############################################
        # Reconstruct inputs
        ##############################################

        img_from_img = self.decoder['resnet_features'](z_from_img)
        att_from_att = self.decoder[self.auxiliary_data_source](z_from_att)

        reconstruction_loss = self.reconstruction_criterion(img_from_img, img) \
                              + self.reconstruction_criterion(att_from_att, att)

        ##############################################
        # Cross Reconstruction Loss
        ##############################################
        img_from_att = self.decoder['resnet_features'](z_from_att)
        att_from_img = self.decoder[self.auxiliary_data_source](z_from_img)

        cross_reconstruction_loss = self.reconstruction_criterion(img_from_att, img) \
                                    + self.reconstruction_criterion(att_from_img, att)

        ##############################################
        # KL-Divergence
        ##############################################

        KLD = (0.5 * torch.sum(1 + logvar_att - mu_att.pow(2) - logvar_att.exp())) \
              + (0.5 * torch.sum(1 + logvar_img - mu_img.pow(2) - logvar_img.exp()))

        ##############################################
        # Distribution Alignment
        ##############################################
        distance = torch.sqrt(torch.sum((mu_img - mu_att) ** 2, dim=1) + \
                              torch.sum((torch.sqrt(logvar_img.exp()) - torch.sqrt(logvar_att.exp())) ** 2, dim=1))

        distance = distance.sum()

        #distanceI = torch.sum((mu_img - mu_att) ** 2, dim=1)
        #distanceT = torch.sum((mu_att - mu_img) ** 2, dim=1)

        #tripletsI = []
        #tripletsT = []

        #for i in range(0, mu_img.shape[0]):
        #    for j in range(0, mu_img.shape[0]):
        #        if i != j:
        #            distI = distanceI[i] - torch.sum((mu_img[i] - mu_att[j]) ** 2) + self.margin
        #            distI = torch.max(torch.FloatTensor([0]), distI)
        #            distI += torch.sum((torch.sqrt(logvar_img[i].exp()) - torch.sqrt(logvar_att[i].exp())) ** 2, dim=0)
        #            distI = torch.sqrt(distI)
        #            tripletsI.append(distI)

        #            distT = distanceT[i] - torch.sum((mu_att[i] - mu_img[j]) ** 2) + self.margin
        #            distT = torch.max(torch.FloatTensor([0]), distT)
        #            distT += torch.sum((torch.sqrt(logvar_att[i].exp()) - torch.sqrt(logvar_img[i].exp())) ** 2, dim=0)
        #            distT = torch.sqrt(distT)
        #            tripletsT.append(distT)

        #tripletI2t = sum(tripletsI)
        #tripletT2i = sum(tripletsT)

        #distance = tripletI2t + tripletT2i

        #distance = torch.FloatTensor([distance]).to(self.device)

        ##############################################
        # scale the loss terms according to the warmup
        # schedule
        ##############################################

        f1 = 1.0*(self.current_epoch - self.warmup['cross_reconstruction']['start_epoch'] )/(1.0*( self.warmup['cross_reconstruction']['end_epoch']- self.warmup['cross_reconstruction']['start_epoch']))
        f1 = f1*(1.0*self.warmup['cross_reconstruction']['factor'])
        cross_reconstruction_factor = torch.cuda.FloatTensor([min(max(f1,0),self.warmup['cross_reconstruction']['factor'])])

        f2 = 1.0 * (self.current_epoch - self.warmup['beta']['start_epoch']) / ( 1.0 * (self.warmup['beta']['end_epoch'] - self.warmup['beta']['start_epoch']))
        f2 = f2 * (1.0 * self.warmup['beta']['factor'])
        beta = torch.cuda.FloatTensor([min(max(f2, 0), self.warmup['beta']['factor'])])

        f3 = 1.0*(self.current_epoch - self.warmup['distance']['start_epoch'] )/(1.0*( self.warmup['distance']['end_epoch']- self.warmup['distance']['start_epoch']))
        f3 = f3*(1.0*self.warmup['distance']['factor'])
        distance_factor = torch.cuda.FloatTensor([min(max(f3,0),self.warmup['distance']['factor'])])

        ##############################################
        # Put the loss together and call the optimizer
        ##############################################

        self.optimizer.zero_grad()

        loss = reconstruction_loss - beta * KLD

        if cross_reconstruction_loss>0:
           loss += cross_reconstruction_factor*cross_reconstruction_loss
        if distance_factor >0:
           loss += distance_factor*distance

        loss.backward()

        self.optimizer.step()

        return loss.item(), reconstruction_loss, beta*KLD, cross_reconstruction_factor*cross_reconstruction_loss, distance_factor*distance

    def train_vae(self):

        elosses = []
        lossesR = []
        lossesK = []
        lossesC = []
        lossesD = []

        self.dataloader = data.DataLoader(self.dataset,batch_size=self.batch_size,shuffle= True,drop_last=True)#,num_workers = 4)

        #self.dataset.novelclasses =self.dataset.novelclasses.long().cuda()
        #self.dataset.seenclasses =self.dataset.seenclasses.long().cuda()

        #leave both statements
        self.train()
        self.fc_ft.train()
        self.fc_at.train()
        self.reparameterize_with_noise = True

        metricsI = []
        metricsT = []

        print('train for reconstruction')
        for epoch in range(0, self.nepoch ):
            self.train()

            self.current_epoch = epoch

            losses = []
            ilossesR = []
            ilossesK = []
            ilossesC = []
            ilossesD = []

            i=-1
            y = 0
            for iters in range(0, len(self.dataset), self.batch_size):

            #for iters in range(0, 1000, self.batch_size):
                i+=1

                features, attributes, idxs = self.dataset.next_batch(self.batch_size) #Si no és test treure la y

                data_from_modalities = [features, attributes.type(torch.FloatTensor)]

                for j in range(len(data_from_modalities)):
                    data_from_modalities[j] = data_from_modalities[j].to(self.device)
                    data_from_modalities[j].requires_grad = False

                loss, lossR, lossK, lossC, lossD = self.trainstep(data_from_modalities[0], data_from_modalities[1] )

                if i%10==0:

                    print('epoch ' + str(epoch) + ' | iter ' + str(i) + '\t'+
                    ' | loss ' +  str(loss))

                losses.append(loss)
                ilossesR.append(lossR)
                ilossesK.append(lossK)
                ilossesC.append(lossC)
                ilossesD.append(lossD)

                idxs = idxs.cpu()

                attributes = attributes.cpu()

                y += 1

            y = 0

            mean_loss = sum(losses)/len(losses)
            elosses.append(mean_loss)
            print('epoch ' + str(epoch) + 'Loss: ' + str(loss))

            lossesR.append(sum(ilossesR)/len(ilossesR))
            lossesK.append(sum(ilossesK)/len(ilossesK))
            lossesC.append(sum(ilossesC)/len(ilossesC))
            lossesD.append(sum(ilossesD)/len(ilossesD))

            for j in range(len(data_from_modalities)):
                    data_from_modalities[j] = data_from_modalities[j].cpu()

            print('Generating gallery set...')
            self.generate_gallery()

            print('Generating t-SNE plot...')
            z_imgs_embedded = TSNE(n_components=2).fit_transform(self.gallery_imgs_z.clone().cpu().detach())
            z_attrs_embedded = TSNE(n_components=2).fit_transform(self.gallery_attrs_z.clone().cpu().detach())

            plt.scatter(z_imgs_embedded[:,0], z_imgs_embedded[:,1], c = 'red')
            plt.scatter(z_attrs_embedded[:,0], z_attrs_embedded[:,1], c = 'blue')
            filename = 't-sne-plot-epoch'+str(epoch)+'.png'
            plt.savefig(filename)
            plt.clf()

            print('Evaluating retrieval...')
            metricsIepoch, metricsTepoch = self.retrieval()
            metricsI.append(metricsIepoch)
            metricsT.append(metricsTepoch)

            print('Evaluation Metrics for image retrieval')
            print("R@1: {}, R@5: {}, R@10: {}, R@50: {}, R@100: {}, MEDR: {}, MEANR: {}".format(metricsIepoch[0], metricsIepoch[1], metricsIepoch[2], metricsIepoch[3], metricsIepoch[4], metricsIepoch[5], metricsIepoch[6]))
            print('Evaluation Metrics for caption retrieval')
            print("R@1: {}, R@5: {}, R@10: {}, R@50: {}, R@100: {}, MEDR: {}, MEANR: {}".format(metricsTepoch[0], metricsTepoch[1], metricsTepoch[2], metricsTepoch[3], metricsTepoch[4], metricsTepoch[5], metricsTepoch[6]))

        # turn into evaluation mode:
        for key, value in self.encoder.items():
            self.encoder[key].eval()
        for key, value in self.decoder.items():
            self.decoder[key].eval()

        #Plot de les losses i desar-lo
        plt.xlabel('Epoch')
        plt.ylabel('Loss')
        plt.plot(np.arange(self.nepoch), elosses)
        plt.plot(np.arange(self.nepoch), lossesR)
        plt.plot(np.arange(self.nepoch), lossesK)
        plt.plot(np.arange(self.nepoch), lossesC)
        plt.plot(np.arange(self.nepoch), lossesD)
        plt.legend()
        plt.show()
        plt.savefig('losses.png')
        plt.clf()

        #Plot de les metrics
        plt.xlabel('Epoch')
        plt.ylabel('Metric')
        plt.plot(np.arange(self.nepoch), metricsI)
        plt.plot(np.arange(self.nepoch), metricsT)
        plt.legend()
        plt.show()
        plt.savefig('metrics.png')
        plt.clf()

        return losses, metricsI, metricsT

    def retrieval(self):

        self.eval()

        def lda(self, x, y):
            distance = torch.sqrt(torch.sum((x[0] - y[0]) ** 2, dim=1) + \
                              torch.sum((torch.sqrt(x[1].exp()) - torch.sqrt(y[1].exp())) ** 2, dim=1))

            return distance

        #nbrsI = NearestNeighbors(n_neighbors=self.dataset.ntest, algorithm='auto').fit(self.gallery_imgs_z.cpu().detach().numpy())
        #nbrsI = NearestNeighbors(n_neighbors=1000, algorithm='auto').fit(self.gallery_imgs_z.cpu().detach().numpy())
        #nbrsT = NearestNeighbors(n_neighbors=self.dataset.ntest, algorithm='auto').fit(self.gallery_attrs_z.cpu().detach().numpy())
        #nbrsT = NearestNeighbors(n_neighbors=5000, algorithm='auto').fit(self.gallery_attrs_z.cpu().detach().numpy())

        distI_dict = {}
        distT_dict = {}

        ranksI = np.zeros((1,5*self.dataset.ntest))
        ranksT = np.zeros((1,self.dataset.ntest))

        for i in range(0, self.dataset.ntest):
        #for i in range(0, 500):

            mu_img = self.gallery_imgs_z[i,:].unsqueeze(0)
            mu_att = self.gallery_attrs_z[5*i:5*i + 5,:]

            #Add L2 norm and BatchNorm?
            #im_ft = F.normalize(im_ft, p=2, dim = 1)
            #im_ft = self.ft_bn(im_ft)

            #im_ft = self.fc_ft(im_ft)

            #attr = F.normalize(attr.type(torch.FloatTensor).to(self.device), p=2, dim=1)
            #attr = self.at_bn(attr)

            #attr = self.fc_at(attr)

            #mu_img, logvar_img = self.encoder['resnet_features'](im_ft)
            #z_from_img = self.reparameterize(mu_img, logvar_img)

            #mu_att, logvar_att = self.encoder['attributes'](attr.type(torch.FloatTensor).to(self.device))
            #z_from_att = self.reparameterize(mu_att, logvar_att)

            #img = [mu_img.cpu().detach().numpy(), logvar_img.cpu().detach().numpy()]
            #att = [mu_att.cpu().detach().numpy(), logvar_att.cpu().detach().numpy()]

            #distancesI, indicesI = nbrsI.kneighbors(mu_att.cpu().detach().numpy())
            #distancesT, indicesT = nbrsT.kneighbors(mu_img.cpu().detach().numpy())

            distancesI = distance.cdist(mu_att.cpu().detach().numpy(), self.gallery_imgs_z.cpu().detach().numpy(), 'correlation')
            distancesT = distance.cdist(mu_img.cpu().detach().numpy(), self.gallery_attrs_z.cpu().detach().numpy(), 'correlation')

            indicesI = np.argsort(distancesI)
            indicesT = np.argsort(distancesT[0,:])

            #Trobar l'índex de la distància mínima entre cada parella.

            for z in range(0,5):
                if len(indicesI[z] == i) != 0:
                    ranksI[:,(5*i) + z] = np.where(indicesI[z] == i)[0][0]
                else:
                    ranksI[:,(5*i) + z] = 1000

            if len(np.where((indicesT >= 5*i) & (indicesT <= ((5*i) + 4)))) != 0:
                ranksT[:,i] = np.where((indicesT >= 5*i) & (indicesT <= ((5*i) + 4)))[0][0]
            else:
                ranksT[:,i] = 1000

            '''
            for z in range(0,5):
                if len(np.where((indicesI[z] >= 5*i) & (indicesI[z] <= (5*i+4)))[0]) != 0:
                        ranksI[:,5*i + z] = np.where((indicesI[z] >= 5*i) & (indicesI[z] <= (5*i+4)))[0][0]
                else:
                    ranksI[:,5*i + z] = 5000

            if len(np.where((indicesT[0] > (5*i-1)) & (indicesT[0] < (5*i+5)))) != 0:
                ranksT[:,i] = np.where((indicesT[0] > (5*i-1)) & (indicesT[0] < (5*i+5)))[0][0]
            else:
                ranksT[:,i] = 1000

            '''
            '''
            for z in range(0,5):
                if len(np.where((indicesT[z] >= 5*i) & (indicesT[z] <= (5*i+4)))[0]) != 0:
                        ranksT[:,5*i + z] = np.where((indicesT[z] >= 5*i) & (indicesT[z] <= (5*i+4)))[0][0]
                else:
                    ranksT[:,5*i + z] = 5000

            if len(np.where(indicesI[0] == i)) != 0:
                ranksI[:,i] = np.where(indicesI[0] == i)[0][0]
            else:
                ranksI[:,i] = 1000

            '''

        r1im = 100.0 * len(np.where(ranksI < 1)[1]) / len(ranksI[0,:])
        r5im = 100.0 * len(np.where(ranksI < 5)[1]) / len(ranksI[0,:])
        r10im = 100.0 * len(np.where(ranksI < 10)[1]) / len(ranksI[0,:])
        r50im = 100.0 * len(np.where(ranksI < 50)[1]) / len(ranksI[0,:])
        r100im = 100.0 * len(np.where(ranksI < 100)[1]) / len(ranksI[0,:])

        r1t = 100.0 * len(np.where(ranksT < 1)[1]) / len(ranksT[0,:])
        r5t = 100.0 * len(np.where(ranksT < 5)[1]) / len(ranksT[0,:])
        r10t = 100.0 * len(np.where(ranksT < 10)[1]) / len(ranksT[0,:])
        r50t = 100.0 * len(np.where(ranksT < 50)[1]) / len(ranksT[0,:])
        r100t = 100.0 * len(np.where(ranksT < 100)[1]) / len(ranksT[0,:])

        medrI = np.floor(np.median(ranksI)) + 1
        meanrI = ranksI.mean() + 1

        medrT = np.floor(np.median(ranksT)) + 1
        meanrT = ranksT.mean() + 1

        metricsI = [r1im, r5im, r10im, r50im, r100im, medrI, meanrI]
        metricsT = [r1t, r5t, r10t, r50t, r100t, medrT, meanrT]

        return metricsI, metricsT

    def generate_gallery(self):

        self.eval()

        z_imgs = []
        z_vars_im = []
        z_attrs = []
        z_vars_att = []
        rec_imgs = []
        rec_attrs = []

        y = 0
        i=-1
        for iters in range(0, self.dataset.ntest, 50):
        #for iters in range(0, 500, 50):
            i+=1

            features, attributes, idxs = self.dataset.next_batch_test(50, y)
            idxs = idxs.cpu()

            data_from_modalities = [features, attributes.type(torch.FloatTensor)]

            for j in range(len(data_from_modalities)):
                data_from_modalities[j] = data_from_modalities[j].to(self.device)
                data_from_modalities[j].requires_grad = False
                if j== 0:
                    #Add L2 norm and BatchNorm?
                    data_from_modalities[j] = F.normalize(data_from_modalities[j], p=2, dim=1)
                    data_from_modalities[j] = self.ft_bn(data_from_modalities[j])
                    data_from_modalities[j] = self.fc_ft(data_from_modalities[j])
                elif j == 1:
                    #Add L2 norm and BatchNorm?
                    data_from_modalities[j] = F.normalize(data_from_modalities[j], p=2, dim=1)
                    data_from_modalities[j] = self.at_bn(data_from_modalities[j])
                    data_from_modalities[j] = self.fc_at(data_from_modalities[j])

            mu_img, logvar_img = self.encoder['resnet_features'](data_from_modalities[0])
            z_from_img = self.reparameterize(mu_img, logvar_img)

            mu_att, logvar_att = self.encoder['attributes'](data_from_modalities[1])
            z_from_att = self.reparameterize(mu_att, logvar_att)

            if y == 0:
                z_imgs = mu_img.cpu()
                z_vars_im = logvar_img.cpu()
                z_attrs = mu_att.cpu()
                z_vars_att = logvar_att.cpu()
            else:
                z_imgs = torch.cat((z_imgs.cpu(),mu_img.cpu()), dim = 0).cpu()
                z_vars_im = torch.cat((z_vars_im.cpu(),logvar_img.cpu()), dim = 0).cpu()
                z_attrs = torch.cat((z_attrs.cpu(),mu_att.cpu()), dim = 0).cpu()
                z_vars_att = torch.cat((z_vars_att.cpu(),logvar_att.cpu()), dim = 0).cpu()

            y = y + 1

            print('iter: '+str(iters))

        self.gallery_imgs_z = z_imgs.cpu()
        self.gallery_vars_im = z_vars_im.cpu()
        print(self.gallery_imgs_z.size())
        self.gallery_attrs_z = z_attrs.cpu()
        self.gallery_vars_att = z_vars_att.cpu()
        print(self.gallery_attrs_z.size())

    def train_classifier(self, show_plots=False):

        if self.num_shots > 0 :
            print('================  transfer features from test to train ==================')
            self.dataset.transfer_features(self.num_shots, num_queries='num_features')

        history = []  # stores accuracies

        cls_seenclasses = self.dataset.seenclasses
        cls_novelclasses = self.dataset.novelclasses

        train_seen_feat = self.dataset.data['train_seen']['resnet_features']
        train_seen_label = self.dataset.data['train_seen']['labels']

        novelclass_aux_data = self.dataset.novelclass_aux_data  # access as novelclass_aux_data['resnet_features'], novelclass_aux_data['attributes']
        seenclass_aux_data = self.dataset.seenclass_aux_data

        novel_corresponding_labels = self.dataset.novelclasses.long().to(self.device)
        seen_corresponding_labels = self.dataset.seenclasses.long().to(self.device)

        # The resnet_features for testing the classifier are loaded here
        novel_test_feat = self.dataset.data['test_unseen'][
            'resnet_features']  # self.dataset.test_novel_feature.to(self.device)
        seen_test_feat = self.dataset.data['test_seen'][
            'resnet_features']  # self.dataset.test_seen_feature.to(self.device)
        test_seen_label = self.dataset.data['test_seen']['labels']  # self.dataset.test_seen_label.to(self.device)
        test_novel_label = self.dataset.data['test_unseen']['labels']  # self.dataset.test_novel_label.to(self.device)

        train_unseen_feat = self.dataset.data['train_unseen']['resnet_features']
        train_unseen_label = self.dataset.data['train_unseen']['labels']

        # in ZSL mode:
        if self.generalized == False:
            # there are only 50 classes in ZSL (for CUB)
            # novel_corresponding_labels =list of all novel classes (as tensor)
            # test_novel_label = mapped to 0-49 in classifier function
            # those are used as targets, they have to be mapped to 0-49 right here:

            novel_corresponding_labels = self.map_label(novel_corresponding_labels, novel_corresponding_labels)

            if self.num_shots > 0:
                # not generalized and at least 1 shot means normal FSL setting (use only unseen classes)
                train_unseen_label = self.map_label(train_unseen_label, cls_novelclasses)

            # for FSL, we train_seen contains the unseen class examples
            # for ZSL, train seen label is not used
            # if self.num_shots>0:
            #    train_seen_label = self.map_label(train_seen_label,cls_novelclasses)

            test_novel_label = self.map_label(test_novel_label, cls_novelclasses)

            # map cls novelclasses last
            cls_novelclasses = self.map_label(cls_novelclasses, cls_novelclasses)

        if self.generalized:
            print('mode: gzsl')
            clf = LINEAR_LOGSOFTMAX(self.latent_size, self.num_classes)
        else:
            print('mode: zsl')
            clf = LINEAR_LOGSOFTMAX(self.latent_size, self.num_novel_classes)

        clf.apply(models.weights_init)

        with torch.no_grad():

            ####################################
            # preparing the test set
            # convert raw test data into z vectors
            ####################################

            self.reparameterize_with_noise = False

            mu1, var1 = self.encoder['resnet_features'](novel_test_feat)
            test_novel_X = self.reparameterize(mu1, var1).to(self.device).data
            test_novel_Y = test_novel_label.to(self.device)

            mu2, var2 = self.encoder['resnet_features'](seen_test_feat)
            test_seen_X = self.reparameterize(mu2, var2).to(self.device).data
            test_seen_Y = test_seen_label.to(self.device)

            ####################################
            # preparing the train set:
            # chose n random image features per
            # class. If n exceeds the number of
            # image features per class, duplicate
            # some. Next, convert them to
            # latent z features.
            ####################################

            self.reparameterize_with_noise = True

            def sample_train_data_on_sample_per_class_basis(features, label, sample_per_class):
                sample_per_class = int(sample_per_class)

                if sample_per_class != 0 and len(label) != 0:

                    classes = label.unique()

                    for i, s in enumerate(classes):

                        features_of_that_class = features[label == s, :]  # order of features and labels must coincide
                        # if number of selected features is smaller than the number of features we want per class:
                        multiplier = torch.ceil(torch.cuda.FloatTensor(
                            [max(1, sample_per_class / features_of_that_class.size(0))])).long().item()

                        features_of_that_class = features_of_that_class.repeat(multiplier, 1)

                        if i == 0:
                            features_to_return = features_of_that_class[:sample_per_class, :]
                            labels_to_return = s.repeat(sample_per_class)
                        else:
                            features_to_return = torch.cat(
                                (features_to_return, features_of_that_class[:sample_per_class, :]), dim=0)
                            labels_to_return = torch.cat((labels_to_return, s.repeat(sample_per_class)),
                                                         dim=0)

                    return features_to_return, labels_to_return
                else:
                    return torch.cuda.FloatTensor([]), torch.cuda.LongTensor([])

            # some of the following might be empty tensors if the specified number of
            # samples is zero :

            img_seen_feat,   img_seen_label   = sample_train_data_on_sample_per_class_basis(
                train_seen_feat,train_seen_label,self.img_seen_samples )

            img_unseen_feat, img_unseen_label = sample_train_data_on_sample_per_class_basis(
                train_unseen_feat, train_unseen_label, self.img_unseen_samples )

            att_unseen_feat, att_unseen_label = sample_train_data_on_sample_per_class_basis(
                    novelclass_aux_data,
                    novel_corresponding_labels,self.att_unseen_samples )

            att_seen_feat, att_seen_label = sample_train_data_on_sample_per_class_basis(
                seenclass_aux_data,
                seen_corresponding_labels, self.att_seen_samples)

            def convert_datapoints_to_z(features, encoder):
                if features.size(0) != 0:
                    mu_, logvar_ = encoder(features)
                    z = self.reparameterize(mu_, logvar_)
                    return z
                else:
                    return torch.cuda.FloatTensor([])

            z_seen_img   = convert_datapoints_to_z(img_seen_feat, self.encoder['resnet_features'])
            z_unseen_img = convert_datapoints_to_z(img_unseen_feat, self.encoder['resnet_features'])

            z_seen_att = convert_datapoints_to_z(att_seen_feat, self.encoder[self.auxiliary_data_source])
            z_unseen_att = convert_datapoints_to_z(att_unseen_feat, self.encoder[self.auxiliary_data_source])

            train_Z = [z_seen_img, z_unseen_img ,z_seen_att    ,z_unseen_att]
            train_L = [img_seen_label    , img_unseen_label,att_seen_label,att_unseen_label]

            # empty tensors are sorted out
            train_X = [train_Z[i] for i in range(len(train_Z)) if train_Z[i].size(0) != 0]
            train_Y = [train_L[i] for i in range(len(train_L)) if train_Z[i].size(0) != 0]

            train_X = torch.cat(train_X, dim=0)
            train_Y = torch.cat(train_Y, dim=0)

        ############################################################
        ##### initializing the classifier and train one epoch
        ############################################################

        cls = classifier.CLASSIFIER(clf, train_X, train_Y, test_seen_X, test_seen_Y, test_novel_X,
                                    test_novel_Y,
                                    cls_seenclasses, cls_novelclasses,
                                    self.num_classes, self.device, self.lr_cls, 0.5, 1,
                                    self.classifier_batch_size,
                                    self.generalized)

        for k in range(self.cls_train_epochs):
            if k > 0:
                if self.generalized:
                    cls.acc_seen, cls.acc_novel, cls.H = cls.fit()
                else:
                    cls.acc = cls.fit_zsl()

            if self.generalized:

                print('[%.1f]     novel=%.4f, seen=%.4f, h=%.4f , loss=%.4f' % (
                k, cls.acc_novel, cls.acc_seen, cls.H, cls.average_loss))

                history.append([torch.tensor(cls.acc_seen).item(), torch.tensor(cls.acc_novel).item(),
                                torch.tensor(cls.H).item()])

            else:
                print('[%.1f]  acc=%.4f ' % (k, cls.acc))
                history.append([0, torch.tensor(cls.acc).item(), 0])

        if self.generalized:
            return torch.tensor(cls.acc_seen).item(), torch.tensor(cls.acc_novel).item(), torch.tensor(
                cls.H).item(), history
        else:
            return 0, torch.tensor(cls.acc).item(), 0, history


from collections import OrderedDict

import numpy as np
import pandas as pd
from astropy import modeling
from scipy import sparse
from scipy.spatial import cKDTree
from scipy.optimize import lsq_linear

from xtool.wcs import PolynomialOrderWCS

FWHM_TO_SIGMA = 1. / (8 * np.log(2))**0.5
SIGMA_TO_FWHM = 1 / FWHM_TO_SIGMA

class VirtualPixelWavelength(modeling.Model):

    inputs = ()
    outputs = ('pixel_table', )

    wave_transform_coef = modeling.Parameter()

    wavelength_sampling_defaults = {'UVB' : 0.04,
                                    'VIS' : 0.04,
                                    'NIR' : 0.03}

    @classmethod
    def from_order(cls, order, poly_order=(2, 3), wavelength_sampling=None,
                   sub_sampling=5):
        """
        Instantiate a Virtualpixel table from the order raw Lookup Table WCS
        and then fitting a Polynomial WCS with given orde
        Parameters
        ----------
        order : xtool.data.Order
            order data or object
        poly_order : tuple or int
        wavelength_sampling: float, optional
            float for wavelength spacing. If `None` will use for different arms
            * UVB - 0.04 nm
            * VIS - 0.04 nm
            * NIR - 0.03 nm

        Returns
        -------
        VirtualPixelWavelength
        """
        polynomial_wcs = PolynomialOrderWCS.from_lut_order_wcs(order.wcs,
                                                               poly_order)

        if wavelength_sampling is None:
            wavelength_sampling = cls.wavelength_sampling_defaults[
                order.instrument_arm]

        wavelength_bins = np.arange(
            order.wcs.pix_to_wave.min(),
            order.wcs.pix_to_wave.max() + wavelength_sampling,
            wavelength_sampling)

        return VirtualPixelWavelength(polynomial_wcs, order.wcs,
                                      wavelength_bins,
                                      sub_sampling=sub_sampling)

    def __init__(self, polynomial_wcs, lut_wcs, raw_wavelength_pixels,
                 sub_sampling=5):
        """
        A model that represents the virtual pixels of the model

        Parameters
        ----------
        polynomial_wcs : xtool.wcs.PolynomialOrderWCS
        lut_wcs : xtool.wcs.LUTOrderWCS
        raw_wavelength_pixels : numpy.ndarray
        sub_sampling : int
            how many subsamples to create in each dimension
        """

        super(VirtualPixelWavelength, self).__init__(
            polynomial_wcs.wave_transform_coef)

        self.lut_wcs = lut_wcs
        self.polynomial_wcs = polynomial_wcs
        self.raw_wavelength_pixels = raw_wavelength_pixels
        self.standard_broadcasting = False
        self.sub_sampling = sub_sampling

    @staticmethod
    def _initialize_sub_pixel_table(x, y, sub_sampling):
        """
        Generate a subgrid for x and y with sampling `sub_sampling`.
        For example, for a subsampling of 5 each real x y coordinate will have
        25 additional entries.

        Parameters
        ----------
        x : numpy.ndarray
        y : numpy.ndarray
        sub_sampling : int

        Returns
        -------
        pandas.DataFrame
            subpixel table with pixel index, sub_x, sub_y
        """
        sub_edges = sub_sampling + 1
        sub_y_delta, sub_x_delta = np.mgrid[-0.5:0.5:sub_edges * 1j,
                                   -0.5:0.5:sub_edges * 1j]

        sub_pixel_size = 1 / np.float(sub_sampling)
        sub_x_delta = sub_x_delta[:-1, :-1] + 0.5 * sub_pixel_size
        sub_y_delta = sub_y_delta[:-1, :-1] + 0.5 * sub_pixel_size

        sub_x = np.add.outer(x.flatten(), sub_x_delta.flatten()).flatten()
        sub_y = np.add.outer(y.flatten(), sub_y_delta.flatten()).flatten()
        pixel_id = np.multiply.outer(
            np.arange(len(x.flatten())),
                      np.ones_like(sub_x_delta.flatten(),
                                   dtype=np.int64)).flatten()

        sub_pixel_table = pd.DataFrame(columns=['pixel_id', 'sub_x', 'sub_y', ],
                                   data={'pixel_id': pixel_id,
                                         'sub_x': sub_x,
                                         'sub_y': sub_y})
        return sub_pixel_table

    @staticmethod
    def _add_wavelength_sub_pixel_table(sub_pixel_table, wavelength_bins,
                                        wave_transform_coef):
        """
        Add a 'sub_wavelength' column that is calculated for each of the
        sub-pixel locations using the PolynomialWCS and add a
        'wavelength_bin_id' column that identifies the wavelength
        bin that this sub pixel corresponds to.
        Parameters
        ----------
        sub_pixel_table : pandas.DataFrame
        wavelength_bins : numpy.ndarray
        wave_transform_coef : numpy.ndarray

        Returns
        -------

        """
        kdt = cKDTree(wavelength_bins[np.newaxis].T)

        sub_pixel_table['sub_wavelength'] = np.polynomial.polynomial.polyval2d(
            sub_pixel_table.sub_x.values, sub_pixel_table.sub_y.values,
            wave_transform_coef)

        _, sub_pixel_table['wavelength_pixel_id'] = kdt.query(
            sub_pixel_table.sub_wavelength.values[np.newaxis].T)

        return sub_pixel_table

    @staticmethod
    def _generate_pixel_table(sub_pixel_table, sub_sampling):
        pixel_table = sub_pixel_table.groupby(
            ['pixel_id', 'wavelength_pixel_id']).sub_x.count()
        pixel_table /= sub_sampling ** 2
        pixel_table = pixel_table.reset_index()

        return pixel_table

    def evaluate(self, wave_transform_coef):
        sub_pixel_table = self._initialize_sub_pixel_table(
            self.lut_wcs.x, self.lut_wcs.y, self.sub_sampling)
        sub_pixel_table = self._add_wavelength_sub_pixel_table(
            sub_pixel_table, self.raw_wavelength_pixels,
            np.squeeze(wave_transform_coef))
        pixel_table = self._generate_pixel_table(sub_pixel_table,
                                                 self.sub_sampling)

        wavelength_pixel_contain_real_pixel_id = np.sort(
            pixel_table.wavelength_pixel_id.unique())

        self.wavelength_pixels = self.raw_wavelength_pixels[
            wavelength_pixel_contain_real_pixel_id]

        pixel_table['wavelength_pixel_id'] = (
            wavelength_pixel_contain_real_pixel_id.searchsorted(
                pixel_table.wavelength_pixel_id))

        pixel_table['wavelength'] = self.wavelength_pixels[
            pixel_table.wavelength_pixel_id].astype(np.float64)

        pixel_table['slit_pos'] = self.lut_wcs.pix_to_slit[
            pixel_table.pixel_id].astype(np.float64)

        pixel_table['normed_wavelength'] = (
                (pixel_table.wavelength - pixel_table.wavelength.min()) /
                (pixel_table.wavelength.max() - pixel_table.wavelength.min())) * 2 - 1

        return pixel_table

class OrderModel(object):

    def __init__(self, model_list, virtual_pixel=None):
        self.parameter_dict = self._generate_parameter_dict(model_list)
        self.model_list = model_list
        self.virtual_pixel = virtual_pixel

    def __getitem__(self, item):
        return self.model_list[item]

    @property
    def fittable_parameter_names(self):

        fittable_parameter_names = self._get_variable_normal_parameters()
        if self.virtual_pixel is not None:
            fittable_parameter_names += ['wave_transform_coef']
        return fittable_parameter_names

    @property
    def fittable_parameter_dict(self):
        return OrderedDict([(param_name, getattr(self, param_name))
                            for param_name in self.fittable_parameter_names])

    def _get_variable_normal_parameters(self):
        """
        Get all parameters that are not fixed and not matrix parameters

        Returns
        -------
        list
        """

        parameter_names = []
        for model in self.model_list:
            for parameter_name in model.param_names:
                if parameter_name in model.matrix_parameter:
                    continue
                elif getattr(model, parameter_name).fixed:
                    continue
                else:
                    parameter_names.append(parameter_name)
        return parameter_names

    def __getattr__(self, item):
        if 'parameter_dict' in self.__dict__ and item in self.parameter_dict:
            return self.parameter_dict[item]
        else:
            return super(OrderModel, self).__getattribute__(item)

    def __setattr__(self, key, value):
        if key in getattr(self, 'parameter_dict', {}):
            setattr(self.parameter_dict[key]._model, key, value)
        else:
            super(OrderModel, self).__setattr__(key, value)

    def __repr__(self):
        parameter_str = [
            '{0}={1.value}{2}'.format(param_name,
                                   getattr(param._model, param_name),
                                      ' [f]' if getattr(
                                          param._model, param_name).fixed
                                      else '')
            for param_name, param in self.parameter_dict.items()]
        return "<OrderModel({0})>".format(', '.join(parameter_str))

    @staticmethod
    def _generate_parameter_dict(model_list):
        """
        Generate a dictionary mapping parameter name to model instance. This
        excludes matrix parameters.

        Parameters
        ----------
        model_list : list of astropy.modeling.Model

        Returns
        -------
        OrderedDict
        """
        parameter_dict = OrderedDict()

        for model in model_list:
            for parameter_name in model.param_names:
                if parameter_name == model.matrix_parameter:
                    continue
                else:
                    if parameter_name in parameter_dict:
                        raise NotImplementedError(
                            'Currently only one model of each type is allowed')
                    else:
                        parameter_dict[parameter_name] = getattr(model,
                                                                 parameter_name)

        return parameter_dict

    @property
    def result_dict(self):
        result_dict = {}
        for i, model in enumerate(self.model_list):
            result_dict[model.__class__.__name__] = self.result[0][self.model_idx[i]:self.model_idx[i+1]]
        return result_dict

    def generate_design_matrix(self, order, **kwargs):
        """
        Generate the Design matrix for solving the Linear Least Squares problem
        Parameters
        ----------
        order : xtool.data.Order objects
        kwargs : fittable kwargs

        Returns
        -------
        """

        if 'wave_transform_coef' in kwargs:

            wave_transform_coef = (
                kwargs['wave_transform_coef'].reshape(
                    self.virtual_pixel.wave_transform_coef.shape))
            self.virtual_pixel.wave_transform_coef = wave_transform_coef
            pixel_table = self.virtual_pixel()
            for model in self.model_list:
                if hasattr(model, 'update_pixel_table'):
                    model.update_pixel_table(pixel_table)

        design_matrices = []
        uncertainties = order.uncertainty.compressed()
        for model in self.model_list:
            model_call_dict = self._generate_model_call_dict(model, kwargs)
            single_design_matrix = model.generate_design_matrix(
                **model_call_dict)
            single_design_matrix.data /= uncertainties[single_design_matrix.row]
            design_matrices.append(single_design_matrix)

        model_widths = [item.shape[1] for item in design_matrices]

        #dmatrix = sparse.hstack(design_matrices)

        return sparse.hstack(design_matrices), model_widths

    @staticmethod
    def _generate_model_call_dict(model, kwargs):
        model_call_dict = {}

        for parameter_name in model.param_names:
            if parameter_name in model.matrix_parameter:
                continue
            elif getattr(model, parameter_name).fixed:
                model_call_dict[parameter_name] = getattr(
                    model, parameter_name).value
            else:
                if parameter_name not in kwargs:
                    raise ValueError(
                        'Fittable parameter {0} has no assigned value'.format(
                            parameter_name))
                else:
                    model_call_dict[parameter_name] = kwargs[parameter_name]

        return model_call_dict

    def solve_design_matrix(self, dmatrix, order, solver='lsmr',
                            solver_dict={}):
        b = order.data.compressed() / order.uncertainty.compressed()
        if solver == 'lsmr':
            result = sparse.linalg.lsmr(dmatrix.tobsr(), b, **solver_dict)
        elif solver == 'lsq':
            lsq_dict = dict(bounds=(0, np.inf), lsmr_tol='auto', verbose=1)
            lsq_dict.update(solver_dict)
            result = lsq_linear(dmatrix, b,  **lsq_dict)
            result = [result.x, result]

        else:
            raise NotImplementedError('Solver {0} is not implemented'.format(
                solver))
        return result

    def set_matrix_parameters(self, result, model_widths):
        matrix_model_columns = np.cumsum(model_widths)
        for i, model in enumerate(self.model_list):
            current_matrix_column = (
                matrix_model_columns[i] - matrix_model_columns[0])
            for (matrix_parameter,
                 matrix_slice) in model.matrix_parameter_slices.items():
                setattr(model, matrix_parameter,
                        result[matrix_slice.start + current_matrix_column:
                        matrix_slice.stop + current_matrix_column])

    def set_matrix_uncertainties(self, order):

        dmatrix, model_widths = self.generate_design_matrix(
            order, **self.fittable_parameter_dict)
        dmatrix_inv = sparse.linalg.inv(dmatrix.T * dmatrix)
        uncertainties = np.sqrt(dmatrix_inv.diagonal())

        matrix_model_columns = np.cumsum(model_widths)
        for i, model in enumerate(self.model_list):
            current_matrix_column = (
                matrix_model_columns[i] - matrix_model_columns[0])
            for (matrix_parameter,
                 matrix_slice) in model.matrix_parameter_slices.items():
                setattr(
                    model, matrix_parameter + '_uncertainty',
                    uncertainties[matrix_slice.start + current_matrix_column:
                    matrix_slice.stop + current_matrix_column])

    def evaluate(self, order, solver='lsmr', solver_dict={}, **kwargs):
        dmatrix, model_widths = self.generate_design_matrix(order, **kwargs)
        result = self.solve_design_matrix(dmatrix, order, solver=solver,
                                          solver_dict=solver_dict)
        self.set_matrix_parameters(result[0], model_widths)
        return (dmatrix * result[0]) * order.uncertainty.compressed()

    def evaluate_to_frame(self, order, solver='lsmr', solver_dict={}, **kwargs):
        result = self.evaluate(order, solver, solver_dict, **kwargs)
        result_frame = np.ma.MaskedArray(
            np.empty_like(order.data, dtype=np.float64), mask=order.data.mask)
        result_frame.data[order.wcs.y, order.wcs.x] = result
        return result_frame

    def evaluate_residuals(self, order, solver='lsmr', solver_dict={},
                           ylim=None, **kwargs):
        result = self.evaluate(order, solver, solver_dict, **kwargs)
        b = order.data.compressed()

        if ylim is not None:
            row_filter = (order.wcs.y > ylim[0]) & (order.wcs.y < ylim[1])
            result = result[row_filter]
            b = b[row_filter]
            uncertainty = order.uncertainty.compressed()[row_filter]
        else:
            uncertainty = order.uncertainty.compressed()
        return ((result - b) / uncertainty)

    def evaluate_chi2(self, order, solver='lsmr', solver_dict={}, **kwargs):
        return np.sum(
            self.evaluate_residuals(order, solver, solver_dict, **kwargs)**2)

    def calculate_to_frame(self, order, solver='lsmr', solver_dict={}):
        return self.evaluate_to_frame(
            order, solver, solver_dict,
            **{param_name:self.parameter_dict[param_name].value
               for param_name in self.fittable_parameter_names})


import const as c
import evaluate, deevaluate, error
import pdb

def summarize_results(rules):
	rules = rules.items()
	to_ret = None
	for ru, val in rules:
		if to_ret == None:
			to_ret = (ru, val)
		else:
			if to_ret[1] in [c.TRUE, c.FALSE]:
				if val in [c.TRUE, c.FALSE] and val != to_ret[1]:
					raise error.ConflictingRules(ru, to_ret[0])
					break
			elif val in [c.TRUE, c.FALSE, c.UNDEFINED]:
				to_ret = (ru, val)
	return to_ret[1]

class Fact:
	def evaluate(self, facts):
		if self.evaluated:
			return self.value
		elif self.is_busy:
			return c.UNDEFINED
		else:
			self.is_busy = True
			true_rules = {}
			if len(self.depends_on) > 0:
				for rule in self.depends_on:
					if evaluate.exec_rpn(rule.condition, facts) == c.TRUE:
						true_rules[rule] = deevaluate.reverse_rpn(rule.implication, facts, self.name)
				self.is_busy = False
				if len(true_rules) > 0:
					self.value = summarize_results(true_rules)
					return self.value
				else:
					return self.value
			else:
				self.is_busy = False
				self.evaluated = True
				return self.value

	def __init__(self, name):
		self.name = name
		self.value = c.DEFAULT
		self.depends_on = []
		self.is_busy = False
		self.evaluated = False


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 12/6/19 1:58 PM
# @Author : xuezhi.zhang
# @File : utils.py
import base64

def base64_decode(base64_encode_str):
    """
    利用 base64.urlsafe_b64decode 对字符串解码
    :param base64_encode_str:
    :return:
    """
    if base64_encode_str:
        need_padding = len(base64_encode_str) % 4
        if need_padding != 0:
            missing_padding = 4 - need_padding
            base64_encode_str += '=' * missing_padding
        base64_encode_str = base64.urlsafe_b64decode(base64_encode_str).decode(
            'utf-8')
    if not isinstance(base64_encode_str, str):
        base64_encode_str = str(base64_encode_str, encoding='utf-8')
    return base64_encode_str

def base64_encode(input, padding=False):
    """
    利用 base64.urlsafe_b64decode 对字符串编码
    :param input: base64_encode_str
    :return:
    """
    assert isinstance(input, str), 'the type of input is not str.'
    if len(input) == 0:
        return ''
    input_encode_utf8 = input.encode('utf-8')
    input_encode_base64 = base64.urlsafe_b64encode(input_encode_utf8)
    if padding:
        need_padding = len(input_encode_base64) % 4
        if need_padding != 0:
            missing_padding = 4 - need_padding
            input_encode_base64 += '=' * missing_padding
    if not isinstance(input_encode_base64, str):
        input_encode_base64 = str(input_encode_base64, encoding='utf-8')
    return input_encode_base64

def dict_type_default():
    '''
    protocol_param <- protoparam
    obfs_param <- obfsparam
    server_port <- port
    :return:
    '''
    conf = {
        "group": "",
        "method": "",
        "obfs": "",
        "obfs_param": "",
        "password": "",
        "protocol": "",
        "protocol_param": "",
        "remarks": "",
        "server": "",
        "server_port": "",
    }
    return conf

def dict_type_url():
    """
    protocol_param -> protoparam
    obfs_param -> obfsparam
    server_port -> port
    :return:
    """
    conf = {
        "group": "",
        "method": "",
        "obfs": "",
        "obfsparam": "",
        "password": "",
        "port": "",
        "protocol": "",
        "protoparam": "",
        "remarks": "",
        "server": "",
    }
    return conf

def dict_type_default2url(conf):
    assert isinstance(conf, dict), "input is not dict."
    # obfs_param
    conf['obfsparam'] = conf['obfs_param']
    conf.pop('obfs_param')
    # protocol_param
    conf['protoparam'] = conf['protocol_param']
    conf.pop('protocol_param')
    # server_port
    conf['port'] = conf['server_port']
    conf.pop('server_port')
    return conf

def dict_type_url2default(conf):
    assert isinstance(conf, dict), "input is not dict."
    # obfs_param
    conf['obfs_param'] = conf['obfsparam']
    conf.pop('obfsparam')
    # protocol_param
    conf['protocol_param'] = conf['protoparam']
    conf.pop('protoparam')
    # server_port
    conf['server_port'] = conf['port']
    conf.pop('port')
    return conf


### Bokeh Plotting for PocketNC ###

import socket
import numpy as np
import threading
import sys
import pickle

from bokeh.plotting import figure, curdoc
from bokeh.driving import linear
from bokeh.models import SingleIntervalTicker, LinearAxis
import random

# Globals
lastNdxReceived = 0
#socket_pnc = 0
pncCommHost = '0.0.0.0'
pncCommPort = 6666
pncIntf = 0

# Data Storage
#plotData = 0
#plotData = dict(Time=[], queueLengths=[], xPosns=[])
pncData = 0

class pncCommListener(threading.Thread):
    def __init__(self, conn):
        super(pncCommListener, self).__init__()
        #self.tcqPlotData = []
        self.needUpdate = 0
        self.conn = conn
        self.data = ""
        print('pncComm thread started')

    def run(self):
        while True:
            (bytes_received, rec_address) = self.conn.recvfrom(65536)
            #dataArray = pickle.loads(bytes_received)
            dataDict = pickle.loads(bytes_received)
            #print(repr(dataDict))
            self.data = dataDict
            self.needUpdate = 1
            #string_received = bytes_received.decode("utf-8")
            #print(string_received)
            #self.data = self.data + dataArray

            #if self.data.endswith(u"&"):
            #    print('Received complete string', string_received)
            #    #process_data_points(self.data)
            #    self.data = ""
            #    self.needUpdate = 1

    def initPlots(self):
        print('initializing plots')

    def send_msg(self,msg):
        self.conn.send(msg)

    def close(self):
        self.conn.close()

def init():
    global pncIntf#, pncSocket, pncCommHost, pncCommPort
    try:
        #establish reader from pncComm
        pncSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        #pncSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        pncSocket.bind((pncCommHost,pncCommPort))
    except socket.error:
        print ('Failed to create feedback socket')
        #pncSocket.close()
        sys.exit()

    pncIntf = pncCommListener(pncSocket)
    pncIntf.start()

#update_lock = threading.Lock()
queueLength = figure(title="Buffer Fill Level", plot_width=800, plot_height=400)
r1 = queueLength.line([], [], color="firebrick", line_width=2)

threadTime = figure(title="Z-Axis Encoder", plot_width=800, plot_height=400)
r2 = threadTime.line([], [], color="firebrick", line_width=2)
ticker = SingleIntervalTicker(interval=5, num_minor_ticks=10)
xaxis = LinearAxis(ticker=ticker)
yaxis = LinearAxis(ticker=ticker)

ds1 = r1.data_source
ds2 = r2.data_source

@linear()
def update(step):
    #print(step)
    if pncIntf.needUpdate:
        #print('need to update plots')
        queue_data = pncIntf.data['TCQ']
        encoder_data = pncIntf.data['Encoder']
        #encoder_len = len(encoder_data)

        pncIntf.needUpdate = 0
        ds1.data = dict(x = np.arange(len(queue_data)), y = queue_data)
        ds1.trigger('data', ds1.data, ds1.data)
        ds2.data = dict(x = np.arange(len(encoder_data)), y = encoder_data)
        ds2.trigger('data', ds2.data, ds2.data)

    #ds1.trigger('data', ds1.data, ds1.data)
    #ds2.trigger('data', ds2.data, ds2.data)

curdoc().add_root(queueLength)
curdoc().add_root(threadTime)
init()

# Add a periodic callback to be run every 500 milliseconds
curdoc().add_periodic_callback(update, 100)
curdoc().title = "PocketNC Interface"


__author__ = 'Dazdingo'

from pytz import timezone
import simplejson as json
import ConfigParser
import sys
from datetime import datetime
reload(sys)
sys.setdefaultencoding('utf-8')

config_parser = ConfigParser.ConfigParser()
config_parser.read('server.cfg')
time_zone = config_parser.get('common', 'time_zone')

def document2json(document):
    for key, value in document._data:
        document[key] = str(document[key])
    return json.dumps(document)

def collection2json(collection):
    temp = list()
    for document in collection:
        for key, value in document._data:
            document[key] = str(document[key])
        temp.append(document)
    return json.dumps(temp)

def get_time_stamp():
    dt_now = datetime.now(timezone(time_zone))
    import time
    return str(int(time.mktime(dt_now.timetuple())))

def item2json(item):
    return json.dumps(item._data)


from sklearn.manifold import TSNE
import pickle
import numpy as np
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import zero_one_loss

#runs PCA to n components and then runs t-SNE to 2 components, plotting the resulting matrix
matrix_in = open('./data/5k_title_matrix.pkl', 'rb')
((matrix, ids), Y), desc = pickle.load(matrix_in)
matrix_in.close()

scaler = StandardScaler()

scaler.fit(matrix)
matrixp = scaler.transform(matrix)
pca = PCA(n_components = 1000) # retains 95% of variance -- we can change this (this can also be the number of components/dimensions we want, eg. n_components = 3)
pca.fit(matrixp)
# print(pca.n_components_)

matrix_pca = pca.transform(matrix)
print("cumulative variation for 1000 components is " + str(np.sum(pca.explained_variance_ratio_)) )

tsne = TSNE(n_components = 2)
matrix_tsne = tsne.fit_transform(matrix_pca)
print('fitted')

for i in range(len(matrix_tsne)):
    #isolating values to analyze, change numbers as needed
    if matrix_pca[i][0] > 15 and matrix_pca[i][0] < 15.02 and matrix_pca[i][1] < -18.5 and matrix_pca[i][1] > -18.6:
        print(f"index {i} has a value of {matrix_tsne[i]}")

targets = [0,1]
colors = ['r', 'b']
for target, color in zip(targets,colors):
    inds_to_keep = []
    for i in range(len(Y)):
        if Y[i][2] == target:
            inds_to_keep.append(i)
    plt.scatter(matrix_tsne[inds_to_keep, 0], matrix_tsne[inds_to_keep, 1], alpha = 0.4,
            c = color, s = 20)
print("computed")
plt.xlabel('component 1')
plt.ylabel('component 2')

plt.show()

pickle_out = open("./data/title_tsne_experiment.pkl", 'wb')
pickle.dump((matrix_tsne, Y), pickle_out)
pickle_out.close()


#!/opt/libreoffice5.2/program/python
# -*- coding: utf-8 -*-
import xml.etree.ElementTree as ET
from helper import Elem, ElemProp, ElemPropLoc
from config import getConfig
import os
class ElemLeaf(Elem):  # node-type=Leaf。オプションページのロードマップコントロールの小項目になる。
	def __init__(self, c, attrs):  # 辞書attrのkeyは"Name"	, "Label", "GroupId", "GroupIndex", "Id"。 valueはすべて文字列でいれる。
		if not "Id" in attrs:  # IdがなければNameを代用する。Idを拡張機能Idと同じにすると拡張機能マネージャーでオプションボタンが表示される。
			attrs["Id"] = attrs["Name"]
		attrs["OptionsPage"] = "%origin%/dialogs/optionsdialog.xdl"  # オプションダイアログxdlファイル名。UnoControlDialogオブジェクトになる。
		attrs["EventHandlerService"] = c["ExtentionID"]  # オプションダイアログの呼び出しと共に呼び出す実装サービス名。
		super().__init__("node", {'oor:name': attrs.pop("Name"), "oor:op": "fuse"})
		for key, val in attrs.items():
			if key=="Label":
				self.append(ElemPropLoc(key, val))
			else:
				self.append(ElemProp(key, val))
class ElemNode(Elem):  # node-type=Node
	def __init__(self, c, attrs, *, leaves=None):  # 辞書attrのkeyはName, Label, OptionsPage, AllModules, GroupId, GroupIndex。 valueはすべて文字列でいれる。
		super().__init__("node", {'oor:name': attrs.pop("Name"), "oor:op": "fuse"})
		for key, val in attrs.items():
			if key=="Label":
				self.append(ElemPropLoc(key, val))
			else:
				self.append(ElemProp(key, val))
		if leaves is not None:
			self.append(Elem("node", {'oor:name': "Leaves"}))  # セットノードLeaves
			for leaf in leaves:
				self[-1].append(leaf)  #  node-type=LeafをセットノードLeavesに追加。
class ElemOrderdNode(Elem):  # node-type=OrderdNode
	def __init__(self, name, index=None):  # nodeはノード名。
		super().__init__("node", {'oor:name': name, "oor:op": "fuse"})
		if index is not None:
			self.append(ElemProp("Index", index))  # 位置の指定。省略可。
class ElemModeule(Elem):  # node-type=Module
	def __init__(self, name, nodes):  # moduleはオプションダイアログを表示させるモジュールを指定。有効にするためにはnode-type=NodeのAllModulesをfalseにする必要がある。
		super().__init__("node", {'oor:name': name, "oor:op": "fuse"})
		self.append(Elem("node", {'oor:name': "Nodes"}))  # セットノードNodes
		for node in nodes:
			self[-1].append(node)  #  node-type=OrderdNodeをセットノードNodesに追加。
class ElemSingleOption(Elem):  #  node-type=SingleOption
	def __init__(self, name):
		super().__init__("node", {'oor:name': name, "oor:op": "fuse"})
		self.append(ElemProp("Hide", "true"))
class ElemOptionsPage(Elem):  #  node-type=OptionsPage
	def __init__(self, name, *, hide=None, singleoptions=None):
		super().__init__("node", {'oor:name': name, "oor:op": "fuse"})
		if hide is not None:
			self.append(ElemProp("Hide", "true"))
		if singleoptions is not None:
			self.append(Elem("node", {'oor:name': "Options"}))  # セットノードOptions。
			for singleoption in singleoptions:
				self[-1].append(singleoption)  #  node-type=SingleOptionをセットノードOptionsに追加。
class ElemOptionsGroup(Elem):  #  node-type=OptionsGroup
	def __init__(self, name, *, hide=None, optionspages=None):
		super().__init__("node", {'oor:name': name, "oor:op": "fuse"})
		if hide is not None:
			self.append(ElemProp("Hide", "true"))
		if optionspages is not None:
			self.append(Elem("node", {'oor:name': "Pages"}))  # セットノードPages。
			for optionspage in optionspages:
				self[-1].append(optionspage)  #  node-type=OptionsPageをセットノードPagesに追加。
def createOptionsDialogXcu(c):  #Creation of OptionsDialog.xcu
	os.chdir(c["src_path"])  # srcフォルダに移動。
	filename =  "OptionsDialog.xcu"
	c["backup"](filename)  # すでにあるファイルをバックアップ
	with open(filename, "w", encoding="utf-8") as f:  # OptionsDialog.xcuファイルの作成
		root = Elem("oor:component-data", {"oor:name": "OptionsDialog", "oor:package": "org.openoffice.Office", "xmlns:oor": "http://openoffice.org/2001/registry", "xmlns:xs": "http://www.w3.org/2001/XMLSchema", "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance"})  # 根の要素を作成。

		#オプションページの作成。拡張機能IDのc["ExtentionID"]をNameに使った時はLeafを一つにしないと拡張機能マネージャーでフリーズする。
		root.append(Elem("node", {'oor:name': "Nodes"}))  # セットノードNodesを追加。
		leaf = ElemLeaf(c, {"Name": c["ExtentionID"], "Label": {"en-US": "Maximum Paper Size", "ja-JP": "最大用紙サイズ"}})  # node-type=Leaf。小項目の設定。Idを拡張機能Idと同じc["ExtentionID"]にする。
		name = "{}.Node1".format(c["ExtentionID"])  # ロードマップコントロールに表示させる大項目のID。ユニークの必要があると考えるので拡張機能IDにくっつける。
		node = ElemNode(c, {"Name": name, "Label": {"en-US": "Extensions Example", "ja-JP": "拡張機能の例"}, "AllModules": "false"}, leaves=(leaf,))  # node-type=Node。大項目の設定。
		root[-1].append(node)  #  node-type=NodeをセットノードNodesに追加。

		# オプションページを表示させるモジュールを限定する。
		root.append(Elem("node", {'oor:name': "Modules"}))  # セットノードModulesを追加。
		orderednode = ElemOrderdNode(name)  #  node-type=OrderdNode。node-type=NodeのNameで表示させる大項目を指定。
		node = ElemModeule("com.sun.star.text.TextDocument", (orderednode,))  # node-type=Module。表示させるモジュールを指定。第一引数はhttps://wiki.openoffice.org/wiki/Framework/Article/Options_Dialog_Configurationのいずれか。
		root[-1].append(node)  #  node-type=ModuleをセットノードModulesに追加。

		# 一部のオプションページを非表示にする。
		root.append(Elem("node", {'oor:name': "OptionsDialogGroups"}))  # セットノードOptionsDialogGroupを追加。
		security = ElemOptionsPage("Security", hide=True)  #  node-type=OptionsPage。非表示にする小項目。セキュリティー。
		memory = ElemOptionsPage("Memory", hide=True)  #  node-type=OptionsPage。非表示にする小項目。メモリー。
		productname = ElemOptionsGroup("ProductName", optionspages=(security, memory))  #  node-type=OptionsGroup。非表示にする大項目。hide=Trueにしていないときは第2引数の小項目のみ非表示にする。
		root[-1].append(productname)  #  node-type=OptionsGroupをセットノードOptionsDialogGroupsに追加。
		internet = ElemOptionsGroup("Internet", hide=True)  #  node-type=OptionsGroup。非表示にする大項目。Internet。
		root[-1].append(internet)  #  node-type=OptionsGroupをセットノードOptionsDialogGroupsに追加。

		tree = ET.ElementTree(root)  # 根要素からxml.etree.ElementTree.ElementTreeオブジェクトにする。
		tree.write(f.name, "utf-8", True)  # xml_declarationを有効にしてutf-8でファイルに出力する。
	print("{} has been created.".format(filename))
if __name__ == "__main__":
	createOptionsDialogXcu(getConfig(False))  # バックアップをとるときはTrue


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import admin_app.core.utils.models

class Migration(migrations.Migration):

    dependencies = [
        ('goods', '0002_goods_title'),
    ]

    operations = [
        migrations.CreateModel(
            name='Offer',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('is_best', models.BooleanField(default=False, verbose_name='\u041b\u0443\u0447\u0448\u0435\u0435 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0435')),
                ('shop_title', models.TextField(null=True, verbose_name='\u041d\u0430\u0437\u0432\u0430\u043d\u0438\u0435 \u043c\u0430\u0433\u0430\u0437\u0438\u043d\u0430', blank=True)),
                ('price', models.TextField(null=True, verbose_name='\u0426\u0435\u043d\u0430(\u0441\u043e \u0441\u043a\u0438\u0434\u043a\u043e\u0439)', blank=True)),
                ('old_price', models.TextField(null=True, verbose_name='\u0426\u0435\u043d\u0430 \u0431\u0435\u0437 \u0441\u043a\u0438\u0434\u043a\u0438', blank=True)),
                ('url', models.URLField(null=True, verbose_name='\u0426\u0435\u043d\u0430 \u0431\u0435\u0437 \u0441\u043a\u0438\u0434\u043a\u0438', blank=True)),
            ],
            bases=(admin_app.core.utils.models.CommonModelMixin, models.Model),
        ),
        migrations.AlterModelOptions(
            name='goods',
            options={'ordering': ('title',), 'verbose_name': '\u0422\u043e\u0432\u0430\u0440', 'verbose_name_plural': '\u0422\u043e\u0432\u0430\u0440\u044b'},
        ),
        migrations.RemoveField(
            model_name='goods',
            name='URL',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='age_from',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='age_to',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='buying_price',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='delivery',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='local_delivery_cost',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='model',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='old_price',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='pickup',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='price',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='store',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='type_prefix',
        ),
        migrations.RemoveField(
            model_name='goods',
            name='weight',
        ),
        migrations.AlterField(
            model_name='goods',
            name='detmir_id',
            field=models.IntegerField(unique=True, verbose_name='Id \u0432 \u043a\u0430\u0442\u0430\u043b\u043e\u0433\u0435 \u041f\u0440\u0430\u0439\u0441\u0430'),
        ),
        migrations.AlterField(
            model_name='goods',
            name='gender',
            field=models.IntegerField(null=True, verbose_name='\u041f\u043e\u043b', choices=[(0, '\u041c\u0443\u0436\u0441\u043a\u043e\u0439'), (1, '\u0416\u0435\u043d\u0441\u043a\u0438\u0439')]),
        ),
        migrations.AlterField(
            model_name='goods',
            name='hidden',
            field=models.BooleanField(default=False, verbose_name='\u0421\u043a\u0440\u044b\u0442\u044b\u0442\u044c \u0442\u043e\u0432\u0430\u0440'),
        ),
        migrations.AddField(
            model_name='offer',
            name='good',
            field=models.ForeignKey(related_name='offers', to='goods.Goods'),
        ),
    ]


def majorityElement(nums):
    """
    :type nums: List[int]
    :rtype: int
    """
    import collections
    collection_dict = dict(collections.Counter(nums))
    ret = list()
    for key in collection_dict:
        if collection_dict[key] > len(nums) / 3:
            ret.append(key)
    return ret

nums_1 = [1, 2, 1]
nums_2 = [1,1,1,3,3,2,2,2]

print(majorityElement(nums_2))


from django.conf.urls import url
from . import views

urlpatterns = [

    url(r'^$', views.index),
    url(r'^registerprocess$', views.register),
    url(r'^loginprocess$', views.login),
    url(r'^quotes', views.quotes),
    url(r'^contribQuote$', views.con_quot),
    url(r'^delete/(?P<id>\d+)$', views.delete),
    url(r'^User/(?P<id>\d+)$', views.postUser),
    url(r'^logout$', views.logout),
    url(r'^dashboard$', views.dashboard),
    url(r'^addToFave/(?P<id>\d+)$', views.addFave),
    url(r'^removeFromFave/(?P<id>\d+)$', views.remove),

]


import math
import pygame
#import const
#import animation
import importlib
const= importlib.import_module("17-3const")
animation= importlib.import_module("17-3animation")

class Stone(animation.Animation):
	def __init__(self):
		self.spriteImage= "17-3stone.png"
		self.spriteWidth= 8
		self.spriteHeight= 8
		self.spriteColumns= 4
		self.fps= 20
		self.state= const.STONE_READY
		self.InitAnimation()

	def update(self):
		self.CalcNextFrame()

		rect= (self.spriteWidth*self.currentFrame, 0, self.spriteWidth, self.spriteHeight)
		self.image.blit( self.spriteSheet, (0, 0), rect)
		self.image.set_colorkey(pygame.color.Color(255, 0, 255))

	def Setup(self, initialPos, power, direction):
		self.initialPos= initialPos
		self.rect.x= initialPos[0]
		self.rect.y= initialPos[1]
		self.power= power
		self.direction= direction
		self.state= const.STONE_FLY

	def Move(self, time, space, DecrementStones):
		pos= self.CalculatePosition(time, const.g, self.direction)
		pos= self.MapPosition(self.initialPos[0], self.initialPos[1], pos[0], pos[1])
		self.rect.x= pos[0]
		self.rect.y= pos[1]

		if pos[0]>space[0] or pos[1]>space[1]:
			self.state= const.STONE_READY
			DecrementStones()

	def CalculatePosition(self, t, g, direction):
		r= math.radians(direction)
		x= self.power*math.cos(r)*t
		y= self.power*math.sin(r)*t - 0.5*g*math.pow(t, 2)

		return (int(x), int(y))

	def MapPosition(self, x, y, newX, newY):
		return (x+newX, y+(newY*-1))


#from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
import requests

url = 'http://stefan.sofa-rockers.org/'
payload = {
    'q': 'Python',
}
r = requests.get(url )

soup = BeautifulSoup(r.text)
titles = [h1.text for h1 in soup.findAll('h1')]

for t in titles:
    print(t)

#print(soup.prettify())


import random
import numpy as np
from deap import tools

def defaultMut(individual, MINPDB):
    return tools.mutFlipBit(individual, MINPDB)

def fixedActivationMut(individual, MINPDB):
    return tools.mutShuffleIndexes(individual, MINPDB)

def defaultGeneWiseTwoPoint(ind1, ind2, GENES, GENESIZE):
    cxpoint1 = random.randint(1, GENES)*GENESIZE
    cxpoint2 = random.randint(1, GENES-1)*GENESIZE
    if cxpoint2 >= cxpoint1:
        cxpoint2 += GENESIZE
    else: # Swap the two cx points
        cxpoint1, cxpoint2 = cxpoint2, cxpoint1

    ind1[cxpoint1:cxpoint2], ind2[cxpoint1:cxpoint2] \
        = ind2[cxpoint1:cxpoint2].copy(), ind1[cxpoint1:cxpoint2].copy()

    return ind1, ind2

def fixActivation(ind, targetActivations):

    misActivatedCount = np.sum(ind) - targetActivations
    misActivatedState = 1 if misActivatedCount > 0 else 0
    targetActivationState = 1 if misActivatedState == 0 else 0
    misActivatedPos = [idx for idx, val in enumerate(ind) if val == misActivatedState]
    fixActivationPos = np.random.choice(misActivatedPos, size=abs(misActivatedCount), replace=False)

    for pos in fixActivationPos:
        ind[pos] = targetActivationState

    return ind

def fixedActivationGeneWiseTwoPoint(ind1, ind2, GENES, GENESIZE):
    activates1 = np.sum(ind1)
    activates2 = np.sum(ind2)

    ind1, ind2 = defaultGeneWiseTwoPoint(ind1, ind2, GENES, GENESIZE)
    ind1, ind2 = fixActivation(ind1, activates1), fixActivation(ind2, activates2)

    return ind1, ind2


import os
import sys

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import tensorflow as tf
from sklearn import preprocessing
from tensorflow import keras

sns.set()

def load_data():
    """
    返回读取的文件中的数据，并进行归一化
    """
    path = os.path.join(sys.path[0], 'Data')
    path_data = os.path.join(path, 'all_samples_net.npy')

    data_ori = np.load(path_data)
    X_ori = data_ori[:, :2].copy()
    Y_ori = data_ori[:, 2:].copy()
    x_scale = preprocessing.MinMaxScaler()
    y_scale = preprocessing.MinMaxScaler()
    X = x_scale.fit_transform(X_ori)
    Y = y_scale.fit_transform(Y_ori)

    data = {}
    data['X'] = X
    data['Y'] = Y

    return data

def build_model(input_dim,out_dim,units_lists = None):
    """
    建立一个4层神经层的model
    """
    inputs = keras.Input(shape=(input_dim,))
    if units_lists is None:
        units_lists = [100,200,100,100]
    x = inputs
    for units in units_lists:
        x = keras.layers.Dense(units,activation='relu')(x)
    predictions = keras.layers.Dense(out_dim, activation='tanh')(x)
    model = keras.Model(inputs,predictions)
    model.summary()
    return model

def plot_history(history):
    """
    画出训练之后得到的结果
    """
    plt.figure()
    plt.xlabel('Epoch')
    plt.ylabel('Mean Abs Error')
    plt.plot(history.epoch, np.array(history.history['mean_absolute_error']),label='Train Loss')
    plt.plot(history.epoch, np.array(history.history['val_mean_absolute_error']), label = 'Val loss')
    plt.legend()

def get_gradients(model,trainingExample):
    """
    得到模型在trainingExample对输入的梯度
    """
    outputTensor = model.output
    gradients = tf.gradients(outputTensor, model.input)
    sess = keras.backend.get_session()
    evaluated_gradients = sess.run(gradients,feed_dict={model.input:trainingExample})
    return evaluated_gradients

if __name__ == '__main__':
    data = load_data()
    X,Y = data['X'],data['Y']

    train = 1
    if train:
        model = build_model(X.shape[1], Y.shape[1])
        model.summary()
        keras.utils.plot_model(model, to_file=os.path.join(sys.path[0], 'Net/model2.png'))
        # 设置训练方式
        model.compile(tf.train.RMSPropOptimizer(0.01), loss=tf.losses.mean_squared_error, metrics=['mae'])

        # 进行训练
        early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=50)  # 20代测试误差没有改进就退出
        history = model.fit(X, Y, batch_size=1024, epochs=1000, validation_split=0.1, callbacks=[early_stop])
        model.save(os.path.join(sys.path[0], 'Net/model2.h5'))
        # 训练结果展示
        plot_history(history)
        plt.show()
    else:
        model = keras.models.load_model(os.path.join(sys.path[0], 'Net/model2.h5'))
        model.summary()
        for _ in range(100):
            print(get_gradients(model,X))


"""
 Copyright (c) 2019 Intel Corporation

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import cv2
import numpy as np

from common import IEModel
import os
class Model(IEModel):
    """ Class that allows worknig with person detectpr models. """

    def __init__(self, model_path, device, ie_core, num_requests = 1):
        """Constructor"""

        super().__init__(model_path, device, ie_core, num_requests)
        _, _, h, w = self.input_size
        self.input_height = h
        self.input_width = w

        keys_path = os.path.join(os.path.dirname(model_path),'keys.txt')
        keys_file = open(keys_path, 'r', encoding ='utf-8')
        self.alphabet  = keys_file.readline().rstrip()

    def _prepare_frame(self, img):
        """Converts input image according model requirements"""
        img=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
        img = cv2.resize(img,(self.input_width,self.input_height))
        img = img.reshape([1, 1,self.input_height, self.input_width])
        return img

    def _process_output(self,pred):
        """Converts network output to text"""
        seq_len = int(120/4)
        pred = pred.transpose((1,0,2)).reshape((1,seq_len,37))
        pred = pred[0,:,:]

        characters = self.alphabet[:]
        characters = characters[:] + u'卍'
        nclass = len(characters)

        res = ""
        prev_pad = False
        pred = np.expand_dims(pred, axis=0)
        pred_idx = pred.argmax(axis=2)[0]

        for idx in pred_idx:
            pred_text = characters[idx]
            if pred_text != u'卍':
                if(len(res) == 0 or prev_pad or (len(res)!= 0 and pred_text!= res[-1])):
                    prev_pad = False
                    res = res + pred_text
            else:
                prev_pad = True
        return res

    def __call__(self, frame):
        """Runs model on the specified input"""
        outs = []
        in_frame = self._prepare_frame(frame)
        result = self.infer(in_frame)
        out = self._process_output(result)
        return out


import numpy as np
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import itertools
import operator

def warn(*args, **kwargs):
    pass
import warnings
warnings.warn = warn

def generateOvOClassifier(classes):
    """
        Génération du classifier 0V0
    """
    o_vs_o_classifiers = {}
    for elem in itertools.combinations(classes,2):
        class0 = [x_train[index] for index, value in enumerate(y_train) if value == elem[0]]
        class1 = [x_train[index] for index, value in enumerate(y_train) if value == elem[1]]
        value = [0] * len(class0) + [1] * len(class1)
        learn = class0 + class1
        #Apprentissage du modèle
        o_vs_o_classifiers['%d_%d'%elem] = LogisticRegression(solver='lbfgs').fit(learn, value)
    return o_vs_o_classifiers

def predictOVO(test_values, o_vs_o_classifiers):
    """
        Prédiction du classifieur 0V0
    """
    results = {}
    i=0
    for elem in test_values:
        intern_result = {}
        for name,classifiers in o_vs_o_classifiers.items():
            result = classifiers.predict([elem[0]])
            members = name.split('_')
            if intern_result.get(members[result[0]]):
                intern_result[members[result[0]]] += 1
            else:
                intern_result[members[result[0]]] = 1
        results[i] = intern_result
        i+=1
    correct = 0
    for key,elem in results.items():
        predicted = max(elem.items(), key=operator.itemgetter(1))[0]
        value = test_values[key][1]
        if int(predicted) == value:
            correct += 1
        print("Predicted %s and value was %s" %(predicted,value))
    prct = (correct/len(results)*100)
    print("The One versus One score a %f precision score " %prct)

if __name__ == "__main__":
    digits = datasets.load_digits()
    data = digits['data']
    target  = digits['target']
    classes = set(target)
    #Séparation des données entre jeu d'entraînement et de test
    x_train, x_test, y_train, y_test = train_test_split(data, target, test_size=0.2)
    #Creation du classifieur O v O
    o_vs_o_classifiers = generateOvOClassifier(classes)
    # Nous générons un tableau contenant des tuples (images,value)
    test_values = [(x_test[index],value) for index, value in enumerate(y_test)]
    # Lancement de la boucle de prediction des éléments sur le jeu de test
    predictOVO(test_values, o_vs_o_classifiers)


import json
import logging

import aiohttp
from pymongo import MongoClient

import datetime, time
import requests, hashlib
from requests import RequestException
from urllib.parse import urlencode

from dquant.config import cfg
from dquant.constants import Constants
from dquant.markets.market import Market

class OkexFutureRest(Market):
    def __init__(self, meta_code):
        base_currency, market_currency, symbol, contract_type = self.parse_meta(meta_code)
        super().__init__(base_currency, market_currency, meta_code, cfg.get_float_config(Constants.OKEX_FEE))
        self.apikey = cfg.get_config(Constants.OKEX_APIKEY)
        self.apisec = cfg.get_config(Constants.OKEX_APISEC)
        self.contract_type = contract_type
        self.symbol = symbol
        self.base_url = Constants.OKEX_FUTURE_REST_BASE
        self.timeout = Constants.OK_HTTP_TIMEOUT

    async def get_ticker(self):
        depth = await self.get_depth()
        if not depth:
            return None
        res = {'ask': {'price': 0, 'amount': 0}, 'bid': {'price': 0, 'amount': 0}}
        if len(depth['asks']) > 0:
            res['ask'] = depth['asks'][0]
        if len(depth["bids"]) > 0:
            res['bid'] = depth['bids'][0]
        return res

    # 轮询获取depth
    async def get_depth(self):
        params = {"symbol": self.symbol, "contract_type": self.contract_type}
        while True:
            try:
                res = await self.request(Constants.OKEX_FUTURE_DEPTH_RESOURCE_REST, params, "get")
                list_of_ask = self.okex_depth_format(res, "asks")
                list_of_ask.reverse()
                list_of_bid = self.okex_depth_format(res, "bids")
                return {"asks": list_of_ask, 'bids': list_of_bid}
            except Exception:
                logging.exception("http error")

    # 格式化depth数据
    def okex_depth_format(self, res, flag):
        result_list = []
        for ticker in res[flag]:
            result_list.append({
                'price': ticker[0],
                'amount': ticker[1]
            })
        return result_list

    async def long(self, amount, price='', lever_rate='10'):
        res = await self.okex_request(price=price, amount=amount, type='1', api_url=Constants.OKEX_FUTURE_TRADE_REST)
        return res

    async def short(self, amount, price='', lever_rate='10'):
        res = await self.okex_request(price=price, amount=amount, type='2', api_url=Constants.OKEX_FUTURE_TRADE_REST)
        return res

    async def close_long(self, amount, price=''):
        res = await self.okex_request(price=price, amount=amount, type='3', api_url=Constants.OKEX_FUTURE_TRADE_REST)
        return res

    async def close_short(self, amount, price=''):
        res = await self.okex_request(price=price, amount=amount, type='4', api_url=Constants.OKEX_FUTURE_TRADE_REST)
        return res

    async def delete_order(self, order_id, tillOK=True):
        '''
        :param order_id:
        :return: {'result': True, 'order_id': '14435081666'}
        '''
        while True:
            try:
                res = await self.okex_request(order_id=order_id, api_url=Constants.OKEX_FUTURE_DELETE_ORDER_REST)
                if 'result' in res:
                    if res['result'] == True:
                        return res
                if tillOK ==True:
                    continue
                else:
                    logging.error(res)
                    break
            except Exception as ex:
                logging.exception("message")
                if tillOK:
                    continue
                return None

    async def get_account(self, coin=[]):
        '''
        :return:{"info": {"btc": {"account_rights": 1,"keep_deposit": 0,"profit_real": 3.33,"profit_unreal": 0,"risk_rate": 10000},"ltc": {"account_rights": 2,"keep_deposit": 2.22,"profit_real": 3.33,"profit_unreal": 2,"risk_rate": 10000},"result": true}
        '''
        res = await self.okex_request(api_url=Constants.OKEX_FUTURE_USERINFO_REST)
        if res['result'] is True:
            if coin:
                ret = {}
                for c in coin:
                    if c.lower() in res["info"]:
                        ret[c.upper()] = res["info"][c.lower()]['balance']
                    else:
                        ret[c.upper()] = 0.0
                return ret
            else:
                return res["info"]

    # 向API发送请求
    async def okex_request(self, api_url, **kwargs):
        '''
        :param price: 默认对手价
        :param amount: 最小为1
        :param type: 1:开多 2:开空 3:平多 4:平空, 'delete_order':取消订单
        :param lever_rate: 杠杆倍数 value:10\20 默认10
        :param match_price: 是否为对手价 0:不是  1:是,当取值为1时,price无效。这里根据price是否为空判断。
        :param contract_type: 合约类型: this_week:当周   next_week:下周   quarter:季度
        :return: {"order_id":986,"result":true}
        '''
        params = {}

        if api_url is Constants.OKEX_FUTURE_DELETE_ORDER_REST:
            order_id = kwargs.get('order_id', None)
            params = {'api_key': self.apikey, 'symbol': self.symbol, 'contract_type': self.contract_type, 'order_id': order_id}

        elif api_url is Constants.OKEX_FUTURE_TRADE_REST:
            params = {'api_key': self.apikey, 'symbol': self.symbol, 'contract_type': self.contract_type, 'amount': str(kwargs.get('amount')),
                      'type': str(kwargs.get('type')), 'match_price': '1', 'lever_rate': str(kwargs.get('lever_rate', 10))}
            price = kwargs.get('price', None)
            if price:
                params['price'] = str(price)
                params['match_price'] = '0'

        elif api_url is Constants.OKEX_FUTURE_USERINFO_REST:
            params = {'api_key': self.apikey}

        params['sign'] = self.buildMySign(params, self.apisec)
        res = await self.request(api_url, params=params, type='post')
        return res

    async def request(self, resource, params, type):
        headers = {
            "Content-type": "application/x-www-form-urlencoded",
        }
        async with aiohttp.ClientSession() as session:
            if type == "post":
                temp_params = urlencode(params)
                async with session.post(url=self.base_url + resource, data=temp_params, timeout=self.timeout, headers=headers) as res:
                    if res.headers['CONTENT-TYPE'] == 'application/json':
                        return await res.json()
                    else:
                        text = await res.text()
                        return json.loads(text)

            elif type == "get":
                async with session.get(url=self.base_url + resource, params=params, timeout=self.timeout) as res:
                    if res.headers['CONTENT-TYPE'] == 'application/json':
                        return await res.json()
                    else:
                        text = await res.text()
                        return json.loads(text)

    def buildMySign(self, params, secretKey):
        sign = ''
        for key in sorted(params.keys()):
            sign += key + '=' + str(params[key]) + '&'
        data = sign + 'secret_key=' + secretKey
        return hashlib.md5(data.encode("utf8")).hexdigest().upper()

    def parse_meta(self, meta_code):
        meta_table = {"btc_usd_this_week": ("btc", "usd", "btc_usd", "this_week"),
                      "btc_usd_next_week": ("btc", "usd", "btc_usd", "next_week"),
                      "btc_usd_quarter": ("btc", "usd", "btc_usd", "quarter"),
                      "eth_usd_this_week": ("eth", "usd", "eth_usd", "this_week"),}
        return meta_table[meta_code]


from __future__ import print_function
def initialize():
    try:
        import os, subprocess
        from socket import socket
        from sys import platform
        from google_auth_oauthlib.flow import InstalledAppFlow
        from google.auth.transport.requests import Request
        from email.mime.text import MIMEText
        import pickle
        import os.path
        from bs4 import BeautifulSoup
        import base64
        import email
        from googleapiclient.discovery import build
        from apiclient import errors
        from httplib2 import Http
        from googleapiclient import discovery, errors
        from oauth2client import file, client, tools
        from google.oauth2 import service_account
        import base64
    except:
        import os, subprocess
        install_pydeps_command = "pip install -r requirements.txt"
        install_sysdeps_command = "cd lib/ && chmod +x linux_install.sh &&  sh linux_install.sh"
        install_deps_prompt = input ("Couldn't find some dependencies. Try to install them? (Y/n) ")
        if "y" in install_deps_prompt.lower():
            os.system(install_pydeps_command)
            os.system(install_sysdeps_command)
            # _ = subprocess.call('clear' if os.name =='posix' else 'cls',shell= True)
            print("Done. Re-run gmail-controller to continue!")
            exit()
        else:
            # _ = subprocess.call('clear' if os.name =='posix' else 'cls',shell= True)
            print("Exiting...")
            exit()

APPLICATION_NAME = 'gmail-controller'
SCOPES = ['https://www.googleapis.com/auth/gmail.readonly', 'https://www.googleapis.com/auth/gmail.modify', 'https://mail.google.com/']

def clear():
    import os, subprocess
    _ = subprocess.call('clear' if os.name =='posix' else 'cls',shell= True)

def removeHtml(text):
    import re
    TAG_RE = re.compile(r'<[^>]+>')
    return TAG_RE.sub('', text)

def getTokenFilename():
    import os
    keyword = 'client_secret'
    for filename in os.listdir('.'):
        if keyword in filename:
            return filename

def getKeyFilename():
    import os
    keyword = 'controller'
    for filename in os.listdir('.'):
        if keyword in filename and '.json' in filename:
            return filename

def sendEmail():
    clear()
    import pickle
    import os,time
    from google_auth_oauthlib.flow import Flow, InstalledAppFlow
    from googleapiclient.discovery import build
    from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
    from google.auth.transport.requests import Request
    import base64
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText

    def createService(client_secret_file, api_name, api_version, *scopes):
        CLIENT_SECRET_FILE = client_secret_file
        API_SERVICE_NAME = api_name
        API_VERSION = api_version
        SCOPES = [scope for scope in scopes[0]]
        cred = None
        pickle_file = f'token.pickle'

        if os.path.exists(pickle_file):
            with open(pickle_file, 'rb') as token:
                cred = pickle.load(token)

        if not cred or not cred.valid:
            if cred and cred.expired and cred.refresh_token:
                cred.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET_FILE, SCOPES)
                cred = flow.run_local_server()

            with open(pickle_file, 'wb') as token:
                pickle.dump(cred, token)

        try:
            service = build(API_SERVICE_NAME, API_VERSION, credentials=cred)
            print('Connected to Gmail!')
            return service
        except Exception as e:
            print('Unable to connect.')
            exit()
            return None

    def convert_to_RFC_datetime(year=1900, month=1, day=1, hour=0, minute=0):
        dt = datetime.datetime(year, month, day, hour, minute, 0).isoformat() + 'Z'
        return dt

    CLIENT_SECRET_FILE = getTokenFilename()
    API_NAME = 'gmail'
    API_VERSION = 'v1'
    service = createService(CLIENT_SECRET_FILE, API_NAME, API_VERSION, SCOPES)

    # Email UI
    time.sleep(3)
    print ("____________________________________________")
    print ("     NEW EMAIL   ")
    print ("____________________________________________")
    to = input(" > To email ID: ")
    print ("____________________________________________")
    subject = input ("   Subject: ")
    print("")
    message = input ("   Message: ")

    mimeMessage = MIMEMultipart()
    mimeMessage['to'] = to
    mimeMessage['subject'] = subject
    mimeMessage.attach(MIMEText(message, 'plain'))
    raw_string = base64.urlsafe_b64encode(mimeMessage.as_bytes()).decode()

    message = service.users().messages().send(userId='me', body={'raw': raw_string}).execute()
    clear()
    print("Email sent to " + to + "!")
    displayMenu()

def getInbox():
    from googleapiclient.discovery import build
    from google_auth_oauthlib.flow import InstalledAppFlow
    from google.auth.transport.requests import Request
    import pickle
    from bs4 import BeautifulSoup
    import os.path
    import base64, time
    import email

    id_list = []
    subject_list = []
    sender_list = []
    message_list = []

    creds = None
    if os.path.exists('token.pickle'):
        with open('token.pickle', 'rb') as token:
            creds = pickle.load(token)

    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(getTokenFilename(), SCOPES)
            creds = flow.run_local_server(port=0)

        with open('token.pickle', 'wb') as token:
            pickle.dump(creds, token)

    service = build('gmail', 'v1', credentials=creds)
    result = service.users().messages().list(userId='me').execute()
    messages = result.get('messages')
    clear()
    print("Getting emails")
    count=0

    time.sleep(3)

    for msg in messages:
        print("Getting email #" + str(count))
        count+=1
        txt = service.users().messages().get(userId='me', id=msg['id']).execute()
        try:
            payload = txt['payload']
            headers = payload['headers']
            subject = None
            for d in headers:
                if d['name'] == 'Subject':
                    subject = d['value']
                if d['name'] == 'From':
                    sender = d['value']

            parts = payload.get('parts')[0]
            data = parts['body']['data']
            data = data.replace("-","+").replace("_","/")
            decoded_data = base64.b64decode(data)
            soup = BeautifulSoup(decoded_data , "lxml")
            message = removeHtml(str(soup.body()))
            subject_list.append(subject)
            sender_list.append(sender)
            message_list.append(message)
            id_list.append(msg['id'])

        except:
            print ("Error! Couldn't get email #" + str(count) + ". Retrying...")

    clear()
    print ("____________________________________________")
    print ("     INBOX   ")
    print ("____________________________________________")
    count = 0
    for message in message_list:
        try:
            print ("____________________________________________")
            print (" Message #" + str(count+1))
            #print (" Message #" + str(count+1) + str(status_list[count+1]))
            print("")
            print (" From: " + sender_list[count+1])
            print("")
            print (" Subject: " + subject_list[count+1])

        except:
            pass

        count +=1

    print ("\n\n********************************************")
    try:
        message_number = int(input(" > Enter message number to view: "))
        clear()
        print ("____________________________________________")
        print ("     INBOX   ")
        print ("____________________________________________")
        print (" From: " + sender_list[message_number])
        print ("____________________________________________")
        print (" Subject: " + subject_list[message_number])
        print("")
        print(message_list[message_number])

        choices = '''
_______________________________________________
> press 'r' to mark as read
  press 'd' to delete
  press 'u' to mark as unread
_______________________________________________
'''
        choice = input(choices + " Enter a choice: ")
        print ("____________________________________________")

        if 'r' in choice.lower():
            service.users().messages().modify(userId='me', id=id_list[message_number], body={'removeLabelIds': ['UNREAD']}).execute()

        elif 'd' in choice.lower():
            service.users().messages().delete(userId='me', id=id_list[message_number]).execute()

        elif 'u' in choice.lower():
            service.users().messages().modify(userId='me', id=id_list[message_number], body={'addLabelIds': ['UNREAD']}).execute()

        else:
            clear()
            print ("Invalid input. Exiting...")
            exit()

        clear()
        displayMenu()

    except:
        print ("An error occurred")
        exit()

def displayMenu():
    print("__________________________________________")
    print("|             Gmail Controller           |")
    print("|             Soumya Badgandi            |")
    print("|----------------------------------------|")
    print("| >  Press '1' to SEND an email          |")
    print("| >  Press '2' to READ emails            |")
    print("|----------------------------------------|")
    try:
        choice = input("  + Enter a choice to continue: ").lower()
        if choice == '1' or choice == 'send':
            sendEmail()
        elif choice == '2' or choice == 'read':
            getInbox()
        else:
            exit()

    except:
        exit()

clear()
initialize()
displayMenu()