import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import numpy as np
import torchvision.datasets as datasets
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms.functional as vF

DATASETS = {
    "cifar": datasets.CIFAR10,
    "mnist": datasets.MNIST,
    "emnist": datasets.EMNIST
}

def get_flat_grad_from(grad):
    flat_grad = torch.cat([torch.flatten(p) for p in grad])
    return flat_grad

def set_flat_params_to(model, flat_params):
    prev_ind = 0
    for param in model:
        flat_size = int(np.prod(list(param.size())))
        param.data.copy_(
            flat_params[prev_ind:prev_ind + flat_size].view(param.size()))
        prev_ind += flat_size

def average_grad(grads):
    # flatten the grads to tensors
    flat_grads = []
    for grad in grads:
        flat_grads.append(get_flat_grad_from(grad))

    average_flat_grad = torch.mean(torch.stack(flat_grads), dim=0)
    grad_0 = grads[0]
    average_grad_a = []
    for p in grad_0:
        average_grad_a.append(torch.zeros_like(p))

    set_flat_params_to(average_grad_a, average_flat_grad)

    # average_grad_b = []
    # for i, p in enumerate(grad_0):
    #     flat_average_grad_p = torch.mean(torch.stack([torch.flatten(grad[i]) for grad in grads]), dim=0)
    #     average_grad_b.append(flat_average_grad_p.view_as(p))

    return average_grad_a

def average_functions(models):
    average_model = models[0]
    sds = [model.state_dict() for model in models]
    average_sd = sds[0]
    for key in sds[0]:
        average_sd[key] = torch.mean(torch.stack([sd[key] for sd in sds]), dim=0)
    average_model.load_state_dict(average_sd)
    return average_model

def average_function_ensembles(function_ensembles):
    new_function_ensemble = FunctionEnsemble(empty=True, device=function_ensembles[0].device)
    n_ensembles = len(function_ensembles)
    for ensemble in function_ensembles:
        new_function_ensemble.add_ensemble(ensemble)
    new_function_ensemble.rescale_weights(1. / n_ensembles)
    return new_function_ensemble

def merge_function_ensembles(function_ensembles):
    new_function_ensemble = FunctionEnsemble(empty=True, device=function_ensembles[0].device)
    for ensemble in function_ensembles:
        for function, weight in zip(ensemble.function_list, ensemble.weight_list):
            new_function_ensemble.add_function(function, weight)
    return new_function_ensemble

class FunctionEnsemble(nn.Module):
    def __init__(self, get_init_function=None, device='cuda', empty=False):
        super(FunctionEnsemble, self).__init__()
        if not empty and get_init_function is None:
            raise NotImplementedError
        self.function_list = [] if empty is True else [get_init_function().to(device)]
        self.weight_list = [] if empty is True else [torch.tensor([0.0], device=device)]
        self.device = device

    def forward(self, x):
        y = torch.tensor((0.0), device=self.device)
        for (function, weight) in zip(self.function_list, self.weight_list):
            y = y + weight * function(x)

        return y

    def add_function(self, f, weight):
        self.function_list.append(f)
        self.weight_list.append(weight)

    def add_ensemble(self, ensemble):
        self.function_list = self.function_list + ensemble.function_list
        self.weight_list = self.weight_list + ensemble.weight_list

    def rescale_weights(self, factor):
        self.weight_list = [weight * factor for weight in self.weight_list]

    def is_empty(self):
        return True if len(self.function_list) == 0 else False

    def to(self, device):
        new_ensemble = FunctionEnsemble(device=device, empty=True)
        new_ensemble.function_list = [function.to(device) for function in self.function_list]
        new_ensemble.weight_list = [weight.to(device) for weight in self.weight_list]
        return new_ensemble

class WeakLearnerConv(nn.Module):
    def __init__(self, height, width, n_class=10, n_channels=3, hidden_size=(384, 192), device='cuda'):
        # super(WeakLearnerConv, self).__init__()
        # self.device = device
        # assert (width == 32 and height == 32)
        # self.activation = F.leaky_relu
        # self.conv1 = nn.Conv2d(n_channels, 6, 5)
        # self.pool = nn.MaxPool2d(2, 2)
        # self.conv2 = nn.Conv2d(6, 16, 5)
        # self.fc1 = nn.Linear(16 * 5 * 5, hidden_size[0])
        # self.fc2 = nn.Linear(hidden_size[0], hidden_size[1])
        # self.fc3 = nn.Linear(hidden_size[1], n_class)
        #
        # self.requires_grad_(False)
        # self.to(device)
        # self.apply(weights_init)

        super(WeakLearnerConv, self).__init__()
        self.device = device
        assert (width == 32 and height == 32)
        self.activation = F.leaky_relu
        self.conv1 = nn.Conv2d(n_channels, 64, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(64, 64, 5)
        self.fc1 = nn.Linear(64 * 5 * 5, hidden_size[0])
        self.fc2 = nn.Linear(hidden_size[0], hidden_size[1])
        self.fc3 = nn.Linear(hidden_size[1], n_class)

        self.requires_grad_(False)
        self.to(device)

    # def forward(self, x):
    #     x = self.pool(self.activation(self.conv1(x)))
    #     x = self.pool(self.activation(self.conv2(x)))
    #     x = x.view(-1, 16 * 5 * 5)
    #     x = self.activation(self.fc1(x))
    #     x = self.activation(self.fc2(x))
    #     x = self.fc3(x)
    #     return x

    def forward(self, x):
        x = self.pool(self.activation(self.conv1(x)))
        x = self.pool(self.activation(self.conv2(x)))
        x = x.view(x.shape[0], -1)
        x = self.activation(self.fc1(x))
        x = self.activation(self.fc2(x))
        x = self.fc3(x)
        return x

class WeakLearnerMLP(nn.Module):
    def __init__(self, height, width, n_class, n_channel=1, hidden_size=(128, 128), device='cuda'):
        super(WeakLearnerMLP, self).__init__()
        self.device = device
        self.height = height
        self.width = width
        self.n_class = n_class
        self.n_channel = n_channel
        self.fc1 = nn.Linear(width * height * n_channel, hidden_size[0])
        nn.init.normal_(self.fc1.weight.data, 0.0, 1 / self.width / self.height / self.n_channel)
        # linear layer (n_hidden -> hidden_2)
        self.fc2 = nn.Linear(hidden_size[0], hidden_size[1])
        nn.init.normal_(self.fc2.weight.data, 0.0, 1 / hidden_size[0])
        # linear layer (n_hidden -> 10)
        self.fc3 = nn.Linear(hidden_size[1], n_class)
        nn.init.normal_(self.fc3.weight.data, 0.0, 1 / hidden_size[1])
        # dropout layer (p=0.2)
        # dropout prevents overfitting of data
        # self.dropout = nn.Dropout(0.2)
        self.activation = F.leaky_relu

        self.to(self.device)
        self.requires_grad_(False)
        # self.apply(weights_init)

    def forward(self, x):
        # flatten image input
        x = x.view(-1, self.height * self.width * self.n_channel)
        # add hidden layer, with relu activation function
        x = self.activation(self.fc1(x))
        x = self.activation(self.fc2(x))
        x = self.activation(self.fc3(x))
        return x

def weak_oracle(target, data, lr, oracle_steps, init_weak_learner, mb_size=500):
    g = init_weak_learner
    MSEloss = nn.MSELoss()

    target = target.detach()
    # target = Dx_loss(f(data), label)
    # print(torch.norm(target)/(1e-8 + torch.norm(residual)))
    # Dx_loss(f(data), label) is the subgradient
    g.requires_grad_(True)
    # index = torch.tensor([0, 1, 2, 3, 4, 5])
    optimizer = optim.Adam(g.parameters(), lr=lr)
    # optimizer = optim.SGD(g.parameters(), lr=lr, momentum=.9, weight_decay=1.e-4)
    num_epochs = oracle_steps * mb_size // data.shape[0] + 1
    for epoch in range(num_epochs):
        shuffle_index = torch.randperm(data.shape[0])
        data, target = data[shuffle_index], target[shuffle_index]
        _p = 0
        while _p + mb_size <= data.shape[0]:
            optimizer.zero_grad()
            loss = MSEloss(target[_p: _p+mb_size], g(data[_p: _p+mb_size]))
            loss.backward()
            optimizer.step()
            _p += mb_size

    # print(MSEloss(target, g(data))/torch.norm(target))
    # print(torch.norm(target))
    g.requires_grad_(False)
    if data.shape[0] <= 5000:
        g_data = g(data)
    else: # make sure the the batch size is no more than 5000
        num_chunk = data.shape[0] // 5000 + 1
        g_data = torch.cat([g(_d) for _d in torch.chunk(data, num_chunk)])
    residual = target - g_data
    # print(torch.norm(residual)/torch.norm(target))
    # print(torch.norm(g_data)/torch.norm(target))
    # print(torch.norm(residual[index])/torch.norm(target[index]))
    # print(torch.norm(g_data[index])/torch.norm(target[index]))

    return g, residual, g_data

def distill_oracle(target, data, lr, oracle_steps, init_weak_learner, mb_size=500):
    g = init_weak_learner
    MSEloss = nn.MSELoss()

    target = target.detach()
    g.requires_grad_(True)
    # index = torch.tensor([0, 1, 2, 3, 4, 5])
    optimizer = optim.Adam(g.parameters(), lr=lr)
    # optimizer = optim.SGD(g.parameters(), lr=lr, momentum=.9, weight_decay=1.e-4)
    num_epochs = oracle_steps * mb_size // data.shape[0] + 1
    for epoch in range(num_epochs):
        shuffle_index = torch.randperm(data.shape[0])
        data, target = data[shuffle_index], target[shuffle_index]
        _p = 0
        while _p + mb_size <= data.shape[0]:
            optimizer.zero_grad()
            loss = MSEloss(target[_p: _p+mb_size], g(data[_p: _p+mb_size]))
            loss.backward()
            optimizer.step()
            _p += mb_size

    del loss
    resi_norm = torch.mean(torch.sum((target - g(data))**2, dim=(1,)))
    print(f"residual norm of the distillation oracle {resi_norm}")
    return g

class FWeakOracle:
    def __init__(self, workers, n_rounds=20, local_sgd_step_size=1e-3):
        self.n_rounds = n_rounds  # number of communication rounds for every federated oracle query
        self.workers = workers
        self.n_workers = len(workers)
        self.local_sgd_step_size = local_sgd_step_size

    def query(self, f_global, init_weak_learner):
        g = init_weak_learner
        for worker in self.workers:
            worker.compute_subgradient(f_global)

        for i in range(self.n_rounds):
            lr = self.local_sgd_step_size / (0.1 * i + 1)
            g = average_functions([worker.local_sgd(g, lr) for worker in self.workers])

        for worker in self.workers:
            worker.update_residual(g)

        print(torch.max(torch.stack([torch.norm(worker.residual) for worker in self.workers])).item())

        return g

def get_step_size_scheme(n_round, step_size_0, local_steps, p=1):
    def step_size_scheme(k):
        # return step_size_0/((n_round+1) * local_steps + k + 1)**p
        return step_size_0 / ((n_round + 1) * local_steps + k + 1) ** p

    return step_size_scheme

def get_init_weak_learner(height, width, n_channel, n_class, hidden_size, type, device='cuda'):
    if type == "MLP":
        return WeakLearnerMLP(height=height, width=width, n_class=n_class, hidden_size=hidden_size, device=device)
    elif type == "Conv":
        return WeakLearnerConv(height=height, width=width, n_class=n_class, n_channels=n_channel,
                               hidden_size=hidden_size, device=device)
    else:
        raise NotImplementedError("Unknown weak learner type")

def data_partition(data, label, n_workers, homo_ratio):
    '''

    :param data:
    :param label:
    :param n_workers:
    :param homo_ratio: the portion of data to be allocated iid among $n_workers$ workers
    :param n_augment: augment the data set
    :return: lists of chunked data and labels
    '''

    if n_workers == 1:
        return [data], [label]

    assert data.shape[0] == label.shape[0]

    if n_workers == 1:
        return [data], [label]

    n_data = data.shape[0]

    n_homo_data = int(n_data * homo_ratio)

    n_homo_data = n_homo_data - n_homo_data % n_workers
    n_data = n_data - n_data % n_workers

    if n_homo_data > 0:
        data_homo, label_homo = data[0:n_homo_data], label[0:n_homo_data]
        data_homo_list, label_homo_list = data_homo.chunk(n_workers), label_homo.chunk(n_workers)

    if n_homo_data < n_data:
        data_hetero, label_hetero = data[n_homo_data:n_data], label[n_homo_data:n_data]
        label_hetero_sorted, index = torch.sort(label_hetero)
        data_hetero_sorted = data_hetero[index]

        data_hetero_list, label_hetero_list = data_hetero_sorted.chunk(n_workers), label_hetero_sorted.chunk(n_workers)

    if 0 < n_homo_data < n_data:
        data_list = [torch.cat([data_homo, data_hetero], dim=0) for data_homo, data_hetero in
                     zip(data_homo_list, data_hetero_list)]
        label_list = [torch.cat([label_homo, label_hetero], dim=0) for label_homo, label_hetero in
                      zip(label_homo_list, label_hetero_list)]
    elif n_homo_data < n_data:
        data_list = data_hetero_list
        label_list = label_hetero_list
    else:
        data_list = data_homo_list
        label_list = label_homo_list

    return data_list, label_list

def make_dataloaders(data_list, label_list, transforms=None):
    dss = make_datasets(data_list, label_list, transforms)
    dataloaders = [DataLoader(ds, batch_size=64, shuffle=True, num_workers=4) for ds in dss]
    return dataloaders

def make_adv_label(label_list, n_classes):
    # For machine i, define r = i%n. All data points on machine i that has label r is placed with label (r+1)%n_classes
    # n_workers = len(label_list)
    n_changed = 0
    n_total = 0
    new_label_list = []
    for i, label in enumerate(label_list):
        n_total += label.shape[0]
        r1 = i % n_classes
        index_1 = (label == r1).nonzero()
        n_changed += index_1.shape[0]
        r2 = (i + 1) % n_classes
        index_2 = (label == r2).nonzero()
        n_changed += index_2.shape[0]
        r1_new = (r1 + 1) % n_classes
        r2_new = (r2 + 1) % n_classes
        label[index_1] = r1_new
        label[index_2] = r2_new
        new_label_list.append(label)

    print(n_changed / n_total)
    return new_label_list

def load_data(args, hidden_size, device, augment_data=False):
    dataset_handle = DATASETS[args.dataset]

    if args.dataset == "mnist":
        dataset = dataset_handle(root='datasets/' + args.dataset, download=True)
        dataset_test = dataset_handle(root='datasets/' + args.dataset, train=False, download=True)
        data, label = dataset.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                      dataset.train_labels.to(device=device)
        data = (data - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data.shape[0] == label.shape[0])
        (n_data, height, width) = data.shape
        n_class = 10

        if augment_data:
            data_h = vF.hflip(data)
            data = torch.cat([data, data_h], dim=0)
            label = torch.cat([label, label], dim=0)

        n_channel = 1
        rand_index = torch.randperm(data.shape[0])
        data, label = data[rand_index], label[rand_index]
        get_init_function = lambda: get_init_weak_learner(height, width, n_channel, n_class,
                                                          hidden_size=hidden_size, type="MLP", device=device)

        data_test, label_test = dataset_test.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                                dataset_test.train_labels.to(device=device)
        data_test = (data_test - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data_test.shape[0] == label_test.shape[0])
        del rand_index

    elif args.dataset == "emnist":
        dataset = dataset_handle(root='datasets/' + args.dataset, split="byclass", download=True)
        dataset_test = dataset_handle(root='datasets/' + args.dataset, train=False, split="byclass", download=True)
        data, label = dataset.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                      dataset.train_labels.to(device=device)
        data = (data - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data.shape[0] == label.shape[0])
        (n_data, height, width) = data.shape
        n_class = torch.unique(label).shape[0]

        if augment_data:
            data_h = vF.hflip(data)
            data = torch.cat([data, data_h], dim=0)
            label = torch.cat([label, label], dim=0)

        n_channel = 1
        rand_index = torch.randperm(data.shape[0])
        data, label = data[rand_index], label[rand_index]
        get_init_function = lambda: get_init_weak_learner(height, width, n_channel, n_class,
                                                          hidden_size=hidden_size, type="MLP", device=device)

        data_test, label_test = dataset_test.train_data.to(dtype=torch.float32, device=device) / 255.0, \
                                dataset_test.train_labels.to(device=device)
        data_test = (data_test - torch.tensor(0.1307, device=device)) / torch.tensor(0.3081, device=device)
        assert (data_test.shape[0] == label_test.shape[0])
        del rand_index

    elif args.dataset == "cifar":
        dataset = dataset_handle(root='datasets/' + args.dataset, download=True)
        dataset_test = dataset_handle(root='datasets/' + args.dataset, train=False, download=True)
        # processing training data
        data, label = torch.tensor(dataset.data, dtype=torch.float32, device=device) / 255.0, \
                      torch.tensor(dataset.targets, device=device)
        # normalize
        data = (data - torch.tensor((0.4914, 0.4822, 0.4465), device=device)) / torch.tensor((0.2023, 0.1994, 0.2010), device=device)
        data = data.permute(0, 3, 1, 2)  # from (H, W, C) to (C, H, W)

        # processing testing data
        data_test, label_test = torch.tensor(dataset_test.data, dtype=torch.float32, device=device) / 255.0, \
                                torch.tensor(dataset_test.targets, device=device)
        # normalize
        data_test = (data_test - torch.tensor((0.4914, 0.4822, 0.4465), device=device)) / torch.tensor((0.2023, 0.1994, 0.2010),
                                                                                device=device)
        data_test = data_test.permute(0, 3, 1, 2)  # from (H, W, C) to (C, H, W)

        assert (data.shape[0] == label.shape[0])
        (n_data, n_channel, height, width) = data.shape
        n_class = 10
        if augment_data:
            data_h = vF.hflip(data)
            data = torch.cat([data, data_h], dim=0)
            label = torch.cat([label, label], dim=0)

        rand_index = torch.randperm(data.shape[0])
        data, label = data[rand_index], label[rand_index]
        get_init_function = lambda: get_init_weak_learner(height, width, n_channel, n_class,
                                                          hidden_size=hidden_size, type="Conv", device=device)

        del rand_index
    else:
        raise NotImplementedError

    return data, label, data_test, label_test, n_class, get_init_function

def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:min(len(lst), i + n)]

def is_nan(x):
    return x != x

def make_dataset(data, label, transform):
    class LocalDataset(Dataset):
        def __init__(self, data, label, transform=None):
            self.data = data
            self.label = label
            self.transform = transform
            assert data.shape[0] == label.shape[0]

        def __len__(self):
            return self.data.shape[0]

        def __getitem__(self, item):
            sample = self.data[item]
            if self.transform:
                sample = self.transform(sample)
            return sample, self.label[item]

    return LocalDataset(data, label)

def make_datasets(data_list, label_list, transform=None):
    return [make_dataset(data, label, transform) for data, label in zip(data_list, label_list)]


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
df = pd.read_csv("heart.csv")
df.head()
#import pandas_profiling
#profile = pandas_profiling.ProfileReport(df)
#profile.to_file('profile.html')
import seaborn as sns
df.corr()#相关性分析
#plt.figure(figsize = (12,12))
#sns.heatmap(df.corr(),vmax =3,center=0,annot=True,linewidths= 5,cbar_kws={"shrink":.5},fmt='.1f',square=True)
#plt.tight_layout()
#plt.show()
#sns.pairplot(df)
#plt.show()
#sns.distplot(df['age'])
#plt.show()
#sns.countplot(x='target',data=df,palette='bwr')
#plt.show()
#plt.xlabel('Sex(0=female,1=male)')
#plt.show()
#pd.crosstab(df.age,df.target).plot(kind="bar",figsize=(20,6))
#plt.xlabel('Age')
#plt.ylabel('Frequency')
#plt.savefig('age-heartdisease')
#plt.show()
#sns.boxplot(x=df['target'],y=df['age'])
#plt.show()
#sns.violinplot(x=df['target'],y=df['age'])
#plt.show()
#plt.scatter(x=df.age[df.target ==1],y=df.thalach[(df.target==1)],c="red")
#plt.scatter(x=df.age[df.target ==0],y=df.thalach[(df.target==0)],c="blue")
#plt.legend(["Disease","Not Disease"])
#plt.xlabel("Age")
#plt.ylabel("Max Rate")
#plt.show()
df.columns = ['age','sex','chest_pain_type','resting_blood_pressure','cholesterol','fasting_blood_sugar','rest_ecg','max_heart_rate','exercise_induced_angina','st_depression','st_slope','num_major_vessels','thalassemia','target']
#df.head()
df['sex'][df['sex']==0]='female'
df['sex'][df['sex']==1]='male'
df['chest_pain_type'][df['chest_pain_type']==0]='typical_angina'
df['chest_pain_type'][df['chest_pain_type']==1]='atypical_angina'
df['chest_pain_type'][df['chest_pain_type']==2]='non_angina pain'
df['chest_pain_type'][df['chest_pain_type']==3]='asymptomatic'

df['fasting_blood_sugar'][df['fasting_blood_sugar']==0]='lower than 120'
df['fasting_blood_sugar'][df['fasting_blood_sugar']==1]='more than 120'

df['rest_ecg'][df['rest_ecg']==0]='normal'
df['rest_ecg'][df['rest_ecg']==1]='ST-T wave abnormality'
df['rest_ecg'][df['rest_ecg']==2]='left ventricular hypertrophy'

df['exercise_induced_angina'][df['exercise_induced_angina']==0]='no'
df['exercise_induced_angina'][df['exercise_induced_angina']==1]='yes'

df['st_slope'][df['st_slope']==0]='upsloping'
df['st_slope'][df['st_slope']==1]='flat'
df['st_slope'][df['st_slope']==2]='downsloping'

df['thalassemia'][df['thalassemia']==0]='unknown'
df['thalassemia'][df['thalassemia']==1]='normal'
df['thalassemia'][df['thalassemia']==2]='fixed defect'
df['thalassemia'][df['thalassemia']==3]='reserable defect'
df.head()
df = pd.get_dummies(df)
df.to_csv('process_heart.csv',index=False)
from pdpbox import pdp,get_dataset,info_plots
#fig,axes,summary_df=info_plots.target_plot(df =df,feature='sex_male',feature_name='gender',target=['target'])
#_=axes['bar_ax'].set_xticklabels(['Female','Male'])
#plt.show()
#summary_df
#fig,axes,summary_df=info_plots.target_plot(df =df,feature='num_major_vessels',feature_name='num_vessels',target=['target'])
#_=axes['bar_ax'].set_xticklabels(['Female','Male'])
#fig,axes,summary_df=info_plots.target_plot(df =df,feature='age',feature_name='age',target=['target'])
#_=axes['bar_ax'].set_xticklabels(['Female','Male'])
print(df)
X = df.drop('target',axis=1)#吧除了target之外的列作为特征
print(X.shape)
Y = df['target']#取出所有属于target下面的数据作为y
print(Y.shape)
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_tests = train_test_split(X,Y,test_size=0.2,random_state = 10)
print(X_train.shape)
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier(n_estimators=100,max_depth=5,random_state=5)
model.fit(X_train,y_train)
len(model.estimators_)
estimator = model.estimators_[7]#指定索引为7的决策树
print(estimator)
#将特征值转化为字符串
feature_names = X_train.columns
y_train_str = y_train.astype('str')
y_train_str[y_train_str=='0']='no disease'
y_train_str[y_train_str=='1']='disease'
y_trian_str=y_train_str.values
from sklearn.tree import export_graphviz
export_graphviz(estimator,out_file='tree.dot',feature_names=feature_names,class_names=['no disease','disease'],rounded = True,proportion=True,label='root',precision=2,filled=True)
#下面的dot是linux下面才有的  是为了把上面生成的文本tree.dot转化为png
#from subprocess import call
#call(['dot','-Tpng','tree.dot','-o','tree.png','-Gdpi=600'])
#由于上一步做不到，本步骤忽略  其实不影响什么，只是树不能可视化了而已
#from IPython.display import Image
#Image(filename ='tree.png')
import eli5
eli5.show_weights(estimator,feature_names=feature_names.to_list())
model.feature_importances_
print('特征排序：')
feature_names = X_test.columns
feature_importances = model.feature_importances_
indices=np.argsort(feature_importances)[::-1]
plt.figure(figsize=(16,8))
plt.title("Feature Importance")
plt.bar(range(len(feature_importances)),feature_importances[indices],color='b')
plt.xticks(range(len(feature_importances)),np.array(feature_names)[indices],color='b',rotation=90)
plt.show()


from django import forms
from .models import Product

class ProductForm(forms.ModelForm):
    id = forms.IntegerField(required=False, widget=forms.HiddenInput())
    image = forms.ImageField(label='Ảnh sản phẩm', required=False)

    class Meta:
        model = Product
        fields = ('category', 'code', 'name', 'description', 'unitPrice', 'freeShip')

        error_messages = {
            "code": {
                "unique": "Mã sản phẩm đã tồn tại"
            }
        }

    def clean_image(self):
        image = self.cleaned_data.get('image')
        id = self.cleaned_data.get('id')

        if id == None and image == None:
            raise forms.ValidationError('Cần chọn ảnh sản phẩm')

        return image

    def save(self):
        product = super().save()
        image = self.cleaned_data['image']

        if image and image.name:
            product.saveImage(image)

class PaymentForm(forms.Form):
    fullname = forms.CharField(label='Họ và tên')
    mobile = forms.CharField(label='Số điện thoại')
    email = forms.EmailField(label='Email')
    address = forms.CharField(label='Địa chỉ')


# Standard Library Imports
import collections
from datetime import datetime, timedelta
import logging
import time
import json
import traceback

# 3rd Party Imports
from requests.packages.urllib3.util import Retry
import requests
from gevent.lock import Semaphore

# Local Imports
from PokeAlarm import Unknown
from PokeAlarm.Utilities.GenUtils import synchronize_with

log = logging.getLogger("Gmaps")

class GMaps(object):

    # Available travel modes for Distance Matrix calls
    TRAVEL_MODES = frozenset(["walking", "biking", "driving", "transit"])

    # Maximum number of requests per second
    _queries_per_second = 50
    # How often to warn about going over query limit
    _warning_window = timedelta(minutes=1)

    def __init__(self, api_key):
        self._key = api_key
        self._lock = Semaphore

        # Create a session to handle connections
        self._session = self._create_session()

        # Sliding window for rate limiting
        self._window = collections.deque(maxlen=self._queries_per_second)
        self._time_limit = datetime.utcnow()

        # Memoization dicts
        self._geocode_hist = {}
        self._reverse_geocode_hist = {}
        self._dm_hist = {key: dict() for key in self.TRAVEL_MODES}

    # TODO: Move into utilities
    @staticmethod
    def _create_session(retry_count=3, pool_size=3, backoff=0.25):
        """Create a session to use connection pooling."""

        # Create a session for connection pooling and
        session = requests.Session()

        # Reattempt connection on these statuses
        status_forcelist = [500, 502, 503, 504]

        # Define a Retry object to handle failures
        retry_policy = Retry(
            total=retry_count, backoff_factor=backoff, status_forcelist=status_forcelist
        )

        # Define an Adapter, to limit pool and implement retry policy
        adapter = requests.adapters.HTTPAdapter(
            max_retries=retry_policy, pool_connections=pool_size, pool_maxsize=pool_size
        )

        # Apply Adapter for all HTTPS (no HTTP for you!)
        session.mount("https://", adapter)

        return session

    def _make_request(self, service, params=None):
        """Make a request to the GMAPs API."""
        # Rate Limit - All APIs use the same quota
        if len(self._window) == self._queries_per_second:
            # Calculate elapsed time since start of window
            elapsed_time = time.time() - self._window[0]
            if elapsed_time < 1:
                # Sleep off the difference
                time.sleep(1 - elapsed_time)

        # Create the correct url
        url = f"https://maps.googleapis.com/maps/api/{service}/json"

        # Add in the API key
        if params is None:
            params = {}
        params["key"] = self._key

        # Use the session to send the request
        log.debug("%s request sending.", service)
        self._window.append(time.time())
        request = self._session.get(url, params=params, timeout=3)

        if not request.ok:
            log.debug(
                "Response body: %s",
                json.dumps(request.json(), indent=4, sort_keys=True),
            )
            # Raise HTTPError
            request.raise_for_status()

        log.debug(
            "%s request completed successfully with response %s.",
            service,
            request.status_code,
        )
        body = request.json()
        if body["status"] == "OK" or body["status"] == "ZERO_RESULTS":
            return body
        elif body["status"] == "OVER_QUERY_LIMIT":
            raise UserWarning("API Quota exceeded.")
        else:
            raise ValueError(f"Unexpected response status:\n {body}")

    @synchronize_with()
    def geocode(self, address, language="en"):
        # type: (str, str) -> tuple
        """Returns 'lat,lng' associated with the name of the place."""
        # Check for memoized results
        address = address.lower()
        if address in self._geocode_hist:
            return self._geocode_hist[address]
        # Set default in case something happens
        latlng = None
        try:
            # Set parameters and make the request
            params = {"address": address, "language": language}
            response = self._make_request("geocode", params)
            # Extract the results and format into a dict
            response = response.get("results", [])
            response = response[0] if len(response) > 0 else {}
            response = response.get("geometry", {})
            response = response.get("location", {})
            if "lat" in response and "lng" in response:
                latlng = float(response["lat"]), float(response["lng"])

            # Memoize the results
            self._geocode_hist[address] = latlng
        except requests.exceptions.HTTPError as e:
            log.error("Geocode failed with HTTPError: %s", e.message)
        except requests.exceptions.Timeout as e:
            log.error("Geocode failed with connection issues: %s", e.message)
        except UserWarning:
            log.error("Geocode failed because of exceeded quota.")
        except Exception as e:
            log.error(
                "Geocode failed because unexpected error has occurred: %s - %s",
                type(e).__name__,
                e,
            )
            log.error("Stack trace: \n %s", traceback.format_exc())
        # Send back tuple
        return latlng

    _reverse_geocode_defaults = {
        "street_num": Unknown.SMALL,
        "street": Unknown.REGULAR,
        "address": Unknown.REGULAR,
        "address_eu": Unknown.REGULAR,
        "postal": Unknown.REGULAR,
        "neighborhood": Unknown.REGULAR,
        "sublocality": Unknown.REGULAR,
        "city": Unknown.REGULAR,
        "county": Unknown.REGULAR,
        "state": Unknown.REGULAR,
        "country": Unknown.REGULAR,
    }

    @synchronize_with()
    def reverse_geocode(self, latlng, language="en"):
        # type: (tuple, str) -> dict
        """Returns the reverse geocode DTS associated with 'lat,lng'."""
        latlng = f"{latlng[0]:.5f},{latlng[1]:.5f}"
        # Check for memoized results
        if latlng in self._reverse_geocode_hist:
            return self._reverse_geocode_hist[latlng]
        # Get defaults in case something happens
        dts = self._reverse_geocode_defaults.copy()
        try:
            # Set parameters and make the request
            params = {"latlng": latlng, "language": language}
            response = self._make_request("geocode", params)
            # Extract the results and format into a dict
            response = response.get("results", [])
            response = response[0] if len(response) > 0 else {}
            details = {}
            for item in response.get("address_components"):
                for category in item["types"]:
                    details[category] = item["short_name"]

            # Note: for addresses on unnamed roads, EMPTY is preferred for
            # 'street_num' and 'street' to avoid DTS looking weird
            dts["street_num"] = details.get("street_number", Unknown.EMPTY)
            dts["street"] = details.get("route", Unknown.EMPTY)
            dts["address"] = f'{dts["street_num"]} {dts["street"]}'
            dts["address_eu"] = f'{dts["street"]} {dts["street_num"]}'
            # Europeans are backwards
            dts["postal"] = details.get("postal_code", Unknown.REGULAR)
            dts["neighborhood"] = details.get("neighborhood", Unknown.REGULAR)
            dts["sublocality"] = details.get("sublocality", Unknown.REGULAR)
            dts["city"] = details.get(
                "locality", details.get("postal_town", Unknown.REGULAR)
            )
            dts["county"] = details.get("administrative_area_level_2", Unknown.REGULAR)
            dts["state"] = details.get("administrative_area_level_1", Unknown.REGULAR)
            dts["country"] = details.get("country", Unknown.REGULAR)

            # Memoize the results
            self._reverse_geocode_hist[latlng] = dts
        except requests.exceptions.HTTPError as e:
            log.error("Reverse Geocode failed with HTTPError: %s", e.message)
        except requests.exceptions.Timeout as e:
            log.error("Reverse Geocode failed with connection issues: %s", e.message)
        except UserWarning:
            log.error("Reverse Geocode failed because of exceeded quota.")
        except Exception as e:
            log.error(
                "Reverse Geocode failed because unexpected error has occurred: %s - %s",
                type(e).__name__,
                e,
            )
            log.error("Stack trace: \n %s", traceback.format_exc())
        # Send back dts
        return dts

    @synchronize_with()
    def distance_matrix(self, mode, origin, dest, lang, units):
        # Check for valid mode
        if mode not in self.TRAVEL_MODES:
            raise ValueError(f"DM doesn't support mode '{mode}'.")
        # Estimate to about ~1 meter of accuracy
        origin = f"{origin[0]:.5f},{origin[1]:.5f}"
        dest = f"{dest[0]:.5f},{dest[1]:.5f}"

        # Check for memorized results
        key = f"{origin}:{dest}"
        if key in self._dm_hist:
            return self._dm_hist[key]

        # Set defaults in case something happens
        dist_key = f"{mode}_distance"
        dur_key = f"{mode}_duration"
        dts = {dist_key: Unknown.REGULAR, dur_key: Unknown.REGULAR}
        try:
            # Set parameters and make the request
            params = {
                "mode": mode,
                "origins": origin,
                "destinations": dest,
                "language": lang,
                "units": units,
            }

            # Extract the results and format into a dict
            response = self._make_request("distancematrix", params)
            response = response.get("rows", [])
            response = response[0] if len(response) > 0 else {}
            response = response.get("elements", [])
            response = response[0] if len(response) > 0 else {}

            # Set the DTS
            dts[dist_key] = response.get("distance", {}).get("text", Unknown.REGULAR)
            dts[dur_key] = response.get("duration", {}).get("text", Unknown.REGULAR)
        except requests.exceptions.HTTPError as e:
            log.error("Distance Matrix failed with HTTPError: %s", e.message)
        except requests.exceptions.Timeout as e:
            log.error("Distance Matrix failed with connection issues: %s", e.message)
        except UserWarning:
            log.error("Distance Matrix failed because of exceeded quota.")
        except Exception as e:
            log.error(
                "Distance Matrix failed because unexpected error has occurred: %s - %s",
                type(e).__name__,
                e,
            )
            log.error("Stack trace: \n %s", traceback.format_exc())
        # Send back DTS
        return dts


#TODO: integrate fully in ClearMap

__author__    = 'Sophie Skriabin, Christoph Kirst <christoph.kirst.ck@gmail.com>'
__license__   = 'GPLv3 - GNU General Pulic License v3 (see LICENSE.txt)'
__copyright__ = 'Copyright © 2020 by Christoph Kirst'
__webpage__   = 'http://idisco.info'
__download__  = 'http://www.github.com/ChristophKirst/ClearMap2'

import vispy
import vispy.color
import vispy.scene
from vispy import scene
import vispy.app
import vispy.visuals
import numpy as np
import os
import vesselSegmentation.TurntableCamera as tc
import vesselSegmentation.IO.MMP as mp

# from skimage.measure import block_reduce

class twoClassesMap(vispy.color.colormap.BaseColormap):
    glsl_map = """
    vec4 threeclasses(float t) {
        if(t == 0){
            return vec4(0, 0, 0, 0);
        }
        else if (t == 1) {
            return vec4(0.8, 0.1, 0.1, 0.3);
        }
        else {
            return vec4(0.8, 0.1, 0.1, 0.3);
        }
    }
    """
    """
    t should take values in 0, 1, 2
    """

    def map(self, t):
        if isinstance(t, np.ndarray):
            return np.hstack([t, 0.1, 1-t, 1]).astype(np.float32)
        else:
            return np.array([t, 0.1, 1-t, 1], dtype=np.float32)

class FireMap(vispy.color.colormap.BaseColormap):
    colors = [(1.0, 1.0, 1.0, 0.0),
              (1.0, 1.0, 0.0, 0.05),
              (1.0, 0.0, 0.0, 0.1)]

    glsl_map = """
    vec4 fire(float t) {
        return mix(mix($color_0, $color_1, t),
                   mix($color_1, $color_2, t*t), t);
    }
    """

def show():
    vispy.app.run()

def get_two_views():
    """
    Get two views in order to plot two graphs/images in a consistent manner
    """
    canvas = vispy.scene.SceneCanvas(keys='interactive', title='plot3d', show=True)
    vb1 = scene.widgets.ViewBox(border_color='yellow', parent=canvas.scene, camera=tc.ArbitraryRotationCamera())
    vb2 = scene.widgets.ViewBox(border_color='blue', parent=canvas.scene, camera=tc.ArbitraryRotationCamera())

    grid = canvas.central_widget.add_grid()
    grid.padding = 6
    grid.add_widget(vb1, 0, 0)
    grid.add_widget(vb2, 0, 1)

    for view in vb1, vb2:
        # view.camera = 'turntable'
        view.camera.fov = 100
        view.camera.distance = 0
        view.camera.elevation = 0
        view.camera.azimuth = 0

    vb1.camera.aspect = vb2.camera.aspect = 1
    vb1.camera.link(vb2.camera)

    return vb1, vb2

def plot3d(data, colormap=FireMap(), view=None):
    VolumePlot3D = vispy.scene.visuals.create_visual_node(vispy.visuals.VolumeVisual)
    # Add a ViewBox to let the user zoom/rotate
    # build canvas
    if view is None:
        canvas = vispy.scene.SceneCanvas(keys='interactive', title='plot3d', show=True)
        view = canvas.central_widget.add_view(camera=tc.ArbitraryRotationCamera())
        # view.camera = 'turntable'
        view.camera.fov = 0
        view.camera.distance = 7200
        view.camera.elevation = 31
        view.camera.azimuth = 0
        view.camera.depth_value = 100000000

    cc = (np.array(data.shape) // 2)
    view.camera.center = cc

    return VolumePlot3D(data.transpose([2, 1, 0]), method='translucent', relative_step_size=1.5,
                        parent=view.scene, cmap=colormap)

def reconstructionView(readdir, N):
    img = np.load(os.path.join(readdir, 'input_' + str(N) + '.npy'))
    output = np.load(os.path.join(readdir, 'output_' + str(N) + '.npy'))
    gt = np.load(os.path.join(readdir, 'groundtruth_' + str(N) + '.npy'))

    print("img", np.unique(img))
    print("output", np.unique(output))

    vb1, vb2 = get_two_views()
    plot3d(img, view=vb1)
    plot3d(gt+img, view=vb2)
    show()

def viewFile(filepath):
    img = np.load(filepath)

    # print("img", np.unique(img))
    # np.save('/mnt/vol00-renier/Sophie/DataToPlot/resDNN/outputcrustfull06thresh.npy', img>0.6)
    super_threshold_indices = img <= 0.6
    img[super_threshold_indices] = 0
    vb1, vb2 = get_two_views()
    plot3d(img, view=vb1)
    show()

if __name__ == '__main__':
    #
    # readdir = 'logs/training240419_sparseMidVessTest2'
    readdir = 'logs/training050719_test'
    #'logs/training060319_axons_recog_test_1'
    N = 15100#49800#41850
    #59200#133150#117300 #139250#
    #40
    reconstructionView(readdir, N)
    # #
    # input = np.load(os.path.join(readdir, 'output_' + str(N) + '.npy'))
    # output = np.load(os.path.join(readdir, 'groundtruth_' + str(N) + '.npy'))
    # #
    # #
    # input=np.load('/mnt/data_2to/Axons/data/test2/data_binary_center.npy')
    # output = np.load('/mnt/data_2to/Axons/data/res_test2.npy')
    # input = np.load('/mnt/data_SSD_2to/190123_7/inputinput.npy')
    # output =np.load('/mnt/data_SSD_2to/190123_7/outputoutput.npy')
    # # input, output[1500:1700, 3750:3950, 2380:2580]
    # # # output=block_reduce(output, block_size=(2, 2, 2), func=np.maximum).astype(int)
    # vb1, vb2 = get_two_views()
    # # # plot3d(arr, view=vb2)
    # # # show()
    # # # print(np.unique(0.47-latent1))
    # plot3d(input>0.5, view=vb1)
    # plot3d(output, view=vb2)
    # show()
    # # viewFile('/media/sophie.skriabine/TOSHIBA EXT/Sophie/teryData/arteryData/data_arteries_crust.npy')
    # arr=np.load('/mnt/vol00-renier/Nicolas/data_binary.npy')
    # plot3d(arr)
    # show()GXFGv


import speech_recognition as sr
import webbrowser
import time
import playsound
import os
import random
from gtts import gTTS
from time import ctime
import pyaudio

recognizer = sr.Recognizer()

def record_audio(ask = False):
  with sr.Microphone() as source:
    recognizer.adjust_for_ambient_noise(source)  # here
    if ask:
      v_speak(ask)
    audio = recognizer.listen(source)
    voice_data = ''
    try:
      voice_data = recognizer.recognize_google(audio)
    except sr.UnknownValueError:
      v_speak('Sorry I did not get that. Try again')
    except sr.RequestError:
      v_speak('Sorry my speech service is down')
    return voice_data

def v_speak(audio_string):
  tts = gTTS(text=audio_string, lang='en')
  r = random.randint(1, 1000000)
  audio_file='audio-' + str(r) + '.mp3'
  tts.save(audio_file)
  playsound.playsound(audio_file)
  print(audio_string)
  os.remove(audio_file)

def respond(voice_data):
  if 'change color' in voice_data:
    v_speak("Of What?")
  if 'background' in voice_data:
    v_speak('What color would you like?')

  if 'red' in voice_data:
    v_speak('ok changing to ' + voice_data)
    with open('styles.css', 'r+') as f:

    #read file
      file_source = f.read()

      #replace 'PHP' with 'PYTHON' in the file
      replace_string = file_source.replace('background-color: aqua', 'background-color: red')

      #save output
      write_file = f.write(replace_string)
      print(write_file)

  if 'what is your name' in voice_data:
    v_speak('My name is Ding')
  if 'what time is it' in voice_data:
    v_speak(ctime())
  if 'search' in voice_data:
    search = record_audio('What do you want to search for?')
    url = 'https://google.com/search?q=' + search
    webbrowser.get().open(url)
    v_speak('Here is what I found for ' + search)
  if 'find location' in voice_data:
    location = record_audio('What is the location?')
    url = 'https://google.nl/maps/place/' + location + '/&'
    webbrowser.get().open(url)
    v_speak('Here is the location of ' + location)
  if 'exit' in voice_data:
    exit()

time.sleep(1)
print('What would you like to do?')
while 1:
  voice_data = record_audio()
  # print(voice_data)

  respond(voice_data)


from sqlalchemy import Column, Integer, String
from sqlalchemy import create_engine
engine = create_engine('postgresql+psycopg2://postgres:admin@localhost:5432/surendra', echo = True)
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()

class Customers(Base):
   __tablename__ = 'customers'
   id = Column(Integer, primary_key =  True)
   name = Column(String)

   address = Column(String)
   email = Column(String)

from sqlalchemy.orm import sessionmaker
Session = sessionmaker(bind = engine)
session = Session()
result = session.query(Customers).all()

for row in result:
   print ("Name: ",row.name, "Address:",row.address, "Email:",row.email)

####******************************
print()
x = session.query(Customers).get(2)
print ("Name: ", x.name, "Address:", x.address, "Email:", x.email)

###########################################
x.address = 'Banjara Hills Secunderabad'
session.commit()

######################################
x = session.query(Customers).first()
print ("Name: ", x.name, "Address:", x.address, "Email:", x.email)

#############################################

session.query(Customers).filter(Customers.id != 2).update({Customers.name:"Mr."+Customers.name}, synchronize_session = False)


"""
Test clari extract
"""
from datetime import datetime

import pytest
import requests

from extract.clari.src.clari import (
    _calc_fiscal_quarter,
    _get_previous_fiscal_quarter,
    get_fiscal_quarter,
    config_dict,
    make_request,
    check_valid_quarter,
)

def test_calc_fiscal_quarter1():
    """Check fiscal quarter1 logic"""
    # test Q1
    actual_q1 = "2021_Q1"

    datetime_q1_1 = datetime(2020, 2, 1)
    res_q1_1 = _calc_fiscal_quarter(datetime_q1_1)
    assert res_q1_1 == actual_q1

    datetime_q1_2 = datetime(2020, 3, 1)
    res_q1_2 = _calc_fiscal_quarter(datetime_q1_2)
    assert res_q1_2 == actual_q1

    datetime_q1_3 = datetime(2020, 4, 1)
    res_q1_3 = _calc_fiscal_quarter(datetime_q1_3)
    assert res_q1_3 == actual_q1

def test_calc_fiscal_quarter2():
    """Check fiscal quarter2 logic"""
    # test Q2
    actual_q2 = "2021_Q2"

    datetime_q2_1 = datetime(2020, 5, 1)
    res_q2_1 = _calc_fiscal_quarter(datetime_q2_1)
    assert res_q2_1 == actual_q2

    datetime_q2_2 = datetime(2020, 6, 1)
    res_q2_2 = _calc_fiscal_quarter(datetime_q2_2)
    assert res_q2_2 == actual_q2

    datetime_q2_3 = datetime(2020, 7, 1)
    res_q2_3 = _calc_fiscal_quarter(datetime_q2_3)
    assert res_q2_3 == actual_q2

def test_calc_fiscal_quarter3():
    """Check fiscal quarter3 logic"""
    # test Q3
    actual_q3 = "2021_Q3"

    datetime_q3_1 = datetime(2020, 8, 1)
    res_q3_1 = _calc_fiscal_quarter(datetime_q3_1)
    assert res_q3_1 == actual_q3

    datetime_q3_2 = datetime(2020, 9, 1)
    res_q3_2 = _calc_fiscal_quarter(datetime_q3_2)
    assert res_q3_2 == actual_q3

    datetime_q3_3 = datetime(2020, 10, 1)
    res_q3_3 = _calc_fiscal_quarter(datetime_q3_3)
    assert res_q3_3 == actual_q3

def test_calc_fiscal_quarter4():
    """Check fiscal quarter4 logic"""
    # test Q4
    actual_q4 = "2021_Q4"

    datetime_q4_1 = datetime(2020, 11, 1)
    res_q4_1 = _calc_fiscal_quarter(datetime_q4_1)
    assert res_q4_1 == actual_q4

    datetime_q4_2 = datetime(2020, 12, 1)
    res_q4_2 = _calc_fiscal_quarter(datetime_q4_2)
    assert res_q4_2 == actual_q4

    datetime_q4_3 = datetime(2021, 1, 1)
    res_q4_3 = _calc_fiscal_quarter(datetime_q4_3)
    assert res_q4_3 == actual_q4

def test_get_previous_fiscal_quarter():
    """Check previous fiscal quarter logic"""
    datetime_q1 = datetime(2020, 2, 1)
    res_q1 = _get_previous_fiscal_quarter(datetime_q1)
    actual_q1 = "2020_Q4"
    assert res_q1 == actual_q1

    datetime_q2 = datetime(2020, 5, 1)
    res_q2 = _get_previous_fiscal_quarter(datetime_q2)
    actual_q2 = "2021_Q1"
    assert res_q2 == actual_q2

    datetime_q3 = datetime(2020, 8, 1)
    res_q3 = _get_previous_fiscal_quarter(datetime_q3)
    actual_q3 = "2021_Q2"
    assert res_q3 == actual_q3

    datetime_q4 = datetime(2020, 11, 1)
    res_q4 = _get_previous_fiscal_quarter(datetime_q4)
    actual_q4 = "2021_Q3"
    assert res_q4 == actual_q4

def test_get_fiscal_quarter():
    """Get fiscal quarter based on env var"""
    config_dict["task_schedule"] = "daily"

    config_dict["execution_date"] = datetime(2020, 2, 1).strftime("%Y-%m-%d %H:%M:%S")
    res_q1 = get_fiscal_quarter()
    actual_q1 = "2021_Q1"
    assert res_q1 == actual_q1

    config_dict["execution_date"] = datetime(2020, 5, 1).strftime("%Y-%m-%d %H:%M:%S")
    res_q2 = get_fiscal_quarter()
    actual_q2 = "2021_Q2"
    assert res_q2 == actual_q2

    config_dict["execution_date"] = datetime(2020, 8, 1).strftime("%Y-%m-%d %H:%M:%S")
    res_q3 = get_fiscal_quarter()
    actual_q3 = "2021_Q3"
    assert res_q3 == actual_q3

    config_dict["execution_date"] = datetime(2020, 11, 1).strftime("%Y-%m-%d %H:%M:%S")
    res_q4 = get_fiscal_quarter()
    actual_q4 = "2021_Q4"
    assert res_q4 == actual_q4

def test_make_request():
    """Test requests"""
    # Test that google request passes
    request_type = "GET"
    url = "https://www.google.com"
    resp = make_request(request_type, url)
    assert resp.status_code == 200

    # Test that an invalid request type throws an error
    error_str = "Invalid request type"
    with pytest.raises(ValueError) as exc:
        request_type = "nonexistent_request_type"
        make_request(request_type, url)
    assert error_str in str(exc.value)

    # Test HTTP error
    request_type = "GET"
    with pytest.raises(requests.exceptions.HTTPError) as exc:
        url = "https://www.google.com/invalid_url"
        make_request(request_type, url)
    error_str = "404 Client Error"
    assert error_str in str(exc.value)

def test_check_valid_quarter():
    """Test valid quarter check"""
    results_dict = {
        "timePeriods": [
            {
                "timePeriodId": "2023_Q4",
                "type": "quarter",
                "label": "Quarter 4",
                "year": "2023",
                "startDate": "2022-11-01",
                "endDate": "2023-01-31",
                "crmId": "0264M000001KFm1QAG",
            }
        ]
    }
    original_fiscal_quarter = "2023_Q4"
    assert check_valid_quarter(original_fiscal_quarter, results_dict) is None
    with pytest.raises(ValueError) as exc:
        original_fiscal_quarter = "2022_Q4"
        error_str = "does not match"
        check_valid_quarter(original_fiscal_quarter, results_dict)
    assert error_str in str(exc.value)


import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import os

df = pd.read_csv(r"data\airbnb_bordeaux.csv")

# Nettoyage du dataset
# On prend le parti de considérer "strictes" les conditions d'annulation non renseignées
df = df.fillna({"conditions_annulation": "Strictes"})
df = df.replace(to_replace={'Flexibles': 'flexibles', 'Modérées': 'moderees', 'Strictes': 'strictes'})
# On effectue un one hot encoding sur la variable conditions_annulation
df = pd.get_dummies(df, columns=["conditions_annulation"])

# Par simplicité, on ne prend en compte que le fait que les animaux soient autorisés ou pas
# (car le type d'animal risque de créer du bruit de manière inutile pour prédire le prix)
# On remplace par des 0 quand pas d'animal autorisé, 1 quand autorisé
df["animal_sur_place"] = df["animal_sur_place"].notna().astype('int')

# On crée une catégorie "multiples" pour les types de lits non renseignés
# (car ils correspondent à plusieurs types de lits à la fois pour le même logement)
df = df.fillna({"type_lit": "multiples"})
df = df.replace(to_replace={'Vrai lit': 'vrai_lit', 'Canapé convertible': 'canape_convertible', 'Canapé': 'canape'})
df = pd.get_dummies(df, columns=["type_lit"])

# On regroupe les types de propriétés similaires
df = df.replace(to_replace={'Appartement': 'appartement', 'Maison': 'maison', 'Maison de ville': 'maison_de_ville',
                            'Bed & Breakfast': 'bed_and_breakfast', 'Appartement en résidence': 'appart_en_residence',
                            'Loft': 'loft', 'Inconnue': 'inconnu', 'Autre': 'autre',
                            'Bungalow': 'bungalow_cabane_dortoir_eco', 'Villa': 'maison',
                            'Cabane': 'bungalow_cabane_dortoir_eco', 'Maison écologique': 'bungalow_cabane_dortoir_eco',
                            'Dortoir': 'bungalow_cabane_dortoir_eco'})
df = pd.get_dummies(df, columns=["type_propriete"])

df = df.replace(to_replace={'Logement entier': 'logement_entier', 'Chambre privée': 'chambre_privee',
                            'Chambre partagée': 'chambre_partagee'})
df = pd.get_dummies(df, columns=["type_logement"])

# On retire les lignes dont le prix est nul
df = df.drop(df[df["prix_nuitee"] == 0].index)

# On modifie les lignes pour lesquelles le nombre de salles de bain est extravagant et ne correspond pas au vrai nombre
# indiqué dans l'annonce
df.loc[df['nombresdb'] >= 15] = 1

# On corrige les fautes de frappes
df = df.rename(columns={'rection_semaine': 'reduction_semaine'})

# On retire la colonne shampooing (qui vaut 0 partout) et les colonnes contenant des chaînes de caractères
# On retire aussi la colonne "prixnuitee" présente deux fois (on ne garde que "prix_nuitee")
df = df.drop(columns=['reglement_interieur', 'description', 'resume', 'titre',
                      'url', 'identifiant', 'shampooing', 'prixnuitee'])

# On veut stratifier équitablement selon les prix des nuitées pour diviser en dataset public et privé
df["y_binned"] = pd.cut(df["prix_nuitee"], bins=3, labels=np.arange(3), right=False)
df_public, df_private = train_test_split(df, shuffle=True, train_size=0.7,
                                         stratify=df["y_binned"], random_state=26)

# On divise chaque dataset en train et test de manière stratifiée sur le prix à la nuit
df_public_train, df_public_test = train_test_split(df_public, shuffle=True, train_size=0.8,
                                                   stratify=df_public["y_binned"], random_state=26)

df_private_train, df_private_test = train_test_split(df_private, shuffle=True, train_size=0.8,
                                                     stratify=df_private["y_binned"], random_state=26)

# On retire la colonne "y_binned" ajoutée pour stratifier
df_private_train = df_private_train.drop(columns=["y_binned"])
df_private_test = df_private_test.drop(columns=["y_binned"])
df_public_train = df_public_train.drop(columns=["y_binned"])
df_public_test = df_public_test.drop(columns=["y_binned"])

# On enregistre les données dans des fichiers
df_private_train.to_csv(os.path.join('data', 'private_train.csv'), index=False)
df_private_test.to_csv(os.path.join('data', 'private_test.csv'), index=False)
df_public_train.to_csv(os.path.join('data', 'train', 'train.csv'), index=False)
df_public_test.to_csv(os.path.join('data', 'test', 'test.csv'), index=False)


from services.payment_service import PaymentService
from unittest import TestCase
from unittest.mock import patch
import requests

class TestPayments(TestCase):
    @patch("services.payment_service")
    def test_payment_status_matches_success_when_payment_is_successful(self, mock_service) -> None:
        response = {
                "status_code": 200,
                "bank_transaction_id": "d5c472f9-fcbd-4f37-bd7e-2f5aac8dae6e",
                "payment_status": "SUCCESS",
                "response": "Payment processed successfully",
            }
        mock_service.process_payment.return_value = response

        self.request = {
            "data": {
                "processPaymentTransactionRequest": {
                    "requestId": "12345",
                    "merchantId": "JaiShreeRam",
                    "userId": "userId1",
                    "cardDetails": {
                        "cardNumber": "1234-4567-5555-5555",
                        "expiryDate": "2021-01",
                        "cvv": "345",
                    },
                    "amount": "100.00",
                    "currency": "USD",
                }
            }
        }

        base_url = f"http://127.0.0.1:5000/v1/payments"

        response = requests.post(base_url, json=self.request)
        assert response.status_code== 200

    def test_payment_amount_matches_original_amount_when_payment_is_successfuk(
        self,
    ) -> None:
        pass


#coding=utf-8
from pymongo import MongoClient

#建立MongoDB数据库连接
# client = MongoClient('10.99.6.9',8013)
client = MongoClient('10.99.6.8:8015,10.99.6.9:8013,10.99.6.11:8014')

#连接所需数据库,test为数据库名
# db=client.test
db=client.parsed_data_storage

#连接所用集合，也就是我们通常所说的表，test为表名
# collection=db.data
# collection=db.myTestCollection
collection=db.parsed_real_data_w04
count =0
# for i in collection.find():
#     count +=1
#     print i
# print count
# print db['a']
# for i in collection.find({'time': {'$gte':"2017-09-05 00:01:00", '$lt':"2017-09-08 00:01:00"}}):
for i in collection.find({'VIN':'LGJE13EA0GM417415','time':'2017-09-21 00:00:00'}):
# for i in collection.find({'VIN':'LGJE13EA2GM454207'}):

# for i in collection.find({'VIN':'LGJE13EA4HM616257','time':{'$lt':'2017-11-29 00:00:00','$gt':'2017-11-28 00:00:00'}}):

# for i in collection.find({'VIN':'LDPGAAAB9HB101503'}):
#     print i
# for i in collection.distinct("VIN",{"time": {'$gte':"2017-09-06 00:00:00",'$lt':"2017-09-08 00:00:00"}}):

    # if i=='LDPGABAC1HB101817':
    # if i =='LDP53A935GC10474':
    #    for f in  collection.find({'VIN':'LDPGABAC1HB101817'}):
    #        print f
    print (i)
    if count == 1:
        break
    count += 1

print (count)


from rply import ParserGenerator
from rply.errors import ParserGeneratorWarning
import warnings

import ast
from ast import *

class Parser:
    def __init__(self, module, builder, printf):
        self.variables = {}
        self.pg = ParserGenerator(
            ['Numb', 'Write', 'LParen', 'RParen','LBracket','RBracket','LogicEquality','LessThan','GreaterThan',
             'LogicalNegation', 'Func', 'While', 'Return','Comma',
             'SemiColon', 'Sum', 'Sub', 'Multi', 'Div', 'Mod', 'ID', 'ASSIGN', 'Int','Begin','End','Float','NumbFlo',
             'If'],
            precedence=[("left", ["Sum","Sub"]),("right", ["Multi", 'Div', 'Mod'])]
        )
        self.module = module
        self.builder = builder
        self.printf = printf
        self.idfstr=0

    def parse(self):
        @self.pg.production('program : Begin body  End')
        def program_expression(p):
            return p[1]

        @self.pg.production('body : stmts')
        def block(p):
            return p[0]

        @self.pg.production('stmts : stmts stmt')
        def stmts_b(p):
            if p[1] is None:
                return p[0]
            else:
                return p[0] + [p[1]]

        @self.pg.production('stmts : stmt')
        def stmts_stmt(p):
            if p[0] is None:
                return []
            else:
                return [p[0]]

        @self.pg.production('stmt : stmt_write SemiColon')
        @self.pg.production('stmt : stmt_assign SemiColon')
        @self.pg.production('stmt : stmt_if SemiColon')
        @self.pg.production('stmt : stmt_while SemiColon')
        @self.pg.production('stmt : stmt_pereassign SemiColon')
        @self.pg.production('stmt : stmt_func SemiColon')
        @self.pg.production('stmt : stmt_return SemiColon')
        def stmt(p):
            return p[0]

        @self.pg.production('stmt_func : Func ID LParen ID RParen LBracket stmts RBracket')
        def funcs(p):
            return FuncStatement(self.builder, self.module, p[1].value, p[3] , p[6])

        @self.pg.production('stmt_if : If LParen expression RParen LBracket stmts RBracket')
        def ifs(p):
            return IfStatement(self.builder, self.module, p[2], p[5], [])

        @self.pg.production('stmt_while : While LParen expression RParen LBracket stmts RBracket')
        def whiles(p):
            return WhileStatement(self.builder, self.module, p[2], p[5])

        @self.pg.production('stmt_write : Write LParen expression RParen')
        def prints(p):
            self.idfstr +=1
            return Write(self.builder, self.module, self.printf, p[2],self.idfstr)

        @self.pg.production('stmt_assign : Int ID ASSIGN expression')
        @self.pg.production('stmt_assign : Float ID ASSIGN expression')
        def assign(p):

            if isinstance(p[3], ast.Numb):
                return ASSIGN(self.builder, self.module, p[1].value, p[3].value)
            else:
                return ASSIGN(self.builder, self.module, p[1].value, p[3])

        @self.pg.production('stmt_pereassign : ID ASSIGN expression')
        def pereassign(p):

           if isinstance(p[2], ast.Numb):
               return PereASSIGN(self.builder, self.module, p[0].value, p[2].value)
           else:
               return PereASSIGN(self.builder, self.module, p[0].value, p[2])

        @self.pg.production('stmt_return : Return expression')
        def ruturn_func(p):
            return ReturnStatement(self.builder, p[1])

        @self.pg.production('expression : expression Sum expression')
        @self.pg.production('expression : expression Sub expression')
        @self.pg.production('expression : expression Multi expression')
        @self.pg.production('expression : expression Div expression')
        @self.pg.production('expression : expression Mod expression')
        @self.pg.production('expression : expression LogicEquality expression')
        @self.pg.production('expression : expression LessThan expression')
        @self.pg.production('expression : expression GreaterThan expression')
        @self.pg.production('expression : expression LogicalNegation expression')
        def expression(p):
            left = p[0]
            right = p[2]
            operator = p[1]
            if operator.gettokentype() == 'Sum':
                return Sum(self.builder, left, right)
            elif operator.gettokentype() == 'Sub':
                return Sub(self.builder, left, right)
            elif operator.gettokentype() == 'Multi':
                return Mult(self.builder, left, right)
            elif operator.gettokentype() == 'Div':
                return Div(self.builder, left, right)
            elif operator.gettokentype() == 'Mod':
                return Mod(self.builder, left, right)
            elif operator.gettokentype() == 'LogicEquality':
                return Equality(self.builder, left, right)
            elif operator.gettokentype() == 'LessThan':
                return LessThan(self.builder, left, right)
            elif operator.gettokentype() == 'GreaterThan':
                return GreaterThan(self.builder, left, right)
            elif operator.gettokentype() == 'LogicalNegation':
                return LogicalNegation(self.builder, left, right)

        @self.pg.production('expression : LParen expression RParen')
        def paren_exp(p):
            return p[1]

        @self.pg.production('expression : Numb')
        @self.pg.production('expression : NumbFlo')
        def number(p):
            return Numb(self.builder,p[0].value)

        @self.pg.production('expression : Float LParen ID RParen')
        @self.pg.production('expression : Int LParen ID RParen')
        def typecast(p):
            return Typecast(self.builder, self.module, p[0].value, p[2].value)

        @self.pg.production('expression : ID')
        def id(p):
            return Id(self.builder, self.module,p[0].value)

        @self.pg.production('expression : ID LParen expression RParen')
        def idfunc(p):
            return CallFunc(self.builder, p[0].value, p[2])

        @self.pg.error
        def error_handle(token):
            warnings.filterwarnings("ignore", category=ParserGeneratorWarning)
            raise ValueError("Syntax error at token {0} at line {1}".format(token.gettokentype(),token.source_pos.lineno-1))

    def get_parser(self):
        warnings.filterwarnings('ignore')

        return self.pg.build()


import requests
import os
from utility.handleErr import handlerErr

def upload_image_imgur(files_list: list[bytes], title: str, details: str):
    url = "https://api.imgur.com/3/image"
    headers = {"Authorization": f"Bearer %s" % os.getenv("AUTH_TOKEN_IMGUR")}
    info_files = list()

    for index, file in enumerate(files_list):
        payload = {
            "name": "Image Product",
            "title": title,
            "description": details,
            "image": file,
        }
        response = requests.post(
            url=url,
            headers=headers,
            data=payload,
        ).json()["data"]

        if "error" in response:
            handlerErr("imgur -> %s" % response["error"]["message"])

        info_files.append(
            {
                "upload_id": response["id"],
                "ref_color": index,  # "file.filename",
                "url_image": response["link"],
                "key_img": response["deletehash"],
            }
        )

    return info_files


import MeCab
import collections

mecab = MeCab.Tagger()

with open('./neko.txt') as in_file, open('./neko.txt.mecab', mode='w') as out_file:
    out_file.write(mecab.parse(in_file.read()))

with open("neko.txt.mecab") as f:
    surface = []
    for i, line in enumerate(f):
        split_line = line.rstrip("\r\n").split("\t")
        if len(split_line) > 1:
            surface.append(split_line[0])
surfaces = []
surfaces = collections.Counter(surface).most_common(10)

surfaces_x = [x[1] for x in surfaces]
surfaces_y = [x[0] for x in surfaces]

import matplotlib.pyplot as plt
import numpy as np

plt.rcParams['font.family'] = 'AppleMyungjo'
a = np.array([1,2,3,4,5,6,7,8,9,10])
plt.bar(a,surfaces_x,color='#7D6BA0',align='center',edgecolor='white')
plt.title('頻度上位10語')

plt.xticks(a,surfaces_y)

plt.show()


# -*- coding: utf-8 -*-
"""A simple class to connect to a MSSQL database engine and execute queries."""
from os.path import exists
from pandas import read_sql_query
from sqlalchemy import create_engine, MetaData

class SQLConnection:
    """A class for connecting to a database and preforming SQL commands."""

    def __init__(self, server, database):
        """Initialize a sql connection and connect to it.

        Parameters
        ----------
        server: str
            A server.
        database: str
            A database.
        """
        self.server = server
        self.db = database
        self._connect_to_sqlserver()

    def _connect_to_sqlserver(self):
        """Build a Microsoft SQL Server connection.

        Creates
        ----------
        self.engine : sqlalchemy.engine.base.Engine
            The database connection engine.
        self.meta : sqlalchemy.sql.schema.MetaData
            The current meta data for the connection engine.
        """
        driver = 'SQL+Server+Native+Client+11.0'
        self.engine = create_engine('mssql://{}/{}?driver={}'.format(
            self.server, self.db, driver
            ), echo=False, legacy_schema_aliasing=False)
        self.meta = MetaData(self.engine)

    def execute_sql(self, query_input, return_results=False):
        """Execute sql.

        Parameters
        ----------
        query_input: str
            Sql_file path or query.
        results: boolean
            Whether to return results or not (default False).

        Returns
        ----------
        df: pandas.core.frame.DataFrame (default: do not return results)
            The results of the query.
        """
        # If query_input is an existing file, pull query from file
        if exists(query_input):
            # if file is a valid path to a file
            query_file = query_input

            # Get query
            with open(query_file) as f:
                query = f.read()

            # Remove unnecessary characters in query
            query = query.replace('\n', ' ').replace('\t', ' ')

        # If query_input is not file, query_input should be a string query
        else:
            query = query_input

        # Ensure there are no -- style comments in the query
        assert '--' not in query, ('unable to parse queries containing -- '
                                   'style comments, use /* */ instead')

        # Open database connection
        connection = self.engine.connect()
        transaction = connection.begin()

        try:
            # Execute query
            connection.execute(query)
            transaction.commit()
        except:
            transaction.rollback()
            raise
        finally:
            # Close database connection
            connection.close()

        if return_results:
            results = read_sql_query(query, con=self.engine)
            return results


import time
import json

sensortype = "tmep"
sensorNum = "0000"
sensorName = "ahao"
current_value = 200
sensorUpdata = 300
sensorDowndata = 50
flag = True

struct_data = {
    "type":sensortype,
    "num": sensorNum,  #设备ID；
    "name":sensorName, #设备名称；
    # "current_data":current_value,
    "updata":sensorUpdata,
    "downdata":sensorDowndata,
    "finish_updata_flag":flag
}

def rawinfo_to_structdata(c_to_s_data,structed_data):
    reversed_kv = {};  #反转键值对在字典中的顺序，前面的键值对放到后面，一会儿会从后面开始弹出；
    while c_to_s_data:
        k,v = c_to_s_data.popitem()
        t_item = {k:v}
        reversed_kv.update(t_item)
    while reversed_kv:
             time.sleep(0.01)
        #if(structed_data["finish_updata_flag"] == False):
             head,item= reversed_kv.popitem()
             # print(head)
             # print(item)
             head_list = head.split("/")
             struct_data["type"] = head_list[0]
             struct_data["num"] = head_list[1]
             info_dir = json.loads(item)
             struct_data["name"] = info_dir["name"]
             # struct_data["current_data"] = info_dir["attr"]["attr1"]
             struct_data["updata"] = info_dir["attr"]["attr2"]
             struct_data["downdata"] = info_dir["attr"]["attr3"]
             struct_data["finish_updata_flag"] = True
             print(struct_data)