import tensorflow as tf
import numpy as np
import facenet
import math
import pickle
from scipy import  misc

import sklearn.metrics as ms
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.model_selection import learning_curve
from sklearn.model_selection import ShuffleSplit
import matplotlib.pyplot as plt  # 可视化绘图
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)

def prewhiten(x):
    mean = np.mean(x)
    std = np.std(x)
    std_adj = np.maximum(std, 1.0 / np.sqrt(x.size))
    y = np.multiply(np.subtract(x, mean), 1 / std_adj)
    return y

def crop(image, random_crop, image_size):
    if image.shape[1] > image_size:
        sz1 = int(image.shape[1] // 2)
        sz2 = int(image_size // 2)
        if random_crop:
            diff = sz1 - sz2
            (h, v) = (np.random.randint(-diff, diff + 1), np.random.randint(-diff, diff + 1))
        else:
            (h, v) = (0, 0)
        image = image[(sz1 - sz2 + v):(sz1 + sz2 + v), (sz1 - sz2 + h):(sz1 + sz2 + h), :]
    return image

def flip(image, random_flip):
    if random_flip and np.random.choice([True, False]):
        image = np.fliplr(image)
    return image

def to_rgb(img):
    w, h = img.shape
    ret = np.empty((w, h, 3), dtype=np.uint8)
    ret[:, :, 0] = ret[:, :, 1] = ret[:, :, 2] = img
    return ret

def load_data(image_paths, do_random_crop, do_random_flip, image_size, do_prewhiten=True):
    nrof_samples = len(image_paths)
    #print(nrof_samples)# 90
    images = np.zeros((nrof_samples, image_size, image_size, 3))
    #print(images.shape) #90 160 160 3
    for i in range(nrof_samples):
        img = misc.imread(image_paths[i])
        #print(img.ndim)3
        #print(img.shape) 160 160 3
        if img.ndim == 2:
            img = to_rgb(img)
        if do_prewhiten:
            img = prewhiten(img)
        img = crop(img, do_random_crop, image_size)
        img = flip(img, do_random_flip)
        images[i,:,:,:] = img
        #print(images.shape)# 90 160 160 3
        #print(images[i,:,:,:].shape) 160 160 3
    return images

def split_dataset(dataset, min_nrof_images_per_class, nrof_train_images_per_class):
    train_set = []
    test_set = []
    for cls in dataset:
        paths = cls.image_paths
        #print("进入函数")
        #print(len(paths))
        # Remove classes with less than min_nrof_images_per_class
        if len(paths)>=min_nrof_images_per_class:
            np.random.shuffle(paths)
            #print(cls.name)
            #print(nrof_train_images_per_class)
            nrof_train_images_per_class = int(0.8*len(paths))
            train_set.append(facenet.ImageClass(cls.name, paths[:nrof_train_images_per_class]))
            test_set.append(facenet.ImageClass(cls.name, paths[nrof_train_images_per_class:]))
    return train_set, test_set

def get_image_paths_and_labels(dataset):
    image_paths_flat = []
    labels_flat = []
    #print(dataset)
    #print(len(dataset))
    for i in range(len(dataset)):
        image_paths_flat += dataset[i].image_paths
        #print(dataset[i].image_paths)
        #print(len(dataset[i].image_paths))
        #print(image_paths_flat)
        #print([i])
        labels_flat += [i] * len(dataset[i].image_paths)#复制10个[0],[1],[2],[3]
        #print(labels_flat)
    #print(len(image_paths_flat))
    #print(len(labels_flat))
    return image_paths_flat, labels_flat

# dataset_tmp = facenet.get_dataset("C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\data\\self_data_1601")
# #print(dataset_tmp)
# train_set, test_set = split_dataset(dataset_tmp, 20,10)
# dataset = train_set
# paths, labels = get_image_paths_and_labels(dataset)
# dataset = test_set
# paths, labels = get_image_paths_and_labels(dataset)
def main1(args):
    with tf.Graph().as_default():
        with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
            np.random.seed(seed=666)
            if True:
                dataset_tmp = facenet.get_dataset(args.data_dir)
                train_set, test_set = split_dataset(dataset_tmp, 20,30)
                # if (args.mode == 'TRAIN'):
                #     dataset = train_set
                # elif (args.mode == 'CLASSIFY'):
                #     dataset = test_set
            else:
                dataset = facenet.get_dataset("C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\data\\self_data_160")
            dataset = train_set
            paths, labels = get_image_paths_and_labels(dataset)
            print('Number of classes: %d' % len(dataset))
            print('Number of images: %d' % len(paths))
            # Load the model
            print('Loading feature extraction model')
            facenet.load_model("C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\models\\20180402-114759")

            # Get input and output tensors
            # 输入的图片
            images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
            # 卷积网络最后输出的特征
            embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
            # 现在是不是训练阶段
            phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
            embedding_size = embeddings.get_shape()[1]

            # Run forward pass to calculate embeddings
            print('Calculating features for images')
            nrof_images = len(paths)  # 图片总数目
            # print(nrof_images)
            nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images / 90))
            # print(nrof_batches_per_epoch)
            emb_array = np.zeros((nrof_images, embedding_size))
            #print(emb_array.shape)

            for i in range(nrof_batches_per_epoch):
                start_index = i * 90
                end_index = min((i + 1) * 90, nrof_images)
                paths_batch = paths[start_index:end_index]
                images = facenet.load_data(paths_batch, False, False, 160)
                # 计算特征
                feed_dict = {images_placeholder: images, phase_train_placeholder: False}
                emb_array[start_index:end_index, :] = sess.run(embeddings, feed_dict=feed_dict)

            classifier_filename_exp = os.path.expanduser(args.classifier_filename)
            #model = SVC(kernel='poly',C=1,gamma=1,degree=3,probability=True)
            #model = SVC(C=100,kernel='linear',probability=True)
            model =SVC(kernel='rbf',C=10,gamma=1,probability=True)
            #model = SVC(kernel='poly', C=10, gamma=1, degree=10, probability=True)
            # model = SVC(kernel='poly',C=1,gamma=1,degree=5,probability=True)
            # model = SVC(kernel='poly',C=1,gamma=1,degree=6,probability=True)
            # model = SVC(kernel='poly',C=1,gamma=1,degree=7,probability=True)

            model.fit(emb_array,labels)
            # param_grid = [#{'kernel': ['linear'], 'C': [0.001,0.01,0.1,1]},
            #               #{'kernel': ['rbf'], 'C': [0.001,0.01,0.1,1], 'gamma': [1, 0.1, 0.01, 0.001]},
            #               {
            #               'degree':[2,3,4,5,6,7,8,9,10]}
            #               ]
            # grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=10,
            #                     scoring='accuracy',n_jobs=4)  # 针对每个参数对进行了10次交叉验证。scoring='accuracy'使用准确率为结果的度量指标。可以添加多个度量指标
            # grid.fit(emb_array, labels)
            print(emb_array.shape)
            #print(type(emb_array))
            #print(emb_array.shape[0])
            # print(emb_array[:,0].shape)
            #print(len(labels))
            #print(type(labels))
            print("_________________")
            # #print('网格搜索-度量记录：', grid.cv_results_)  # 包含每次训练的相关信息
            # print('网格搜索-最佳度量值:', grid.best_score_)  # 获取最佳度量值
            # print('网格搜索-最佳参数：', grid.best_params_)  # 获取最佳度量值时的代定参数的值。是一个字典
            # print('网格搜索-最佳模型：', grid.best_estimator_)  # 获取最佳度量时的分类器模型

            dataset= test_set
            paths1, labels1 = get_image_paths_and_labels(dataset)
            #
            print('Number of classes: %d' % len(dataset))
            print('Number of images: %d' % len(paths1))
            # print(paths)
            # Load the model
            print('Loading feature extraction model')
            facenet.load_model("C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\models\\20180402-114759")

            # Get input and output tensors
            # 输入的图片
            images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
            # 卷积网络最后输出的特征
            embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
            # 现在是不是训练阶段
            phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
            embedding_size = embeddings.get_shape()[1]

            # Run forward pass to calculate embeddings
            print('Calculating features for images')
            nrof_images = len(paths1)  # 图片总数目
            # print(nrof_images)
            nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images / 1))
            # print(nrof_batches_per_epoch)
            emb_array1 = np.zeros((nrof_images, embedding_size))
            print(emb_array1.shape)

            for i in range(nrof_batches_per_epoch):
                start_index = i * 1
                end_index = min((i + 1) * 1, nrof_images)
                paths_batch = paths1[start_index:end_index]
                images = facenet.load_data(paths_batch, False, False, 160)
                # 计算特征
                feed_dict = {images_placeholder: images, phase_train_placeholder: False}
                emb_array1[start_index:end_index, :] = sess.run(embeddings, feed_dict=feed_dict)

            # 使用获取的最佳参数生成模型，预测数据
            #knn = SVC(n_neighbors=grid.best_params_['n_neighbors'],
            #                           weights=grid.best_params_['weights'])  # 取出最佳参数进行建模
            # model = SVC(kernel='poly',C=1,gamma=1,degree=grid.best_params_['degree'],
            #             probability=True)
            #,gamma=grid.best_params_['gamma'],
            # model = SVC(kernel='linear',probability=True)
            # model.fit(emb_array,labels)
            #print(model.predict(emb_array1))
            predictions = model.predict_proba(emb_array1)
            # #print(predictions)
            # #print(predictions.shape)
            best_class_indices = np.argmax(predictions, axis=1)
            #best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
            # #print(best_class_indices)
            # #print(len(best_class_indices))
            #
            # #print(best_class_probabilities)
            # #print(len(best_class_probabilities))
            accuracy = np.mean(np.equal(best_class_indices, labels1))
            print('Accuracy: %.3f' % accuracy)

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 10)):
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Number of training samples")
    plt.ylabel("Accuracy")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv,
        n_jobs=n_jobs, train_sizes=train_sizes,
        scoring='accuracy', verbose=0)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training accuracy")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="validation accuracy")

    plt.legend(loc="best")
    return plt

# def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
#                         n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
#     plt.figure()
#     plt.title(title)
#     if ylim is not None:
#         plt.ylim(*ylim)
#     plt.xlabel("Training examples")
#     plt.ylabel("Score")
#     train_sizes, train_scores, test_scores = validation_curve(
#         estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
#     train_scores_mean = np.mean(train_scores, axis=1)
#     train_scores_std = np.std(train_scores, axis=1)
#     test_scores_mean = np.mean(test_scores, axis=1)
#     test_scores_std = np.std(test_scores, axis=1)
#     plt.grid()
#
#     plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
#                      train_scores_mean + train_scores_std, alpha=0.1,
#                      color="r")
#     plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
#                      test_scores_mean + test_scores_std, alpha=0.1, color="g")
#     plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
#              label="Training score")
#     plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
#              label="Cross-validation score")
#
#     plt.legend(loc="best")
#     return plt

def main():
    with tf.Graph().as_default():
        with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
            np.random.seed(seed=666)

            dataset = facenet.get_dataset("C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\data\\self_data_160_not_aligned")

            paths, labels = get_image_paths_and_labels(dataset)
            #X_train, X_test, y_train, y_test = train_test_split(paths, labels, test_size=0.4, random_state=0)
            # print(X_train)
            # print(X_test)
            # print(y_train)
            # print(y_test)
            #
            # print(len(X_train))
            # print(len(X_test))
            # print(len(y_train))
            # print(len(y_test))

            print('Number of classes: %d' % len(dataset))
            print('Number of images: %d' % len(paths))
            # Load the model
            print('Loading feature extraction model')
            facenet.load_model("C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\models\\20180402-114759")

            # Get input and output tensors
            # 输入的图片
            images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
            # 卷积网络最后输出的特征
            embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
            # 现在是不是训练阶段
            phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
            embedding_size = embeddings.get_shape()[1]
            #print(embeddings) #Tensor("embeddings:0", shape=(?, 512), dtype=float32)
            #print(embedding_size)#512
            # Run forward pass to calculate embeddings
            print('Calculating features for images')
            nrof_images = len(paths)  # 图片总数目
            #print(nrof_images)
            nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images / 90))
            #print(nrof_batches_per_epoch)
            emb_array = np.zeros((nrof_images, embedding_size))
            #print(emb_array.shape)
            import os
            for i in range(nrof_batches_per_epoch):
                start_index = i * 90
                end_index = min((i + 1) * 90, nrof_images)
                paths_batch = paths[start_index:end_index]
                images = load_data(paths_batch, False, False, 160)
                # 计算特征
                feed_dict = {images_placeholder: images, phase_train_placeholder: False}
                emb_array[start_index:end_index, :] = sess.run(embeddings, feed_dict=feed_dict)
            classifier_filename_exp = os.path.expanduser("C:/Users/rjx/PycharmProjects/untitled1/facenet-master/models/sd_82a.pkl")
            from sklearn.model_selection import learning_curve
            from sklearn.model_selection import ShuffleSplit
            from sklearn.model_selection import validation_curve

    ################1#############
            model = SVC(C=1,kernel='linear', probability=True)
            model.fit(emb_array, labels)
            ylim = (0.75, 1.05)
            title = r"Learning Curves(SVM) for self data(not aligned)"
            #title = r"Learning Curves(SVM) for not aligned data"
            cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
            plt.figure()
            plt.title(title)
            plt.ylim(*ylim)
            plt.xlabel("Number of training samples")
            plt.ylabel("Accuracy")
            train_sizes, train_scores, test_scores = learning_curve(
                model, emb_array, labels, cv=cv,
                n_jobs=4, train_sizes=np.linspace(.1, 1.0, 10),
                scoring='accuracy', verbose=0)
            train_scores_mean = np.mean(train_scores, axis=1)
            train_scores_std = np.std(train_scores, axis=1)
            test_scores_mean = np.mean(test_scores, axis=1)
            test_scores_std = np.std(test_scores, axis=1)
            plt.grid()

            plt.plot(train_sizes, test_scores_mean, 'o-', color="y",
                     label="SVM Linear(C=1) validation accuracy")

            model = SVC(C=10, kernel='linear', probability=True)
            model.fit(emb_array, labels)

            train_sizes, train_scores, test_scores = learning_curve(
                model, emb_array, labels, cv=cv,
                n_jobs=4, train_sizes=np.linspace(.1, 1.0, 10),
                scoring='accuracy', verbose=0)
            train_scores_mean = np.mean(train_scores, axis=1)
            train_scores_std = np.std(train_scores, axis=1)
            test_scores_mean = np.mean(test_scores, axis=1)
            test_scores_std = np.std(test_scores, axis=1)
            plt.plot(train_sizes, test_scores_mean, 'o-', color="r",
                     label="SVM Linear(C=10) validation accuracy")

            model = SVC(C=10,degree=5, gamma=1,kernel='poly', probability=True)
            model.fit(emb_array, labels)

            train_sizes, train_scores, test_scores = learning_curve(
                model, emb_array, labels, cv=cv,
                n_jobs=4, train_sizes=np.linspace(.1, 1.0, 10),
                scoring='accuracy', verbose=0)
            train_scores_mean = np.mean(train_scores, axis=1)
            train_scores_std = np.std(train_scores, axis=1)
            test_scores_mean = np.mean(test_scores, axis=1)
            test_scores_std = np.std(test_scores, axis=1)
            plt.plot(train_sizes, test_scores_mean, 'o-', color="b",
                     label="SVM Poly(C=10,gamma=1,degree=5) validation accuracy")

            model = SVC(C=10,gamma=2.4, kernel='rbf', probability=True)
            model.fit(emb_array, labels)

            train_sizes, train_scores, test_scores = learning_curve(
                model, emb_array, labels, cv=cv,
                n_jobs=4, train_sizes=np.linspace(.1, 1.0, 10),
                scoring='accuracy', verbose=0)
            train_scores_mean = np.mean(train_scores, axis=1)
            train_scores_std = np.std(train_scores, axis=1)
            test_scores_mean = np.mean(test_scores, axis=1)
            test_scores_std = np.std(test_scores, axis=1)
            # title = r"Learning Curves(SVM) for not aligned data"
            plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
                     label="SVM RBF(C=10,gamma=2.4) validation accuracy")

            # model = SVC(C=10,degree=6, gamma=1, kernel='poly', probability=True)
            # model.fit(emb_array, labels)
            #
            # train_sizes, train_scores, test_scores = learning_curve(
            #     model, emb_array, labels, cv=cv,
            #     n_jobs=4, train_sizes=np.linspace(.1, 1.0, 10),
            #     scoring='accuracy', verbose=0)
            # train_scores_mean = np.mean(train_scores, axis=1)
            # train_scores_std = np.std(train_scores, axis=1)
            # test_scores_mean = np.mean(test_scores, axis=1)
            # test_scores_std = np.std(test_scores, axis=1)
            # plt.plot(train_sizes, test_scores_mean, 'o-', color="y",
            #          label="SVM Poly(C=10,degree=6,gamma=1) validation accuracy")

            # model = SVC(C=10, gamma=1,degree=10, kernel='poly', probability=True)
            # model.fit(emb_array, labels)
            #
            # train_sizes, train_scores, test_scores = learning_curve(
            #     model, emb_array, labels, cv=cv,
            #     n_jobs=4, train_sizes=np.linspace(.1, 1.0, 10),
            #     scoring='accuracy', verbose=0)
            # train_scores_mean = np.mean(train_scores, axis=1)
            # train_scores_std = np.std(train_scores, axis=1)
            # test_scores_mean = np.mean(test_scores, axis=1)
            # test_scores_std = np.std(test_scores, axis=1)
            # # title = r"Learning Curves(SVM) for not aligned data"
            # plt.plot(train_sizes, test_scores_mean, 'o-', color="y",
            #          label="SVM Poly(C=10,degree=10,gamma=1) validation accuracy")

            plt.legend(loc="best")
            plt.show()

import argparse
def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument('--mode', type=str, choices=['TRAIN', 'CLASSIFY'],
                        help='Indicates if a new classifier should be trained or a classification ' +
                             'model should be used for classification', default="TRAIN")
    parser.add_argument('--data_dir', type=str,
                        help='Path to the data directory containing aligned LFW face patches.', default=
                        "C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\data\\self_data_160_aligned")
    parser.add_argument('--model', type=str,
                        help='Could be either a directory containing the meta_file and ckpt_file or a model protobuf (.pb) file',
                        default="C:\\Users\\rjx\\PycharmProjects\\untitled1\\facenet-master\\models\\20180402-114759")
    parser.add_argument('--classifier_filename',
                        help='Classifier model file name as a pickle (.pkl) file. ' +
                             'For training this is the output and for classification this is an input.',
                        default="C:/Users/rjx/PycharmProjects/untitled1/facenet-master/models/sdd_p_82na.pkl")
    parser.add_argument('--use_split_dataset',
                        help='Indicates that the dataset specified by data_dir should be split into a training and test set. ' +
                             'Otherwise a separate test set can be specified using the test_data_dir option.',
                        action='store_true',
                        default=True)
    parser.add_argument('--test_data_dir', type=str,
                        help='Path to the test data directory containing aligned images used for testing.')
    parser.add_argument('--batch_size', type=int,
                        help='Number of images to process in a batch.', default=90)
    parser.add_argument('--image_size', type=int,
                        help='Image size (height, width) in pixels.', default=160)
    parser.add_argument('--seed', type=int,
                        help='Random seed.', default=666)
    parser.add_argument('--min_nrof_images_per_class', type=int,
                        help='Only include classes with at least this number of images in the dataset', default=20)
    # parser.add_argument('--nrof_train_images_per_class', type=int,
    #     help='Use this number of images from each class for training and the rest for testing', default=50)

    return parser.parse_args(argv)

if __name__ == '__main__':
    main()


from utils import timeit

@timeit
def get_data():
    data = []
    with open('input.txt') as input_file:
        for line in input_file:
            value = line.strip()
            data.append(value)
    return data

@timeit
def part_1(data):
    key_pad = [[i*3+j+1 for j in range(3)] for i in range(3)]
    x, y = 1, 1
    code = 0
    for line in data:
        for move in line:
            if move == 'U':
                if y > 0:
                    y -= 1
            elif move == 'D':
                if y < 2:
                    y += 1
            elif move == 'L':
                if x > 0:
                    x -= 1
            elif move == 'R':
                if x < 2:
                    x += 1
            else:
                raise ValueError('Unknown direction.')

        code *= 10
        code += key_pad[y][x]

    return code

@timeit
def part_2(data):

    key_pad = ( "#######",
                "###1###",
                "##234##",
                "#56789#",
                "##ABC##",
                "###D###",
                "#######")

    row, col = 3, 1
    code = []
    for line in data:
        for move in line:
            if move == 'U':
                if key_pad[row-1][col] != '#':
                    row -= 1
            elif move == 'D':
                if key_pad[row+1][col] != '#':
                    row += 1
            elif move == 'L':
                if key_pad[row][col-1] != '#':
                    col -= 1
            elif move == 'R':
                if key_pad[row][col+1] != '#':
                    col += 1

        code.append(key_pad[row][col])

    return ''.join(code)

def main():
    data = get_data()
    part_1(data)
    part_2(data)

if __name__ == "__main__":
    main()


# this is a really slow network scanner.
# the idea is that it will slowly poke around and check who's up and whos not
# over the course of a while, ideally without being discoverd or tripping over
# network detection systems. who knows if it works

import socket as s
import netifaces
import arprequest
import argparse
import netaddr
import time
import sys
import os
import common
from random import randint, random
from signal import signal, SIGINT

# set up arguments
parser = argparse.ArgumentParser(description="Slow network scanner")
parser.add_argument('-i', "--interface", metavar="<INTERFACE>", type=str, help="Interface to use when scanning", required=True)
parser.add_argument('-a', "--arp", dest="arpscan", action="store_true", help="Utilize ARP scanning instead of TCP connections")
parser.set_defaults(arpscan=False)

# some random ports to try to connect to while working
random_ports = [80, 443, 8080, 22]

# log file object
log_f = None

# global list of current ips
ips = []
alive_ips = []

# logs some stuff to stdout and log.txt
def log(string, e="\n"):
    global log_f
    if log_f == None:
        log_f = open('log.txt', 'w')

    log_f.write(string)
    print(string, end=e)

# CTRL-C handler
def sig_handler(signal_recvd, frame):
    global alive_ips
    global ips

    print("[INFO] Caught CTRL-C. Halting...")

    f = open(".resume", "w")

    for ip in ips:
        f.write(ip)
        f.write("\n")
    f.close()

    complete()

    sys.exit(1)

# cleans the resume file
def clear_resume():
    os.remove(".resume")

# done with scanning (either quit or completion)
def complete():
    global alive_ips

    # print info
    log("[INFO] COMPLETE")
    log("_" * 40)

    print("[INFO] Alive addresses:")

    for ip in alive_ips:
        print("\t{}".format(ip))

# generates a list of all valid IP addresses on the network
def generate_targets(interface):
    targets = []
    netstr = ""

    # generate the network string
    addr = netifaces.ifaddresses(interface)[2][0]['addr']
    netmask = netifaces.ifaddresses(interface)[2][0]['netmask']

    mask = [int(x) for x in netmask.split(".")]
    cidr = sum((bin(x).count('1') for x in mask))

    netstr = "{}/{}".format(addr, cidr)

    # add all of the valid hosts on the network to the target list
    net = netaddr.IPNetwork(netstr)

    for i in net.iter_hosts():
        targets.append(str(i))

    return targets

# main function
def run(interface, scantype=0):
    global ips
    global alive_ips

    # check if stuff has already been run
    if os.path.exists(".resume"):
        cont = input("[INFO] Detected previously incomplete run\n\tContinue? (y/n) > ").lower()
        if cont == 'n':
            # generate a new set
            ips = generate_targets(interface)
        else:
            print("[INFO] Resuming scan...")
            # read in the old data
            f = open(".resume", "r")
            dat = f.read()
            ips = dat.replace("\00", "").splitlines()[0:-1]
            f.close()
            print("[INFO] Remaining addresses: {}".format(len(ips)))

    else:
        ips = generate_targets(interface)

    # set up the terminating signal handler
    signal(SIGINT, sig_handler)
    num_ips = len(ips)
    total_done = 0

    # normal TCP scan type
    if scantype == 0:
        sock = s.socket(s.AF_INET, s.SOCK_STREAM)
        #sock.setsockopt()

        # loop over the addresses
        for i in range(num_ips):
            # randomly choose an ip from the list
            address = ips[randint(0, len(ips)-1)]
            # try to connect on random known port
            try:
                port = random_ports[randint(0, len(random_ports)-1)]
                log("[INFO] Trying {} on {} ({:>.1f}%)".format(address, port, total_done/len(ips)), e="\r")

                sock.connect((address, port))

                # what are the chances we get this far without exceptions?
                # low... but not impossible
                sock.close()
                alive_ips.append(address)
                log("[INFO] Address '{}' is up AND port '{}' is open".format(address, port))

            # host is up and port is closed
            except ConnectionRefusedError:
                alive_ips.append(address)
                log("[INFO] Address '{}' is up                ".format(address))

            # host is down
            except OSError:
                pass

            # user interrupt
            except KeyboardInterrupt:
                print("[CANCELED]")
                break

            # we have gotten the info we wanted on the address, so remove it
            ips.remove(address)

            total_done += 1
            time.sleep(random())

    # ARP IP scan
    elif scantype == 1:
        for i in range(num_ips):

            # choose an address
            address = ips[randint(0, len(ips)-1)]
            log("[INFO] Trying {} ({:>.1f}%)".format(address, total_done/len(ips)), e="\r")

            # create request
            alive = arprequest.ArpRequest(address, interface)
            if alive.request():
                alive_ips.append(address)
                log("[INFO] Address '{}' is up                ".format(address))

            # move on
            ips.remove(address)
            total_done += 1
            time.sleep(random())

    clear_resume()
    complete()

def main():
    # check privilages
    if not common.has_root():
        print("[ERROR] Insufficient privilages! Please run as root")
        sys.exit(1)

    args = parser.parse_args()
    scantype = 0
    interface = args.interface

    # see if we are scanning using ARP
    if args.arpscan:
        scantype = 1

    run(interface, scantype=1)

if __name__ == '__main__':
    main()


from matplotlib.colors import ListedColormap
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt
import numpy as np

def plot_decision_regions(X, y, classifier, test_idx = None, resolution = 0.02):
    labels = np.unique(y)
    markers = ('s', 'x', 'o', '^', 'v')
    colors = ('red', 'blue', 'lightgreen', 'grey', 'cyan')
    cmap = ListedColormap(colors[:len(labels)])

    x1_min = X[:, 0].min() - 1
    x1_max = X[:, 0].max() + 1

    x2_min = X[:, 1].min() - 1
    x2_max = X[:, 1].max() + 1

    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution), np.arange(x2_min, x2_max, resolution))

    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)
    Z = Z.reshape(xx1.shape)

    plt.contourf(xx1, xx2, Z, alpha = 0.3, cmap = cmap)
    plt.xlim(xx1.min(), xx1.max())
    plt.ylim(xx2.min(), xx2.max())

    for idx, cl in enumerate(labels):
        plt.scatter(x = X[y == cl, 0], y = X[y == cl, 1], alpha = 0.8, c = colors[idx], marker = markers[idx], label = cl, edgecolor = 'black')

    if test_idx:
        X_test, y_test = X[test_idx, :], y[test_idx]

        plt.scatter(X_test[:, 0], X_test[:, 1], c = '', edgecolor = 'black', alpha = 1.0, linewidth = 1, marker = 'o', s = 100, label = 'test set')

def get_accuracy(prediction, target):
    assert len(prediction) == len(target)
    accuracy = accuracy_score(prediction, target)
    return accuracy

def get_mse(prediction, target):
    assert len(prediction) == len(target)
    n = len(prediction)
    dif = (prediction - target) ** 2
    s = dif.sum()
    return s / n


import os
import json
import mikochiku_alarm
from PyQt5.QtWidgets import QWidget, QMainWindow
from PyQt5.QtWidgets import QComboBox, QLabel, QFrame, QLineEdit, QPushButton, QCheckBox
from PyQt5.QtGui     import QFont
from PyQt5.QtCore import Qt
import settings

class ConfigTab(QMainWindow):

    def __init__(self, parent=None):
        super(ConfigTab, self).__init__(parent)
        self.initUI_sub()
        self.parent_obj = parent

    def initUI_sub(self):
        self.header("Language", 10)
        self.language_cmb()

        self.header("Channel", 80)
        self.channel_cmb()

        self.header("Add channel", 210)
        self.channel_adder()

        self.toggle_tasktray(335)

        self.setGeometry(580, 300, 220, 370)
        self.setWindowTitle("Config")
        self.show()

    def language_cmb(self):
        self.language_cmb = QComboBox(self)
        self.language_cmb.addItem("language")
        self.language_cmb.addItems(["日本語", "中文", "English"])
        self.language_cmb.currentTextChanged.connect(self.replace_locale_json)
        self.language_cmb.setGeometry(10, 40, 200,25)

    def channel_cmb(self):
        with open("./res/channel/nijisanji.json", encoding="UTF-8") as file:
            self.nijisanji = json.load(file)
        with open("./res/channel/hololive.json" , encoding="UTF-8") as file:
            self.hololive  = json.load(file)
        with open("./res/channel/other_ch.json" , encoding="UTF-8") as file:
            self.other_ch  = json.load(file)

        self.nijisanji_cmb = QComboBox(self)
        self.hololive_cmb  = QComboBox(self)
        self.other_ch_cmb  = QComboBox(self)

        self.nijisanji_cmb.addItem("nijisanji")
        self.hololive_cmb .addItem("hololive")
        self.other_ch_cmb .addItem("other")

        for v in self.nijisanji:
            self.nijisanji_cmb.addItem(v["name"])
        for v in self.hololive:
            self.hololive_cmb.addItem(v["name"])
        for v in self.other_ch:
            self.other_ch_cmb.addItem(v["name"])

        self.nijisanji_cmb.setGeometry(10, 110, 200, 25)
        self.hololive_cmb .setGeometry(10, 140, 200, 25)
        self.other_ch_cmb .setGeometry(10, 170, 200, 25)

    def channel_adder(self):
        self.label("name", 240)
        name_inp = QLineEdit(self)
        name_inp.setGeometry(50, 240, 160, 25)

        self.label("URL", 270)
        name_inp = QLineEdit(self)
        name_inp.setGeometry(50, 270, 160, 25)

        clear_btn = QPushButton("clear", self)
        clear_btn.setGeometry(50, 300, 75, 25)

        ok_btn = QPushButton("ok", self)
        ok_btn.setGeometry(135, 300, 75, 25)

    def header(self, text, y):
        header = QLabel(self)
        header.setFont(QFont("Yu Gothic", 15))
        header.setText(text)
        header.setGeometry(10, y, 200, 25)

    def label(self, text, y):
        label = QLabel(self)
        # label.setFont(QFont("Yu Gothic", 10))
        label.setFont(QFont("Helvetica [Cronyx]", 10))
        label.setText(text)
        label.setGeometry(10, y, 40, 25)

    def replace_locale_json(self):
        # miko = mikochiku_alarm.MikochikuAlarm(self)

        path = "./res/lang/locale.json"
        with open(path, mode='r') as file:
            dict_json = json.load(file)
            selected = self.get_locale_cmb()
        if not selected:
            pass
        else:
            dict_json["locale"] = selected
            with open(path, mode='w') as file:
                json.dump(dict_json, file)

        self.parent_obj.update_ui_language()

    def get_locale_cmb(self):
        if   self.language_cmb.currentText() == "日本語" : return "ja_JP"
        elif self.language_cmb.currentText() == "中文"   : return "zh_CN"
        elif self.language_cmb.currentText() == "English": return "en_US"

    def toggle_tasktray(self, y):
        cb = QCheckBox(self)
        cb.setFont(QFont("Yu Gothic", 9))
        cb.setText("Store in the task tray")
        cb.setCheckState(Qt.Checked)
        cb.setGeometry(10, y, 200, 25)


from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

df = pd.read_csv('/home/filip/Desktop/MachineLearning/dataset.csv')
#df.drop('date',axis=1)
y = df['deaths']
X = df[['cases','date']]
benchmark = []
knn = KNeighborsClassifier(n_neighbors = 2)

#Benchmark na test_size
i=0
testSplitNumber=0
XAxisNumbers=[]
XAxisCounter=0
testSplitNumberExponent=0.01
whileLoopLenght = 1 / testSplitNumberExponent -5
while i<=whileLoopLenght:
	print("benchmark loop: ",i)
	testSplitNumber+=testSplitNumberExponent
	XAxisCounter+=testSplitNumberExponent*100
	X_train,X_test, y_train,y_test = train_test_split(X,y,test_size = testSplitNumber, random_state=20)
	knn.fit(X_train,y_train)
	benchmark.append(round(knn.score(X_test,y_test),3)*100)
	XAxisNumbers.append(XAxisCounter)
	i+=1

#Predikce s optimální test_size
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.16, random_state=20)
knn.fit(X_train,y_train)

print("\n \n \n \n")
print("accuracy: ",round(knn.score(X_test,y_test),3)*100,"%")
print("\n accuracy raw:", knn.score(X_test,y_test))

plt.plot(benchmark)
plt.ylabel('Model accuracy %')
plt.xlabel('Size of train split %')
plt.show()
#print(y)


from django.http import HttpResponse
from django.shortcuts import render, redirect
from myapp.models import  *
from django.contrib import messages

def show_about_page(request):
    print("requesting show about page")
    name="Channel name is Learn Code with Durgesh"
    link="https://www.youtube.com/watch?v=697_0UGxDPk&t=182s"
    data={
        "name":name,
        "link":link
    }
    return render(request,"about.html",data)

def show_home_page(request):
    images = Image.objects.all()
    cats = Category.objects.all()
    data = {"images":images,"cats":cats}
    return render(request,"home.html",data)

def show_category(request,cid):
    cats = Category.objects.all()
    category=Category.objects.get(pk=cid)

    images = Image.objects.filter(cat=category)
    data = {"images": images, "cats": cats}
    return render(request, "home.html", data)

def home(request):
    return redirect('/home')

def search(request):
    query = request.GET['query']
    if len(query) > 78:
        images = Image.objects.none()
    else:
        imageTitle = Image.objects.filter(title__icontains=query)
        imageDescription = Image.objects.filter(description__icontains=query)
        images = imageTitle.union(imageDescription)

    if images.count() == 0:
        messages.warning(request,"No search results found. Please refine your query")
    cats = Category.objects.all()
    data = {"images": images,"cats": cats,"query":query}
    return render(request,"search.html",data)


from flask import abort
from sqlalchemy.exc import SQLAlchemyError
from app_client.app_client.models import Client, db
from app_client.app_client.handle_excep import valid_none

def list_client():
    try:
        cliente_list = Client.query.all()
    except SQLAlchemyError as error:
        abort(500, str(error.__dict__["orig"]))
    return cliente_list

def list_client_by_id(_id):
    try:
        client = Client.query.filter_by(id=_id).first()
    except SQLAlchemyError as error:
        abort(500, str(error.__dict__["orig"]))

    valid_none(404,"client {} not found".format(_id), client )
    return client

def insert_client(body):

    valid_none(400, "'name' is required", body)
    valid_none(400, "'name' is required", body.get("name"))
    valid_none(400, "'email' is required", body.get("email"))

    try:
        client = Client(email=body.get("email"),name=body.get("name"))
        db.session.add(client)
        db.session.commit()
    except SQLAlchemyError as error:
        abort(500, str(error.__dict__["orig"]))

    return client

def update_client(body, _id):

    valid_none(400, "'id' is required", _id)
    valid_none(400, "'name' is required", body)
    valid_none(400, "'name' is required", body.get("name"))
    valid_none(400, "'email' is required", body.get("email"))

    try:
        client_to_update = Client.query.filter_by(id=_id).first()
    except SQLAlchemyError as error:
        abort(500, str(error.__dict__["orig"]))

    valid_none(404,"client {} not found".format(_id), client_to_update )

    client_to_update.name = body.get("name")
    # client_to_update.email = body["email"] if "email" in body else client_to_update.email
    client_to_update.email = body.get("email")

    try:
        db.session.add(client_to_update)
        db.session.commit()
    except SQLAlchemyError as error:
        abort(500, str(error.__dict__["orig"]))

    return client_to_update


from rest_framework import serializers
from s2016.models import Doctor, LANGUAGE_CHOICES, STYLE_CHOICES

class DoctorSerializer(serializers.Serializer):
  pk = serializers.IntegerField(read_only=True)
  name = serializers.CharField(max_length=100)
  education = serializers.CharField(max_length=100)
  specialty = serializers.CharField(max_length=100)
  is_active = serializers.BooleanField(required=True)

  def create(self, validated_data):
    '''
    Create and return a new Doctor data
    '''
    return Doctor.objects.create(**validated_data)

  def update(self, instance, validated_data):
    '''
    Update and return existing 'Doctor' instance
    '''

    instance.name = validated_data.get('name', instance.name)
    instance.education = validated_data.get('education', instance.education)
    instance.specialty = validated_data.get('specialty', instance.name)
    instance.is_active = validated_data.get('is_active', instance.is_active)
    instance.save()
    return instance


# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
from lxml import etree
from WaiBaoSpider.utils.csvWriter import CSVDumper
from WaiBaoSpider.utils.base import unicode_body, deal_ntr
import os

class ChuzhouSpider(scrapy.Spider):
    name = 'chuzhou'
    base_url = "http://www.chuzhou.gov.cn/content/column/2983442?organId=&pageIndex={}"
    data_path = os.getcwd() + "/WaiBaoSpider/data/"
    if os.path.exists(data_path):
        pass
    else:
        os.mkdir(data_path)
    dump_list = CSVDumper(data_path + "%s_list.csv" % name)
    dump_detail = CSVDumper(data_path + "%s_detail.csv" % name)
    custom_settings = {
        # 'DOWNLOAD_DELAY': 2,
    }

    def start_requests(self):
        for i in range(1, 17):
            # for i in range(1, 2):
            url = self.base_url.format(i)
            print(url)
            yield Request(url, callback=self.parse_list)

    def parse_list(self, response):
        body = unicode_body(response)
        html = etree.HTML(body)
        lines = html.xpath("//div[@class='lyy_listbox']/ul/li")
        print(len(lines))
        for info in lines:
            item = {}
            item[u"信件类型"] = info.xpath(".//p[@class='p1']/text()")[0].strip() if info.xpath(
                ".//p[@class='p1']/text()") else ""
            item[u"信件编号"] = "'{}'".format(
                info.xpath(".//p[@class='p2']/text()")[0].strip() if info.xpath(".//p[@class='p2']/text()") else "")
            item[u"信件标题"] = info.xpath(".//p[@class='p3']/a/@title")[0].strip() if info.xpath(
                ".//p[@class='p3']/a/@title") else ""
            item[u"留言时间"] = info.xpath(".//p[@class='p4']/text()")[0].strip() if info.xpath(
                ".//p[@class='p4']/text()") else ""
            item[u"评价结果"] = info.xpath(".//p[@class='p5']/text()")[0].strip() if info.xpath(
                ".//p[@class='p5']/text()") else ""
            item[u"链接"] = info.xpath(".//p[@class='p3']/a/@href")[0].strip() if info.xpath(
                ".//p[@class='p3']/a/@href") else ""
            self.dump_list.process_item(item)
            yield Request(item[u"链接"], callback=self.parse_detail, meta={"url": item[u"链接"], "bnum": item[u"信件编号"]})

    def parse_detail(self, response):
        body = unicode_body(response)
        data = response.meta
        html = etree.HTML(body)
        item = {}
        item[u"标题"] = html.xpath("//h1[@class='mesgtitle']/text()")[0].strip() if html.xpath(
            "//h1[@class='mesgtitle']/text()") else ""
        item[u"编号"] = data["bnum"]
        item[u"来信时间"] = html.xpath("(//td[@class='nr'])[2]/div/text()")[0].strip() if html.xpath(
            "(//td[@class='nr'])[2]/div/text()") else ""
        item[u"类型"] = html.xpath("(//td[@class='nr'])[3]/text()")[0].strip() if html.xpath(
            "(//td[@class='nr'])[3]/text()") else ""
        item[u"网友"] = deal_ntr(html.xpath("(//td[@class='nr'])[4]/text()")[0].strip() if html.xpath(
            "(//td[@class='nr'])[4]/text()") else "")
        content = html.xpath("(//td[@class='nrtxt'])[1]//text()") if html.xpath(
            "(//td[@class='nrtxt'])[1]//text()") else []
        item[u"内容"] = deal_ntr("".join(content).strip())
        item[u"转办单位"] = html.xpath("(//td[@class='nr'])[5]/text()")[0].strip() if html.xpath(
            "(//td[@class='nr'])[5]/text()") else ""
        re_content = html.xpath("(//td[@class='nrtxt'])[2]//text()") if html.xpath(
            "(//td[@class='nrtxt'])[2]//text()") else []
        item[u"答复情况"] = deal_ntr("".join(re_content).strip())
        item[u"用户满意度评价"] = html.xpath("(//td[@class='nr'])[6]/text()")[0].strip() if html.xpath(
            "(//td[@class='nr'])[6]/text()") else ""
        item[u"链接"] = data["url"]
        self.dump_detail.process_item(item)


# -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2020-11-26 16:07
import glob
import json
import re
from urllib.error import HTTPError
import os

from hanlp_common.io import load_json

from elit.datasets.srl.ontonotes4 import ONTONOTES4_TASKS_HOME, ONTONOTES4_HOME
from elit.datasets.srl.ontonotes5._utils import convert_jsonlines_to_IOBES
from elit.utils.io_util import get_resource, path_from_url
from elit.utils.log_util import cprint
from elit.utils.time_util import CountdownTimer

_ONTONOTES4_CHINESE_HOME = ONTONOTES4_HOME + 'files/data/chinese/annotations/'
_ONTONOTES4_CHINESE_TASKS_HOME = ONTONOTES4_TASKS_HOME + 'ner/chinese/'

ONTONOTES4_NER_4TYPES_CHINESE_TRAIN = _ONTONOTES4_CHINESE_TASKS_HOME + 'train.chinese.v4.ner.jsonlines'
ONTONOTES4_NER_4TYPES_CHINESE_DEV = _ONTONOTES4_CHINESE_TASKS_HOME + 'development.chinese.v4.ner.jsonlines'
ONTONOTES4_NER_4TYPES_CHINESE_TEST = _ONTONOTES4_CHINESE_TASKS_HOME + 'test.chinese.v4.ner.jsonlines'

ONTONOTES4_NER_4TYPES_CHINESE_TSV_TRAIN = _ONTONOTES4_CHINESE_TASKS_HOME + 'train.chinese.v4.ner.tsv'
ONTONOTES4_NER_4TYPES_CHINESE_TSV_DEV = _ONTONOTES4_CHINESE_TASKS_HOME + 'development.chinese.v4.ner.tsv'
ONTONOTES4_NER_4TYPES_CHINESE_TSV_TEST = _ONTONOTES4_CHINESE_TASKS_HOME + 'test.chinese.v4.ner.tsv'

ENAMEX = re.compile('<ENAMEX TYPE=\"(\w+?)\".*?>(.+?)</ENAMEX>')

try:
    get_resource(ONTONOTES4_HOME, verbose=False)
except HTTPError:
    intended_file_path = path_from_url(ONTONOTES4_HOME)
    cprint('Ontonotes 4.0 is a [red][bold]copyright[/bold][/red] dataset owned by LDC which we cannot re-distribute. '
           f'Please apply for a licence from LDC (https://catalog.ldc.upenn.edu/LDC2011T03) '
           f'then download it to {intended_file_path}')
    exit(1)

def load_ner(name_file):
    with open(name_file) as src:
        lines = src.readlines()
        lines = lines[1:-1]  # remove <DOC> and </DOC>
        sentences = []
        ner = []
        for line in lines:
            line = line.strip()
            if not line:
                continue
            if line == 'ＥＭＰＴＹ':
                continue
            tokens = []
            entities = []
            offset = 0
            for entity in ENAMEX.finditer(line):
                b, e = entity.span()
                if b > offset:
                    form = line[offset:b]
                    words = tokenize(form)
                    tokens.extend(words)
                tag = entity.group(1).strip()
                form = entity.group(2).strip()
                words = tokenize(form)
                # According to Named Entity Recognition with Bilingual Constraints:
                # In this paper, we selected the four most common named entity types,
                # i.e., PER (Person), LOC (Location), ORG (Organization) and GPE (Geo-Political Entities)
                if tag in ('PERSON', 'LOC', 'ORG', 'GPE'):
                    entities.append((len(tokens), len(tokens) + len(words) - 1, tag))
                tokens.extend(words)
                offset = e
            if offset < len(line):
                form = line[offset:]
                words = tokenize(form)
                tokens.extend(words)

            sentences.append(tokens)
            ner.append(entities)
            # print(tokens)
            # print([(tokens[b: e + 1], l) for b, e, l in entities])
        return {'doc_key': '/'.join(name_file.split(os.path.sep)[-4:])[:-len('.name')], 'sentences': sentences,
                'ner': ner}

def tokenize(form):
    words = [x.strip() for x in form.split()]
    words = [x for x in words if x]
    return words

def make_ner_jsonlines_if_needed():
    root = get_resource(ONTONOTES4_HOME)
    jsonfiles = dict(zip(['train', 'dev', 'test'], [os.path.join(root, x[len(ONTONOTES4_HOME):]) for x in
                                                    [ONTONOTES4_NER_4TYPES_CHINESE_TRAIN,
                                                     ONTONOTES4_NER_4TYPES_CHINESE_DEV,
                                                     ONTONOTES4_NER_4TYPES_CHINESE_TEST]]))
    if all(os.path.isfile(x) for x in jsonfiles.values()):
        return

    for file in jsonfiles.values():
        os.makedirs(os.path.dirname(file), exist_ok=True)

    zh_root = get_resource(_ONTONOTES4_CHINESE_HOME)
    name_files = glob.glob(f'{zh_root}/**/*.name', recursive=True)
    fps = dict((k, open(v, 'w', encoding='utf-8')) for k, v in jsonfiles.items())
    # glyce = load_json('data/ner/ontonotes4/splits.json')
    # glyce_splits = dict()
    # unsure = set()
    # for portion, sents in glyce.items():
    #     for sent in sents:
    #         sent = sent.replace('TYPE=\"PRODUCT\"E_OFF=\"1\">', '')
    #         exsisting = glyce_splits.get(sent, None)
    #         if exsisting and exsisting != portion:
    #             print(f'Conflict {sent} in both {exsisting} and {portion}')
    #             unsure.add(sent)
    #         glyce_splits[sent] = portion
    timer = CountdownTimer(len(name_files))
    for file in sorted(name_files):
        # According to Named Entity Recognition with Bilingual Constraints:
        # This corpus includes about 400 document pairs (chtb 0001-0325, ectb 1001-1078).
        # We used odd-numbered documents as development data and even-numbered documents as test data.
        # We used all other portions of the named entity annotated corpus as training data for the monolingual systems
        basename = os.path.basename(file)
        genre, remaining = basename.split('_')
        fid = int(remaining[:-len('.name')])
        if genre in ('chtb', 'ectb') and (1 <= fid <= 325 or 1001 <= fid <= 1078):
            if fid % 2:
                portion = 'dev'
            else:
                portion = 'test'
        else:
            portion = 'train'
        timer.log(f'Pre-processing {basename} to {portion} set [blink][yellow]...[/yellow][/blink]')
        doc = load_ner(file)
        # for sent in doc['sentences']:
        #     text = ''.join(sent)
        #     expected_portion = glyce_splits.get(text, None)
        #     # from difflib import SequenceMatcher
        #     # similar_keys = sorted(glyce_splits.keys(),
        #     #                       key=lambda k: SequenceMatcher(None, k, text).ratio(),
        #     #                       reverse=True)[:5]
        #     if expected_portion != portion and text not in unsure:
        #         print(text)
        fps[portion].write(json.dumps(doc, ensure_ascii=False) + '\n')
    for f in fps.values():
        f.close()

def make_ner_tsv_if_needed():
    root = get_resource(ONTONOTES4_HOME)
    tsv_files = [os.path.join(root, x[len(ONTONOTES4_HOME):]) for x in
                 [ONTONOTES4_NER_4TYPES_CHINESE_TSV_TRAIN, ONTONOTES4_NER_4TYPES_CHINESE_TSV_DEV,
                  ONTONOTES4_NER_4TYPES_CHINESE_TSV_TEST]]

    if all(os.path.isfile(x) for x in tsv_files):
        return

    for file in tsv_files:
        os.makedirs(os.path.dirname(file), exist_ok=True)

    for j, t in zip(
            [ONTONOTES4_NER_4TYPES_CHINESE_TRAIN, ONTONOTES4_NER_4TYPES_CHINESE_DEV,
             ONTONOTES4_NER_4TYPES_CHINESE_TEST],
            tsv_files):
        convert_jsonlines_to_IOBES(j, t, doc_level_offset=False, normalize_token=True)

make_ner_jsonlines_if_needed()
make_ner_tsv_if_needed()


revision = '044'
down_revision = '043'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.sql.expression import table

MYSQL_ENGINE = 'InnoDB'
MYSQL_CHARSET = 'utf8'

def upgrade(active_plugins=None, options=None):
    op.create_table(
        'story_types',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.Column('updated_at', sa.DateTime(), nullable=True),
        sa.Column('name', sa.String(50), nullable=True),
        sa.Column('icon', sa.String(50), nullable=True),
        sa.Column('restricted', sa.Boolean(), default=False),
        sa.Column('private', sa.Boolean(), default=False),
        sa.Column('visible', sa.Boolean(), default=True),
        sa.PrimaryKeyConstraint('id'),
        mysql_engine=MYSQL_ENGINE,
        mysql_charset=MYSQL_CHARSET
    )

    bind = op.get_bind()

    story_types_table = table(
        'story_types',
        sa.Column('name', sa.String(50), nullable=True),
        sa.Column('icon', sa.String(50), nullable=True),
        sa.Column('restricted', sa.Boolean(), default=False),
        sa.Column('private', sa.Boolean(), default=False),
        sa.Column('visible', sa.Boolean(), default=True),
    )

    bind.execute(story_types_table.insert().values(
        name='bug',
        icon='fa-bug'
    ))

    bind.execute(story_types_table.insert().values(
        name='feature',
        icon='fa-lightbulb-o',
        restricted=True
    ))

    bind.execute(story_types_table.insert().values(
        name='private_vulnerability',
        icon='fa-lock',
        private=True
    ))

    bind.execute(story_types_table.insert().values(
        name='public_vulnerability',
        icon='fa-bomb',
        visible=False
    ))

def downgrade(active_plugins=None, options=None):
    op.drop_table('story_types')


'''
https://www.codewars.com/kata/520b9d2ad5c005041100000f/python
'''

from curses.ascii import isalpha

def pig_it1(text:str):
    content = []
    txt_lst = text.split()
    for txt in txt_lst:
        if not isalpha(txt[0]):
            content.append(txt)
            continue
        first_letter = txt[0]
        txt = txt.replace(first_letter,'',1)
        txt += first_letter+'ay'
        content.append(txt)
    return ' '.join(content)

def pig_it(text:str):
    txt_lst = text.split()
    return ' '.join([txt[1:]+txt[0]+'ay' if isalpha(txt[0]) else txt for txt in txt_lst])

print(pig_it('Pig latin is cool !'))


#%%
import re
import pandas as pd
import nltk
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import plotly.express as px
from termcolor import colored
from DataUtility import DataUtility

pd.set_option('display.max_rows', 100)
pd.options.display.max_colwidth = 200
pd.options.mode.chained_assignment = None
# this powers the word_tokenize function
nltk.download('punkt')

#%%
# read in data
url_scriptures = 'https://github.com/wilfordwoodruff/wilford_woodruff_hack23/raw/main/data/lds-scriptures.csv'
url_woodruff = "https://github.com/wilfordwoodruff/Main-Data/raw/main/data/derived/derived_data.csv"

data_scriptures = pd.read_csv(url_scriptures)
data_woodruff = pd.read_csv(url_woodruff)

data_woodruff = data_woodruff.query("`Document Type` == 'Journals'")

# data_woodruff2 = pd.read_csv('../data/data_woodruff.csv')
data_scriptures
data_woodruff

#%%

class Scriptures:

    def __init__(self, data_scriptures) -> None:
        self.data_scriptures = data_scriptures

    def clean(self):
        books = ["New Testament","Book of Mormon", "Docrtine and Covenants", "Pearl of Great Price"]
        self.data_scriptures = self.data_scriptures.query('volume_title == @books')
        self.data_scriptures['scripture_text'] = self.data_scriptures['scripture_text'].str.lower()

class WoodruffPapers:

    def __init__(self, data_woodruff, sample = True) -> None:
        if sample:
            self.data_woodruff = data_woodruff.sample(100)
        else:
            self.data_woodruff = data_woodruff

    # clean suggestions
    @staticmethod
    def clean_suggestions(string):
        pattern = r'\[\[(.*?)\]\]'
        suggestions = re.findall(pattern, string)
        for suggestion in suggestions:
            replacement = suggestion.split('|')[0] # split the words before and after the '|'
            suggestion_exact_match = '[[' + suggestion + ']]'
            string = string.replace(suggestion_exact_match, replacement)
        return string

    @staticmethod
    def clean_ampersands(string):
        string = DataUtility.replace_regex(string, r'\'', r"'")
        string = DataUtility.replace_regex(string, r'\&c', r"'")
        string = DataUtility.replace_regex(string, r'\&', r'and')
        # print(len(re.findall(r'\&', string)))
        return string

    @staticmethod
    def remove_birthyears(string):
        string = re.sub(r'(, b\.)', string)
        return string

    @staticmethod
    def fix_typos(string):
        DataUtility.replace_regex('travling', regex = r'', replacement = r'')

    def clean(self):
        # rename Text Only Transcript to 'text'
        self.data_woodruff = self.data_woodruff.rename(columns={"Text Only Transcript": "text"})
        self.data_woodruff['text'] = self.data_woodruff['text'].apply(DataUtility.replace_regex, regex = r'\[(\w+)\]')
        self.data_woodruff['text'] = self.data_woodruff['text'].apply(DataUtility.replace_regex, regex = r',', replacement = r'')
        self.data_woodruff['text'] = self.data_woodruff['text'].apply(DataUtility.replace_regex, regex = r'[\{\}\~]', replacement = r'')
        self.data_woodruff['text'] = self.data_woodruff['text'].apply(DataUtility.replace_regex, regex = r'\s{2}', replacement = r' ')
        self.data_woodruff['text'] = self.data_woodruff['text'].apply(DataUtility.remove_new_lines)
        self.data_woodruff['text'] = self.data_woodruff['text'].apply(WoodruffPapers.clean_suggestions)
        self.data_woodruff['text'] = self.data_woodruff['text'].apply(WoodruffPapers.clean_ampersands)
        # TODO: implement typos and symbols
        typos = {'sacrafice':'sacrifice',
                 'discours':'discourse'}
        symbols = {
                'b. 1795':'',
                '<U+25CA>':'',
                '&':'and',
                '&c':'and',
                ''':"'",
                '[FIGURE]':'',
                'discours':'discourse'
            }
        things_to_remove = ['WW 1841-2',
                            'Front cover',
                            'THE SECOND BOOK OF WILLFORD FOR 1839',
                            'W\. WOODRUFFs DAILY JOURNAL AND HISTORY IN 1842',
                            "WILFORD WOODRUFF's DAILY JOURNAL AND HISTORY IN 1843",
                            "WILLFORD WOODRUFF'S JOURNAL VOL\. 2\. AND A SYNOPSIS OF VOL\. 1\.",
                            "Willford Woodruff's Journal Containing an Account Of my life and travels from the time of my first connextion with the Church of Jesus Christ of Latter-day Saints",
                            'THE FIRST BOOK OF WILLFORD VOL\. 2\. FOR 1838',
                            r"WILLFORD\. WOORUFF's DAILY JOURNAL AND TRAVELS",
                            r"(\^?FIGURES?\^?)"]

        for regex in things_to_remove:
            self.data_woodruff = DataUtility.regex_filter(self.data_woodruff, column = 'text', regex=regex)
        # lowercase all entries
        self.data_woodruff['text'] = self.data_woodruff['text'].str.lower()

    def preprocess(self):
        self.data_woodruff['phrase'] = self.data_woodruff['text'].apply(DataUtility.split_string_into_list, n = 15)
        # explode dataset so each row contains a single 15 word phrase
        self.data_woodruff = self.data_woodruff.explode('phrase')

    @staticmethod
    def compute_match_percentage(text_woodruff, text_scripture):
        words_woodruff = DataUtility.split_string(text_woodruff)
        words_scripture = DataUtility.split_string(text_scripture)
        vectorizer = TfidfVectorizer()
        ### vectorize words
        # Compute TF-IDF matrices
        tfidf_matrix_woodruff = vectorizer.fit_transform(words_woodruff)
        tfidf_matrix_verse = vectorizer.transform(words_scripture)

        similarity_scores = cosine_similarity(tfidf_matrix_woodruff, tfidf_matrix_verse)
        vectorizer.get_feature_names_out()
        woodruff_word_match_ids   = np.unique(np.where(similarity_scores == 1)[0])
        scriptures_word_match_ids = np.unique(np.where(similarity_scores == 1)[1])
        percent_match_woodruff    = round(len(woodruff_word_match_ids) * 100 / len(words_woodruff), 2)
        return percent_match_woodruff

woodruff_papers = WoodruffPapers(data_woodruff, sample = True)
woodruff_papers.clean()

scriptures = Scriptures(data_scriptures)
scriptures.clean()

scriptures.data_scriptures
woodruff_papers.data_woodruff

#%%

woodruff_papers.preprocess()

woodruff_papers.data_woodruff

#%%
from tqdm import tqdm
results = pd.DataFrame()

for i in tqdm(range(10), desc='processing'):
    verse_title = list(woodruff_papers.data_scriptures['verse_title'])[i]
    text_scripture = list(woodruff_papers.data_scriptures.query('verse_title == @verse_title')['scripture_text'])[0]

    # print('comparing:', verse_title)

    data_sample['text_scripture'] = text_scripture
    data_sample['percentage_match'] = (data_sample['phrase'].apply(WoodruffPapers.compute_match_percentage,
                                                        text_scripture=text_scripture))
    data_sample.sort_values(by = 'percentage_match', ascending=False)

    if data_sample['percentage_match'].max() > 50:
        # print(colored('attaching results', 'green'), data_sample['percentage_match'].max())
        top_3_rows = data_sample.nlargest(3, 'percentage_match')[['phrase', 'text_scripture', 'percentage_match']]
        results = pd.concat([results, top_3_rows])

#%%
data_sample.to_csv('sample.csv', index = False)

#%%
## EDA
text_woodruff = DataUtility.combine_rows(woodruff_papers.data_woodruff.head(1000)['text'])
text_verse = DataUtility.combine_rows(woodruff_papers.data_scriptures.head(1000)['scripture_text'])

print(text_verse)
print(text_woodruff)
freq = DataUtility.create_frequency_distribution(text_woodruff)
print(freq.head(100))
freq.tail(100)

chart = px.bar(freq.head(100), x='frequency', y='word', text_auto='.2s',orientation='h')
chart.show()

# chart = px.bar(freq.tail(100), x='frequency', y='word', text_auto='.2s',orientation='h')
# chart.show()

# chart = px.bar(freq.tail(100), x='frequency', y='word', text_auto='.2s',orientation='h')
# chart.show()

#%%


from fastai.vision.all import *
from fastai.vision.all import cnn_learner #imports relevant libraries to create a CNN model

from fastaudio.core.all import *
from fastaudio.augment.all import *
from fastaudio.ci import skip_if_ci       #imports relevant libraries to prepare audio data

from pandas.api.types import is_string_dtype, is_numeric_dtype, is_categorical_dtype #imports library for dataframe manipulation
import json #imports library for JSON manipulation
import os   #imports library for directory control
import traceback #library to produce error traces

#function designed to create a JSON file
def writeToJSONFile(fileName,data):
    with open(fileName,'w') as fp:
        json.dump(data,fp)

@skip_if_ci
#function designed to train a cnn model
def run_learner():
    # epochs are a bit longer due to the chosen melspectrogram settings
    learn.fine_tune(10,cbs = CSVLogger())
    out  = pd.read_csv('history.csv')
    print(out.to_string())
    values = {}
    results = []
    print(type(out.to_string()))
    results.append(out.to_string())
    values['results'] = results
    writeToJSONFile('results.json',values) #creates a results json file

try:
    with open("..\experiments\experiment.json", "r") as read_file:
        data = json.load(read_file)
        path = data['train'] #fetches necessary file paths
        for file in os.listdir(os.path.abspath(path)):
            if file.endswith(".csv") and file!='dataset.csv':
                df = pd.read_csv(path+"\\"+file) #initialises dataframe
                break
        #identifies files and their labels
        x = str('filename')
        y = str('label')
        z = path + '\\'

        #Describes necessary data preprocessing for training
        cfg = AudioConfig.BasicMelSpectrogram(n_fft=512)
        a2s = AudioToSpec.from_cfg(cfg)
        item_tfms = [ResizeSignal(4500),a2s]

        #initialises a data block from the dataset
        auds = DataBlock(blocks = (AudioBlock, CategoryBlock),
                         get_x = ColReader(x, pref=z),
                         splitter = TrainTestSplitter(test_size=0.2, random_state=42, stratify=None, shuffle=True),
                         item_tfms = item_tfms,
                         get_y = ColReader(y))

        #converts data block into a data loader
        dbunch = auds.dataloaders(df, bs=32,num_workers=0)
        learn = cnn_learner(dbunch,
                    resnet18,
                    n_in=1,  # <- This is the only audio specific modification here
                    loss_func=CrossEntropyLossFlat(),
                    metrics=[accuracy,error_rate])

        os.chdir(data['trained_cnn'])#changes directory for storing results and exported model

        run_learner() #trains the cnn
        learn.export()#outputs the trained model as a .pkl file
except Exception:
    traceback.print_exc()


# -*- coding: utf-8 -*-
"""Test s3 class."""

import pytest
from unittest.mock import Mock, patch

from s3_client import s3_client

from conftest import BUCKET_NAME

def test_download_object(s3, s3_bucket, tmpdir):
    """Test object download."""
    s3_client.log = Mock()
    obj_name = "my_object"
    obj_body = "Test object content"
    dest_name = "{}/{}".format(tmpdir, obj_name)
    s3_bucket.Bucket(BUCKET_NAME).put_object(Key=obj_name, Body=obj_body)
    s3.download_object(BUCKET_NAME, obj_name, dest_name)
    assert (tmpdir / obj_name).read() == obj_body

@pytest.mark.parametrize(
    "extraargs, versionid", [(None, None), ({"VersionId": "1234567890"}, "1234567890")]
)
def test_download_object_extraargs(s3, extraargs, versionid):
    """Test download object ExtraArgs option."""
    s3_client.log = Mock()
    obj_name = "my_object"
    dest_name = "/tmp"
    with patch.object(s3, "s3_resource"):
        with patch.object(s3_client, "ProgressBar"):
            s3_client.ProgressBar.return_value.__enter__.return_value.update_to = None

            s3.download_object(BUCKET_NAME, obj_name, dest_name, versionid)

            s3.s3_resource.Bucket(BUCKET_NAME).download_file.assert_called_with(
                obj_name, dest_name, ExtraArgs=extraargs, Callback=None
            )


import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import pandas as pd
from scipy.stats import hmean
from productiecapaciteit.src.capaciteit_strang import strangWeerstand
import numpy as np
from scipy.optimize import least_squares
from scipy import linalg
from dawacotools import get_daw_ts_meteo

from productiecapaciteit.src.strang_analyse_fun2 import (
    get_config,
    get_false_measurements,
)
from wvp_transient_funs import objective
from wvp_transient_funs import dis

from productiecapaciteit.src.weerstand_pandasaccessors import LeidingResistanceAccessor
from productiecapaciteit.src.weerstand_pandasaccessors import WellResistanceAccessor
from productiecapaciteit.src.weerstand_pandasaccessors import WvpResistanceAccessor

data_fd = os.path.join("..", "data")
config_fn = "strang_props6.xlsx"
config = get_config(os.path.join(data_fd, config_fn))

r = 0.6
kD0 = 200.0  # m/d2 at reference temperature
c = 50.0  # d
S = 0.2

strang = "Q200" # "IK102"
ci = config.loc[strang]

# get observations
df_fp = os.path.join(data_fd, "Merged", f"{strang}-PKA-DSEW036680.feather")
df = pd.read_feather(df_fp)
df["Datum"] = pd.to_datetime(df["Datum"])
df.set_index("Datum", inplace=True)
include_rules = ["Unrealistic flow"]
untrusted_measurements = get_false_measurements(
    df, ci, extend_hours=1, include_rules=include_rules
)
df = df.loc[~np.array(untrusted_measurements)]
df = df.loc[~np.isnan(df.Q)]
dfm = df.resample("12h", label="right").mean()
dfm = dfm.loc[~np.isnan(dfm.Q)]

# get initial estimates
filterweerstand_fp = os.path.join(
    "..", "results", "Filterweerstand", "Filterweerstand_modelcoefficienten.xlsx"
)
wvpweerstand_fp = os.path.join(
    "..", "results", "Wvpweerstand", "Wvpweerstand_modelcoefficienten.xlsx"
)
df_a_wvp = pd.read_excel(wvpweerstand_fp, sheet_name=strang, index_col=0).squeeze(
    "columns"
)

# initial, lower limit, upper limit
params = dict(
    alpha=[(r**2 * S / 4) ** 0.5, 0.0, 100 * (r**2 * S / 4) ** 0.5],
    beta=[(1 / (c * S)) ** 0.5, 0.0, (1 / (c * S)) ** 0.5],
    kD0=[140.0, 50.0, 500.0],
    time_delta=[df_a_wvp.temp_delta, 0.0, 10.0],
    temp_time_offset=[df_a_wvp.time_offset, 31 + 2 * 20, 365],
    # alpha_multi=[(1**2 * S / 4) ** 0.5, 0.0, 100 * (1**2 * S / 4) ** 0.5],
    alpha_multi=[2.7, 0.1, 25.]
)

dx_put = int(ci.dx_tussenputten)
dx_mirrorwell = pd.eval(ci.dx_mirrorwell)  # [li for li in pd.eval(ci.dx_mirrorwell) if li[1] < 200]
nput_model = 15

multiwell = (
    [(li[0], 2 * li[1]) for li in dx_mirrorwell] +
    [(2, i * dx_put) for i in range(1, nput_model + 1)] +
    [(li[0], 2 * dis(i * dx_put, 2 * li[1])) for li in dx_mirrorwell for i in range(1, nput_model + 1)]
)
ds_rain = get_daw_ts_meteo('235', "Neerslag")
pextra = dict(
    index=dfm.index.values[:-1],
    drawdown_obs=(dfm.pandpeil - dfm.gws0).values[:-1],
    Q_obs=dfm.Q.values[1:] / ci.nput,
    temp_ref=dfm.gwt0.median(),
    dt_lower=pd.Timedelta(value=hmean(np.diff(dfm.index) / pd.Timedelta(1, unit='D')), unit="D"),
    multiwell_contains_r_self=False,
    multiwell=multiwell,
    # rain=np.interp(dfm.index[1:], ds_rain.index, ds_rain),
    frac_step_max=0.95
)
assert np.isnan(pextra["Q_obs"]).sum() == 0

args = [i[0] for i in params.values()]

res = least_squares(
    objective,
    x0=[i[0] for i in params.values()],
    bounds=([i[1] for i in params.values()], [i[2] for i in params.values()]),
    # loss="arctan",
    f_scale=0.5,
    kwargs=pextra
)

def get_perr(res):
    if np.any(res.active_mask):
        print(f"{res.active_mask} True for params at bounds")
    U, s, Vh = linalg.svd(res.jac, full_matrices=False)
    tol = np.finfo(float).eps * s[0] * max(res.jac.shape)
    w = s > tol
    cov = (Vh[w].T / s[w]**2) @ Vh[w]  # robust covariance matrix
    chi2dof = np.sum(res.fun**2) / (res.fun.size - res.x.size)
    cov *= chi2dof
    perr = np.sqrt(np.diag(cov))
    perr_rel = perr / res.x

    sl = []
    for xi, perr_ri in zip(res.x, perr_rel):
        sl.append(f"{xi} +/- {perr_ri * 100:.1f}%")

    print('\n'.join(sl))
    return perr

perr = get_perr(res)
a = objective(res.x, return_result=True, **pextra)
plt.plot(pextra["index"], pextra["drawdown_obs"])
plt.plot(pextra["index"], a)

"""
0.11959938975024936 +/- 121.1%
0.3798944820153483 +/- 6.6%
179.63725616239432 +/- 59.3%
6.058479229670381 +/- 5.5%
176.36014752388652 +/- 0.9%
0.16542716576149427 +/- 44.2%
"""
# a = objective((alpha, beta, kD0, temp_delta, temp_time_offset))
print("hoi")


""" Populate database command module """

import os
import json

from django.core.management.base import BaseCommand
from core.settings import BASE_DIR

from schemas.models.cities import City
from schemas.models.countries import Country

class Command(BaseCommand):
    help = "Populate the databse with multiple countries"
    mock_path = os.path.join(BASE_DIR, "tools", "mocks", "MOCK_DATA.json")

    def _load_json_file(self, filename:str) -> dict:
        """ read the filemane and return a dict """

        with open(filename) as rf:
            data = json.loads(rf.read())
        return data

    def _populate_cities(self, values:dict) -> None:
        """ populate city table """

        try:
            _country = Country.objects.get(name=values.get("country"))

            obj, created = City.objects.get_or_create(
                name=values.get("city"),
                state=values.get("state"),
                country=_country,
                state_short=values.get("state_short"),
                postal_code=values.get("postal_code"),
            )

            if created:
                print(f"Created {obj.name}")
            else:
                pass

        except (Country.MultipleObjectsReturned, Country.DoesNotExist) as exc:
            print(exc)

    def handle(self, *args, **options):

        data = self._load_json_file(self.mock_path)

        for element in data:
            try:
                obj, created = Country.objects.get_or_create(
                    name=element.get("country"),
                    code=element.get("coutry_code"),
                    timezone=element.get("timezone"),
                )

                if created:
                    print(f"Created {obj.name}")

            except Exception as EXC:
                continue

        for element in data:
            self._populate_cities(element)