import os
import time
import datetime
import threading
import smtplib
import json

def run_script():
    """Run main script."""
    start = time.time()

    os_to_search = []
    volume_to_search = []
    search_all = []
    not_search = ('BOOTCAMP', 'com.apple.TimeMachine.localsnapshots')

    black_list = set(('Backups.backupdb', 'Backups', 'Backup',
                      '.Spotlight-V100', '.TemporaryItems',
                      '.Trashes', '.DocumentRevisions-V100',
                      '.fseventsd', 'backup', 'Library',
                      'Guest', 'Shared'))

    with os.scandir('/Volumes/') as it:
        for entry in it:
            if entry.name not in not_search:
                for item in os.listdir(f'/Volumes/{entry.name}'):
                    if 'Users' in item:
                        os_to_search.append(entry.name)
                if entry.name not in os_to_search:
                    volume_to_search.append(entry.name)

    for item in os_to_search:
        search_all.append(f'/Volumes/{item}/Users/')
    for item in volume_to_search:
        search_all.append(f'/Volumes/{item}/')

    report = []
    paths = []

    def search(path, ext='.als'):
        for folders, subfolders, filenames in os.walk(path, topdown=True):
            subfolders[:] = [d for d in subfolders if d not in black_list]
            for filename in filenames:
                if filename.endswith(ext):
                    report.append(f'{folders}\n{filename}')
                    paths.append(folders)

    def call_thread(arg, ext):
        n_loop = range(len(arg))
        threads = []
        for i in n_loop:
            search_thread = threading.Thread(target=search, args=(arg[i], ext))
            threads.append(search_thread)
        for i in n_loop:
            threads[i].start()   # ? why all this for i in n_loop
        for i in n_loop:
            threads[i].join()

    def main_script(activation=False, ext='.als', custom=''):
        filename = './src/Update check/.path_report.json'

        # if custom path is assigned in update_settings.json
        # * custom path in update_settings.json must be a list
        if custom:
            call_thread(custom, ext)
            # print('calling with USER custom path')
            return

        # (RESETS SEARCH FOR EVERYTHING)
        # if activation in update_settings.json is true
        # then it will delete the self custom path that has created previously
        if activation is True:
            try:
                os.remove(filename)
            except FileNotFoundError:
                pass

        # call search with self custom path that has created
        # after first search
        try:
            with open(filename, encoding='utf-8') as f:
                # print('calling with SELF custom path ')
                read = json.load(f)
                call_thread(read, ext)

        # if file not found searches everything from the beginning
        # and creates the self custom path for the next search
        except FileNotFoundError:
            call_thread(search_all, ext)
            ordered_paths = set(paths)
            # print('search everything from zero')
            ordered_list = list(ordered_paths)
            with open(filename, 'w', encoding='utf-8') as f:
                json.dump(ordered_list, f)

    # main call for running the main_script()
    try:
        with open('update_settings.json') as f:
            settings = json.load(f)
            main_script(activation=settings['activation'],
                        ext=settings['ext'],
                        custom=settings['custom'])
    except FileNotFoundError:
        main_script(activation=True)

    def send_yandex(text):
        smtp_obj = smtplib.SMTP_SSL('smtp.yandex.com', 465)
        smtp_obj.ehlo()
        # smtp_obj.starttls()
        smtp_obj.login('virgil.sisoe@yandex.com', 'umvZ8JFNHZhm')
        smtp_obj.sendmail('virgil.sisoe@yandex.com',
                          'virgil.sisoe@yandex.com', text.encode('utf-8'))
        smtp_obj.quit()

    end = time.time()
    total_time = datetime.timedelta(seconds=end - start)
    try:
        send_yandex('\n\n'.join(report) + '\n' + str(total_time))
    except Exception as e:
        print('Error 03M.4.1(.000lx@)')

if __name__ == '__main__':
    run_script()


import time
start_time = time.time()
from pathlib import Path
import numpy as np
import sys
from contextlib import redirect_stdout
from matplotlib import pyplot
from sklearn.metrics import confusion_matrix
import itertools

# save the final model to file
from keras.datasets import cifar10
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.optimizers import SGD

from keras.preprocessing.image import load_img
from keras.preprocessing.image import img_to_array
from keras.models import load_model

from keras.preprocessing.image import ImageDataGenerator
import tensorflow as tf
#import tensorflow.compat.v1 as tf
##tf.disable_v2_behavior()
print(tf.__version__)

physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
config = tf.config.experimental.set_memory_growth(physical_devices[0], True)

# load train and test dataset
def load_dataset():
    # load dataset
    (trainX, trainY), (testX, testY) = cifar10.load_data()
    # one hot encode target values
    trainY = to_categorical(trainY)
    testY = to_categorical(testY)
    return trainX, trainY, testX, testY

# scale pixels
def prep_pixels(train, test):
    # convert from integers to floats
    train_norm = train.astype('float32')
    test_norm = test.astype('float32')
    # normalize to range 0-1
    train_norm = train_norm / 255.0
    test_norm = test_norm / 255.0
    # return normalized images
    return train_norm, test_norm

# plot diagnostic learning curves
def summarize_diagnostics(history):
    # plot loss
    pyplot.subplot(211)
    pyplot.title('Cross Entropy Loss')
    pyplot.plot(history.history['loss'], color='blue', label='train')
    pyplot.plot(history.history['val_loss'], color='orange', label='test')
    # plot accuracy
    pyplot.subplot(212)
    pyplot.title('Classification Accuracy')
    pyplot.plot(history.history['accuracy'], color='blue', label='train')
    pyplot.plot(history.history['val_accuracy'], color='orange', label='test')
    # save plot to file
    #filename = sys.argv[0].split('/')[-1]
    results_dir = Path('dsc650/assignments/assignment06/').joinpath('results')
    filename = results_dir.joinpath('Assignment_6.2B_Summarized_Diagnostics_Plot.png')
    pyplot.savefig(filename)
    pyplot.close()

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=pyplot.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    pyplot.imshow(cm, interpolation='nearest', cmap=cmap)
    pyplot.title(title)
    pyplot.colorbar()
    tick_marks = np.arange(len(classes))
    pyplot.xticks(tick_marks, classes, rotation=45)
    pyplot.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        pyplot.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    pyplot.tight_layout()
    pyplot.ylabel('True label')
    pyplot.xlabel('Predicted label')

    results_dir = Path('dsc650/assignments/assignment06/').joinpath('results')
    img_file = results_dir.joinpath('Assignment_6.2B_Confusion Matrix.png')
    pyplot.savefig(img_file)
    pyplot.show()

# define cnn model
def define_model():
    model = Sequential()
    model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(32, 32, 3)))
    model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(MaxPooling2D((2, 2)))
    model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(MaxPooling2D((2, 2)))
    model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(MaxPooling2D((2, 2)))
    model.add(Flatten())
    model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))
    #model.add(Dropout(0.2, input_shape=(60,)))
    model.add(Dropout(0.2))
    model.add(Dense(10, activation='softmax'))
    # compile model
    opt = SGD(lr=0.001, momentum=0.9)
    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
    return model

# load and prepare the image
def load_image(filename):
    # load the image
    img = load_img(filename, target_size=(32, 32))
    # convert to array
    img = img_to_array(img)
    # reshape into a single sample with 3 channels
    img = img.reshape(1, 32, 32, 3)
    # prepare pixel data
    img = img.astype('float32')
    img = img / 255.0
    return img

# run the test harness for evaluating a model
def run_test_harness():
    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
    # load dataset
    print("loaing data set")
    trainX, trainY, testX, testY = load_dataset()
    print("--- %s seconds ---" % (time.time() - start_time))

    # prepare pixel data
    print("preparing pixel data")
    trainX, testX = prep_pixels(trainX, testX)

    # plot first few images
    for i in range(9):
        pyplot.subplot(330 + 1 + i)
        x = trainX[i]
        x = np.reshape(x, (32, 32, 3))
        pyplot.imshow(x)
        #pyplot.imshow(x.transpose(1, 2, 0))
        #plt.imshow(tf.shape(tf.squeeze(x)))
    # show the figure

    results_dir = Path('dsc650/assignments/assignment06/').joinpath('results')
    img_file = results_dir.joinpath('Assignment_6.2B_Sample Review of 9 CiFar Image.png')
    pyplot.savefig(img_file)
    pyplot.show()
    print("--- %s seconds ---" % (time.time() - start_time))

    # define model
    print("defining the model")
    model = define_model()
    summary_file1 = results_dir.joinpath('Assignment_6.2B_ModelSummary.txt')
    with open(summary_file1, 'w') as f:
        with redirect_stdout(f):
            model.summary()
    print("--- %s seconds ---" % (time.time() - start_time))

    # create data generator
    print("adding augmentation creating data generator")
    #datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True)
    train_datagen = ImageDataGenerator(rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True)
    test_datagen = ImageDataGenerator(rescale=1. / 255)
    print("--- %s seconds ---" % (time.time() - start_time))

    # prepare iterator
    print("preparing iterator")
    #it_train = datagen.flow(trainX, trainY, batch_size=64)
    train_generator = train_datagen.flow(trainX, trainY, batch_size=64)
    validation_generator = test_datagen.flow(trainX, trainY, batch_size=64)
    print("--- %s seconds ---" % (time.time() - start_time))

    # fit model
    print("fitting the model")
    #history = model.fit(trainX, trainY, epochs=100, batch_size=64, validation_data=(testX, testY), verbose=0)
    #history = model.fit(it_train, trainY, epochs=100, batch_size=64, validation_data=(testX, testY), verbose=0)

    nb_train_samples = 18000
    nb_validation_samples = 7000
    epochs = 20
    batch_size = 144
    history = model.fit_generator(
        train_generator,
        steps_per_epoch=nb_train_samples // batch_size,
        epochs=epochs,
        validation_data=validation_generator,
        validation_steps=nb_validation_samples // batch_size)

    #model.fit(trainX, trainY, epochs=100, batch_size=64, verbose=0)
    print("--- %s seconds ---" % (time.time() - start_time))

    # evaluate model
    print("evaluating the model")
    _, acc = model.evaluate(testX, testY, verbose=0)
    print('> %.3f' % (acc * 100.0))
    print("--- %s seconds ---" % (time.time() - start_time))

    # save model
    results_dir = Path('dsc650/assignments/assignment06/').joinpath('results')
    result_model_file = results_dir.joinpath('Assignment_6.2B_model.h5')
    model.save(result_model_file)
    print('Saved trained model at %s ' % result_model_file)

    # learning curves
    print("preparing summary diagnostics")
    summarize_diagnostics(history)
    print("--- %s seconds ---" % (time.time() - start_time))

    print("preparing confusion matrix")
    # Predict the values from the validation dataset
    Y_pred = model.predict(testX)
    # Convert predictions classes to one hot vectors
    Y_pred_classes = np.argmax(Y_pred, axis=1)
    # Convert validation observations to one hot vectors
    Y_true = np.argmax(testY, axis=1)
    confusion_mtx = confusion_matrix(Y_true, Y_pred_classes)
    # plot the confusion matrix
    plot_confusion_matrix(confusion_mtx, classes=range(10))
    print("--- %s seconds ---" % (time.time() - start_time))

    print("preparing correct vs incorrect classifications review")
    correct_indices = np.nonzero(Y_pred_classes == Y_true)[0]
    incorrect_indices = np.nonzero(Y_pred_classes != Y_true)[0]
    print(len(Y_pred_classes))
    print(len(Y_true))
    print(len(correct_indices), " classified correctly")
    print(len(incorrect_indices), " classified incorrectly")

    # adapt figure size to accomodate 18 subplots
    pyplot.rcParams['figure.figsize'] = (7, 14)
    figure_evaluation = pyplot.figure()

    # plot 9 correct predictions
    for i, correct in enumerate(correct_indices[:14]):
        pyplot.subplot(6, 3, i + 1)
        pyplot.imshow(testX[correct], cmap='gray', interpolation='none')
        pyplot.title(
            "Predicted: {}, Truth: {}".format(classes[Y_pred[correct].argmax()],
                                              classes[testY[correct].argmax()]))
        pyplot.xticks([])
        pyplot.yticks([])
    img_file = results_dir.joinpath('Assignment_6.2B_Correct Predictions Sample.png')
    pyplot.savefig(img_file)
    pyplot.show()

    # plot 9 incorrect predictions
    for i, incorrect in enumerate(incorrect_indices[:9]):
        pyplot.subplot(6, 3, i + 10)
        pyplot.imshow(testX[incorrect], cmap='gray', interpolation='none')
        pyplot.title(
            "Predicted {}, Truth: {}".format(classes[Y_pred[incorrect].argmax()],
                                             classes[testY[incorrect].argmax()]))
        pyplot.xticks([])
        pyplot.yticks([])

    img_file = results_dir.joinpath('Assignment_6.2B_Incorrect Predictions.png')
    pyplot.savefig(img_file)
    pyplot.show()
    print("--- %s seconds ---" % (time.time() - start_time))

def run_example_prediction():
    # predict the class
    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
    print("Attempting load model to predict based on image: sample_image-1.png")
    results_dir = Path('C:/Users/aland/class/DSC650/dsc650/dsc650/assignments/assignment06/').joinpath('results')
    result_model_file = results_dir.joinpath('Assignment_6.2B_model.h5')
    model = load_model(result_model_file)
    summary_file1 = results_dir.joinpath('Assignment_6.2B_ModelSummary_AfterLoad.txt')
    with open(summary_file1, 'w') as f:
        with redirect_stdout(f):
            model.summary()

    data_dir = Path('dsc650/assignments/assignment06/').joinpath('Data')
    filename = data_dir.joinpath('sample_image-1.png')
    img = load_image(filename)

    result = model.predict_classes(img)
    print("The picture prediction is:......")
    print(classes[result[0]])
    print("--- %s seconds ---" % (time.time() - start_time))

# entry point, run the test harness
run_test_harness()

run_example_prediction()


"""SendWater URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from SendWaterBG import views

urlpatterns = [

    path('admin', admin.site.urls),
    path('creatuser', views.creatUser),
    path('committable', views.commitTable),
    path('getallshop', views.getAllShop),
    path('getoneshop', views.getOneShop),
    path('gethistory', views.getMyHistory),
    path('login', views.login),
    path('addofferman', views.addOfferMan),
    path('addadmin', views.addAdmin),
    path('adminlogin', views.adminLogin),
    path('getmycustomers', views.getMyCustomers),
    path('getallcommittable', views.getAllCommitTable)

]


import stft
import cluster
import numpy as np

import matplotlib.pyplot as plt

if __name__ == "__main__":

    in_dir = "/Users/bergamaschi/Documents/HumpbackSong/dataset/"
    stft.main(in_dir=in_dir)

    k_vals = np.logspace(1, 4, 10, base=10, dtype=int)

    inertia_scores = []

    for k in k_vals:
        inertia = cluster.main(k=k, return_inertia=True)
        inertia_scores.append(inertia)

    fig, ax = plt.subplots(figsize=(5, 5))

    print(k_vals)
    print(inertia_scores)

    plt.plot(k_vals, inertia_scores, marker="x")
    plt.title("Inertia vs. K Value")
    plt.xlabel("K Value")
    plt.ylabel("Inertia")
    plt.show()


import pandas as pd
from sklearn import ensemble
from sklearn.feature_extraction import DictVectorizer
import locale
import datetime
import numpy as np
import matplotlib.pyplot as plt
import pylab as py
from sklearn.metrics import r2_score
from scipy.stats import spearmanr, pearsonr
from sklearn.cross_validation import train_test_split
# xl = pd.ExcelFile('C:\\Users\\Chetan.Chougle\\Desktop\\Book1.xlsx')
# df = xl.parse("Sheet1")
# print(df.head())
def score_eval(regressor , X_train ,y_train):
    print('evaluating scores')
    print(X_train)
    print(y_train)
    scores = []
    estimators = np.arange(10, 200, 10)
    for n in estimators:
        regressor.set_params(n_estimators=100)
        regressor.fit(X_train, y_train)
        scores.append(regressor.score([[12, 2017]], [10040]))
        print("sdhfsdjhfeii\n")
        print(regressor.score([[12, 2017]], [10040]  ))
    py.title("Effect of n_estimators")
    py.xlabel("n_estimator")
    py.ylabel("score")
    py.plot(estimators, scores)
    py.show()

def r2_eval2(rf,X_train, X_test, y_train, y_test):
    predicted_train = rf.predict(X_train)
    predicted_test = rf.predict(X_test)
    test_score = r2_score(y_test, predicted_test)
    spearman = spearmanr(y_test, predicted_test)
    pearson = pearsonr(y_test, predicted_test)
    print(test_score)
    print('Out-of-bag R-2 score estimate: '+ str(rf.oob_score_))
    print('Test data R-2 score: '+ str(test_score))
    print('Test data Spearman correlation: '+ str(spearman[0]) )
    print('Test data Pearson correlation: '+ str(pearson[0]) )

def predict_heuristic(previous_predict, month_predict, actual_previous_value, actual_value):
    """ Heuristic that tries to mark the deviance from real and prediction as
        suspicious or not.
    """
    if (
         actual_value < month_predict and
         abs((actual_value - month_predict) / month_predict) > .3):
        if (
             actual_previous_value < previous_predict and
             abs((previous_predict - actual_previous_value) / actual_previous_value) > .3):
            return False
        else:
            return True
    else:
        return False

def get_dataframe():
    xl = pd.ExcelFile('D:\\customerinsight\\Book2.xlsx')
    # df = xl.parse("Sheet1")
    # rows = df

    df = pd.DataFrame.from_records(
        xl.parse("Sheet1"),
        columns=['CustomerName', 'Sales', 'Month', 'Year','PreviousMonthSales']
    )
    #df["CustomerName"] = df["CustomerName"].astype('category')
    # print(df)
    #print(df['CustomerName'].unique().tolist())
    return df

def missed_customers():
    """ Returns a list of tuples of the customer name, the prediction, and
        the actual amount that the customer has bought.
    """

    raw = get_dataframe()
    vec = DictVectorizer()
    today = datetime.date.today()
    currentMonth = today.month
    currentYear = today.year
    lastMonth = (today.replace(day=1) - datetime.timedelta(days=1)).month
    lastMonthYear = (today.replace(day=1) - datetime.timedelta(days=1)).year
    results = []

    # Exclude this month's value
    #df = raw.loc[(raw['Month'] != currentMonth) & (raw['Year'] != currentYear)]
    df = raw
    print('aa')
    #print(df['CustomerName'].unique())
    #print(df['CustomerName'].unique().tolist())
    for customer in set(df['CustomerName'].unique().tolist()):
        # compare this month's real value to the prediction
        actual_value = 0.0
        actual_previous_value = 0.0
        # print("here2")
        # Get the actual_value and actual_previous_value
        # print("sddjd")
        # print(raw.loc[(raw['CustomerName'] == customer) & (raw['Year'] ==currentYear ) ]['Sales'])
        # print("sdfs")
        # new_raw = raw.loc[(raw['CustomerName'] == customer)     , 'Sales']
        # new_raw2 = new_raw.loc[(raw['Year'] == currentYear)   ]
        # print( new_raw.iloc[0] )
        # print( raw.loc[(raw['CustomerName'] == customer )['Sales']])
        # print("\n")
        # print("Current year")
        # print(currentYear)
        print("currentMonth")
        print(currentMonth)
        # print("last month")
        # print(lastMonth)
        # print("lastMonthYear")
        # print(lastMonthYear)
        print("sales")
        # print(raw.loc[
        #     (raw['CustomerName'] == customer) &
        #     (raw['Year'].astype(float) == currentYear) &
        #     (raw['Month'].astype(float) == currentMonth)
        # ]['Sales'])
        #
        # print(float(pd.to_numeric( raw.loc[
        #     (raw['CustomerName'] == customer) &
        #     (raw['Year'].astype(float) == currentYear) &
        #     (raw['Month'].astype(float) == currentMonth)
        # ]['Sales'])))
        # actual_previous_value = float(raw.loc[    (raw['CustomerName'] == customer) &
        #     (raw['Year'].astype(float) == currentYear )  & (raw['Month'] == int(currentMonth)) ]['Sales'])
        # print(actual_previous_value)
        # print('before me')
        try:
            actual_previous_value = float(
                raw.loc[
                    (raw['CustomerName'] == customer) &
                    (raw['Year'] == currentYear) &
                    (raw['Month'] == currentMonth)
                ]['Sales']
            )
            actual_value = float(
                raw[
                    (raw['CustomerName'] == customer) &
                    (raw['Year'] == lastMonthYear) &
                    (raw['Month'] == lastMonth)
                ]['Sales']
            )
        except TypeError:
            # If the customer had no sales in the target month, then move on
            continue

        # Transforming Data
        print('Data')
        print(actual_previous_value)
        print(actual_value)
        print('before me')
        temp = df.loc[df['CustomerName'] == customer]
        targets = temp['Sales']

        del temp['CustomerName']
        del temp['Sales']
        del temp['PreviousMonthSales']
        #print(temp)
        #print(targets)
        X_train, X_test, y_train, y_test = train_test_split(temp, targets, train_size=0.8, random_state=42)
        records = temp.to_dict(orient="records")
        vec_data = vec.fit_transform(records).toarray()
        print('\ntemp\n')
        #print(temp)
        #print(records)
        #print(vec_data)
        #print(targets)
        # Fitting the regressor, and use all available cores
        regressor = ensemble.RandomForestRegressor(n_jobs=-1 , oob_score=True , max_features=0.33)
        regressor.fit(vec_data, targets)
        #score_eval(regressor ,vec_data , targets )
        print('CustomerName ')
        print(customer)
        r2_eval2(regressor ,X_train, X_test, y_train, y_test)

        # Predict the past two months using the regressor
        previous_predict = regressor.predict(vec.transform({
            'Year': lastMonthYear,
            'Month': lastMonth
        }).toarray())[0]
        month_predict = regressor.predict(vec.transform({
            'Year': currentYear,
            'Month': currentMonth
        }).toarray())[0]
        print('bb')
        print(previous_predict)
        print('cc')
        print(month_predict)
        if (predict_heuristic(previous_predict, month_predict, actual_previous_value, actual_value)):
            results.append((
                customer,
                month_predict,
                actual_previous_value
            ))

    return results

if __name__ == '__main__':
    locale.setlocale(locale.LC_ALL, '')
    customers = missed_customers()
    print("here")
    # print(customers)
    for customer in  set(customers):
        print("{} was predicted to buy around {}, they bought only {}".format(
            customer[0],
            locale.currency(customer[1], grouping=True),
            locale.currency(customer[2], grouping=True)
        ))


# -*- coding:utf-8 -*-
"""数据增强
   1. 翻转变换 flip
   2. 随机修剪 random crop
   3. 色彩抖动 color jittering
   4. 平移变换 shift
   5. 尺度变换 scale
   6. 对比度变换 contrast
   7. 噪声扰动 noise
   8. 旋转变换/反射变换 Rotation/reflection
"""

from PIL import Image, ImageEnhance, ImageOps, ImageFile
import numpy as np
import random
import threading, os, time
import logging

logger = logging.getLogger(__name__)
ImageFile.LOAD_TRUNCATED_IMAGES = True

class DataAugmentation:
    """
    包含数据增强的八种方式
    """

    def __init__(self):
        pass

    @staticmethod
    def openImage(image):
        return Image.open(image, mode="r")

    @staticmethod
    def FR_flip(image, label, dsm):
        """
         将图像和label进行左右翻转
        :param image PIL的图像image
        :return: 翻转之后的图像和label
        """
        return image.transpose(Image.FLIP_LEFT_RIGHT), label.transpose(Image.FLIP_LEFT_RIGHT), dsm.transpose(Image.FLIP_LEFT_RIGHT)

    @staticmethod
    def TB_flip(image,label,dsm):
        """
         将图像和label进行上下翻转
        :param image PIL的图像image
        :return: 翻转之后的图像和label
        """
        return image.transpose(Image.FLIP_TOP_BOTTOM), label.transpose(Image.FLIP_TOP_BOTTOM), dsm.transpose(Image.FLIP_TOP_BOTTOM)

    @staticmethod
    def randomRotation(image, label, dsm, mode=Image.BICUBIC):
        """
         对图像进行随机任意角度(0~360度)旋转
        :param mode 邻近插值,双线性插值,双三次B样条插值(default)
        :param image PIL的图像image
        :return: 旋转转之后的图像
        """
        random_angle = np.random.randint(1, 360)
        return image.rotate(random_angle, mode), label.rotate(random_angle, Image.NEAREST), dsm.rotate(random_angle, Image.NEAREST)

    @staticmethod
    def randomColor(image, label, dsm):
        """
        对图像进行颜色抖动
        :param image: PIL的图像image
        :return: 有颜色色差的图像image
        """
        random_factor = np.random.randint(0, 31) / 10.  # 随机因子
        color_image = ImageEnhance.Color(image).enhance(random_factor)  # 调整图像的饱和度
        random_factor = np.random.randint(10, 21) / 10.  # 随机因子
        brightness_image = ImageEnhance.Brightness(color_image).enhance(random_factor)  # 调整图像的亮度
        random_factor = np.random.randint(10, 21) / 10.  # 随机因1子
        contrast_image = ImageEnhance.Contrast(brightness_image).enhance(random_factor)  # 调整图像对比度
        random_factor = np.random.randint(0, 31) / 10.  # 随机因子
        return ImageEnhance.Sharpness(contrast_image).enhance(random_factor), label, dsm  # 调整图像锐度

    @staticmethod
    def saveImage(image, path):
        image.save(path)

def makeDir(path):
    try:
        if not os.path.exists(path):
            if not os.path.isfile(path):
                # os.mkdir(path)
                os.makedirs(path)
            return 0
        else:
            return 1
    except Exception as e:
        print(str(e))
        return -2

def imageOps(func_name, image, label, dsm_data, img_des_path, label_des_path, dsm_data_des_path, img_file_name, label_file_name, dsm_data_file_name,
    times=1):
    funcMap = {"randomRotation": DataAugmentation.randomRotation}
    if funcMap.get(func_name) is None:
        logger.error("%s is not exist", func_name)
        return -1

    for _i in range(0, times, 1):
        new_image, new_label, new_dsm = funcMap[func_name](image, label, dsm_data)
        DataAugmentation.saveImage(new_image, os.path.join(img_des_path, func_name + str(_i) + img_file_name))
        DataAugmentation.saveImage(new_label, os.path.join(label_des_path, func_name + str(_i) + label_file_name))
        DataAugmentation.saveImage(new_dsm, os.path.join(dsm_data_des_path, func_name + str(_i) + dsm_data_file_name))

#opsList = {"randomRotation","randomColor", "randomGaussian"}
opsList = {"randomRotation"}
#opsList = {"FR_flip","TB_flip"}

def threadOPS(img_path, new_img_path, label_path, new_label_path, dsm_data_path, new_dsm_data_path):
    """
    多线程处理事务
    :param src_path: 资源文件
    :param des_path: 目的地文件
    :return:
    """
    # img path
    if os.path.isdir(img_path):
        img_names = os.listdir(img_path)
    else:
        img_names = [img_path]

    # label path
    if os.path.isdir(label_path):
        label_names = os.listdir(label_path)
    else:
        label_names = [label_path]

    if os.path.isdir(dsm_data_path):
        dsm_data_names = os.listdir(dsm_data_path)
    else:
        dsm_data_names = [dsm_data_path]

    img_num = 0
    label_num = 0
    dsm_data_num = 0

    # img num
    for img_name in img_names:
        tmp_img_name = os.path.join(img_path, img_name)
        if os.path.isdir(tmp_img_name):
            print('contain file folder')
            exit()
        else:
            img_num = img_num + 1;
    # label num
    for label_name in label_names:
        tmp_label_name = os.path.join(label_path, label_name)
        if os.path.isdir(tmp_label_name):
            print('contain file folder')
            exit()
        else:
            label_num = label_num + 1

    for dsm_data_name in dsm_data_names:
        tmp_dsm_data_name = os.path.join(dsm_data_path, dsm_data_name)
        if os.path.isdir(tmp_dsm_data_name):
            print('contain file folder')
            exit()
        else:
            dsm_data_num = dsm_data_num + 1

    if img_num != label_num != dsm_data_num:
        print('the num of img and label and dsm_data is not equl')
        exit()
    else:
        num = img_num

    for i in range(num):
        img_name = img_names[i]
        print('img:'+ img_name)
        label_name = label_names[i]
        print('label:' + label_name)
        dsm_data_name = dsm_data_names[i]
        print('dsm:' + dsm_data_name)

        tmp_img_name = os.path.join(img_path, img_name)
        tmp_label_name = os.path.join(label_path, label_name)
        tmp_dsm_data_name = os.path.join(dsm_data_path, dsm_data_name)

        # 读取文件并进行操作
        image = DataAugmentation.openImage(tmp_img_name)
        image.load()
        label = DataAugmentation.openImage(tmp_label_name)
        image.load()
        dsm_data = DataAugmentation.openImage(tmp_dsm_data_name)
        image.load()
        threadImage = [0] * 5
        _index = 0
        for ops_name in opsList:
            threadImage[_index] = threading.Thread(target=imageOps,
                                                   args=(ops_name, image, label, dsm_data, new_img_path, new_label_path, new_dsm_data_path, img_name,
                                                         label_name, dsm_data_name))
            threadImage[_index].start()
            _index += 1
            time.sleep(0.2)

if __name__ == '__main__':
    threadOPS("data/dataset4/road_car_images_292/",
              "data/dataset4/road_car_images_292_aug/",
              "data/dataset4/road_car_anno_292/",
              "data/dataset4/road_car_anno_292_aug/",
              "data/dataset4/dsm_292",
              "data/dataset4/road_car_dsm_292_aug"
              )


import Models
import keras
import numpy as np
import pandas as pd

# df = pd.read_csv("/home/amithbn/Desktop/TABL/fi_2010_dataset/FI2010_train.csv")
# arr = df["148"].values

# n = len(df)

# n /= 10
# n = int(n)
# arr = arr[0:n]

# up = pd.DataFrame(data={"UP":np.where(arr == 1, 1, 0)})
# dn = pd.DataFrame(data={"DN":np.where(arr == 3, 1, 0)})
# #eq = pd.DataFrame(data={"EQ":np.where(arr == 2, 1, 0)})

# df = df.iloc[:, :40]
# lst = np.array_split(df,n)
# lst = np.array(lst)

# lbls = up.join(dn)
# #lbls = lbls.join(eq)

# 1 hidden layer network with input: 40x10, hidden 120x5, output 3x1
# template = [[10,40], [120,5], [2,1]]
template = [[40,10], [120,5], [3,1]]

x = np.random.rand(1000,40,10)
y = keras.utils.to_categorical(np.random.randint(0,3,(1000,)),3)

# random data
#y = keras.utils.to_categorical(np.random.randint(0,3,(n,)),3) # - low, high,

# get Bilinear model
projection_regularizer = None
projection_constraint = keras.constraints.max_norm(3.0,axis=0)
attention_regularizer = None
attention_constraint = keras.constraints.max_norm(5.0, axis=1)
dropout = 0.1

model = Models.TABL(template, dropout, projection_regularizer, projection_constraint,
                    attention_regularizer, attention_constraint)
model.summary()

# create class weight
class_weight = {0 : 1e6/300.0,
                1 : 1e6/400.0,
                2 : 1e6/300.0}

# training
model.fit(x,y, batch_size=256, epochs=100, class_weight=class_weight)

# model.fit(lst,lbls, batch_size=256, epochs=10, class_weight=class_weight)
# model.save('tabl_model.h5')


# Acoustic wave equations with Devito
# Forward/adjoint nonlinear and Born modeling
# Authors: Mathias Louboutin, Philipp Witte
# Date: November 2017
#

# Import modules
from __future__ import print_function
import numpy as np
import gc, os, psutil
from numpy.random import randint
from sympy import solve, cos, sin
from sympy import Function as fint
from devito.logger import set_log_level
from devito import Eq, Function, TimeFunction, Dimension, Operator, clear_cache, first_derivative
from PySource import PointSource, Receiver
from PyModel import Model
from checkpoint import DevitoCheckpoint, CheckpointOperator
from pyrevolve import Revolver

def forward_modeling(model, src_coords, wavelet, rec_coords, save=False, space_order=8, nb=40, op_return=False, dt=None):
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp

    # Create the forward wavefield
    if save is False:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
    else:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order, save=nt)

    # Set up PDE and rearrange
    eqn = m * u.dt2 - u.laplace + damp * u.dt
    stencil = solve(eqn, u.forward)[0]
    expression = [Eq(u.forward, stencil)]

    # Source symbol with input wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, offset=model.nbpml, expr=src * dt**2 / m)

    # Data is sampled at receiver locations
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=u, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression += src_term + rec_term
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Forward%s" % randint(1e5))
    if op_return is False:
        op(dt=dt)
        return rec.data, u
    else:
        return op

def adjoint_modeling(model, src_coords, rec_coords, rec_data, space_order=8, nb=40, dt=None):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp

    # Create the adjoint wavefield
    v = TimeFunction(name="v", grid=model.grid, time_order=2, space_order=space_order)

    # Set up PDE and rearrange
    eqn = m * v.dt2 - v.laplace - damp * v.dt
    stencil = solve(eqn, v.backward)[0]
    expression = [Eq(v.backward, stencil)]

    # Adjoint source is injected at receiver locations
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec.data[:] = rec_data[:]
    adj_src = rec.inject(field=v.backward, offset=model.nbpml, expr=rec * dt**2 / m)

    # Data is sampled at source locations
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    adj_rec = src.interpolate(expr=v, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression += adj_src + adj_rec
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Backward%s" % randint(1e5))
    op(dt=dt)

    return src.data

def forward_born(model, src_coords, wavelet, rec_coords, space_order=8, nb=40, isic=False, dt=None):
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, dm, damp = model.m, model.dm, model.damp

    # Create the forward and linearized wavefield
    u = TimeFunction(name="u", grid=model.grid, time_order=2, space_order=space_order)
    du = TimeFunction(name="du", grid=model.grid, time_order=2, space_order=space_order)
    if len(model.shape) == 2:
        x,y = u.space_dimensions
    else:
        x,y,z = u.space_dimensions

    # Set up PDEs and rearrange
    eqn = m * u.dt2 - u.laplace + damp * u.dt
    stencil1 = solve(eqn, u.forward)[0]
    if isic is not True:
        eqn_lin = m * du.dt2 - du.laplace + damp * du.dt + dm * u.dt2   # born modeling
    else:
        du_aux = sum([first_derivative(first_derivative(u, dim=d, order=space_order//2) * dm, order=space_order//2, dim=d) for d in u.space_dimensions])
        eqn_lin = m * du.dt2 - du.laplace + damp * du.dt + (dm * u.dt2 * m - du_aux)

    if isic is not True:
        stencil2 = solve(eqn_lin, du.forward)[0]
    else:
        stencil2 = solve(eqn_lin, du.forward, simplify=False, rational=False)[0]

    expression_u = [Eq(u.forward, stencil1)]
    expression_du = [Eq(du.forward, stencil2)]

    # Define source symbol with wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, offset=model.nbpml, expr=src * dt**2 / m)

    # Define receiver symbol
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=du, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression = expression_u + src_term + expression_du + rec_term
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Born%s" % randint(1e5))
    op(dt=dt)

    return rec.data

def adjoint_born(model, rec_coords, rec_data, u=None, op_forward=None, is_residual=False, space_order=8, nb=40, isic=False, dt=None):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp

    # Create adjoint wavefield and gradient
    v = TimeFunction(name='v', grid=model.grid, time_order=2, space_order=space_order)
    gradient = Function(name='gradient', grid=model.grid)

    # Set up PDE and rearrange
    eqn = m * v.dt2 - v.laplace - damp * v.dt
    stencil = solve(eqn, v.backward)[0]
    expression = [Eq(v.backward, stencil)]

    # Data at receiver locations as adjoint source
    rec_g = Receiver(name='rec_g', grid=model.grid, ntime=nt, coordinates=rec_coords)
    if op_forward is None:
        rec_g.data[:] = rec_data[:]
    adj_src = rec_g.inject(field=v.backward, offset=model.nbpml, expr=rec_g * dt**2 / m)

    # Gradient update
    if u is None:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)

    if isic is not True:
        gradient_update = [Eq(gradient, gradient - u * v.dt2)]  # zero-lag cross-correlation imaging condition
    else:
        # linearized inverse scattering imaging condition (Op't Root et al. 2010; Whitmore and Crawley 2012)
        if len(model.shape) == 2:
            gradient_update = [Eq(gradient, gradient - (u * v.dt2 * m + u.dx * v.dx + u.dy * v.dy))]
        else:
            gradient_update = [Eq(gradient, gradient - (u * v.dt2 * m + u.dx * v.dx + u.dy * v.dy + u.dz * v.dz))]

    # Create operator and run
    set_log_level('ERROR')
    expression += adj_src + gradient_update
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Gradient%s" % randint(1e5))

    # Optimal checkpointing
    if op_forward is not None:
        rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
        cp = DevitoCheckpoint([u])
        n_checkpoints = None
        wrap_fw = CheckpointOperator(op_forward, u=u, m=model.m.data, rec=rec, dt=dt)
        wrap_rev = CheckpointOperator(op, u=u, v=v, m=model.m.data, rec_g=rec_g, dt=dt)

        # Run forward
        wrp = Revolver(cp, wrap_fw, wrap_rev, n_checkpoints, nt-2)
        wrp.apply_forward()

        # Residual and gradient
        if is_residual is True:  # input data is already the residual
            rec_g.data[:] = rec_data[:]
        else:
            rec_g.data[:] = rec.data[:] - rec_data[:]   # input is observed data
            fval = .5*np.linalg.norm(rec_g.data[:])**2
        wrp.apply_reverse()
    else:
        op(dt=dt)
    clear_cache()

    if op_forward is not None and is_residual is not True:
        return fval, gradient.data
    else:
        return gradient.data

########################################################################################################################

def forward_freq_modeling(model, src_coords, wavelet, rec_coords, freq, space_order=8, nb=40, dt=None):
    # Forward modeling with on-the-fly DFT of forward wavefields
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp
    freq_dim = Dimension(name='freq_dim')
    time = model.grid.time_dim

    # Create wavefields
    nfreq = freq.shape[0]
    u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
    f = Function(name='f', dimensions=(freq_dim,), shape=(nfreq,))
    f.data[:] = freq[:]
    ufr = Function(name='ufr', dimensions=(freq_dim,) + u.indices[1:], shape=(nfreq,) + model.shape_domain)
    ufi = Function(name='ufi', dimensions=(freq_dim,) + u.indices[1:], shape=(nfreq,) + model.shape_domain)

    # Set up PDE and rearrange
    eqn = m * u.dt2 - u.laplace + damp * u.dt
    stencil = solve(eqn, u.forward)[0]
    expression = [Eq(u.forward, stencil)]
    expression += [Eq(ufr, ufr + u*cos(2*np.pi*f*time*dt))]
    expression += [Eq(ufi, ufi - u*sin(2*np.pi*f*time*dt))]

    # Source symbol with input wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, offset=model.nbpml, expr=src * dt**2 / m)

    # Data is sampled at receiver locations
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=u, offset=model.nbpml)

    # Create operator and run
    set_log_level('ERROR')
    expression += src_term + rec_term
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Forward%s" % randint(1e5))
    op(dt=dt)

    return rec.data, ufr, ufi

def adjoint_freq_born(model, rec_coords, rec_data, freq, ufr, ufi, space_order=8, nb=40, dt=None):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, damp = model.m, model.damp
    nfreq = ufr.shape[0]
    time = model.grid.time_dim

    # Create the forward and adjoint wavefield
    v = TimeFunction(name='v', grid=model.grid, time_order=2, space_order=space_order)
    f = Function(name='f', dimensions=(ufr.indices[0],), shape=(nfreq,))
    f.data[:] = freq[:]
    gradient = Function(name="gradient", grid=model.grid)

    # Set up PDE and rearrange
    eqn = m * v.dt2 - v.laplace - damp * v.dt
    stencil = solve(eqn, v.backward)[0]
    expression = [Eq(v.backward, stencil)]

    # Data at receiver locations as adjoint source
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec.data[:] = rec_data[:]
    adj_src = rec.inject(field=v.backward, offset=model.nbpml, expr=rec * dt**2 / m)

    # Gradient update
    gradient_update = [Eq(gradient, gradient + (2*np.pi*f)**2/nt*(ufr*cos(2*np.pi*f*time*dt) - ufi*sin(2*np.pi*f*time*dt))*v)]

    # Create operator and run
    set_log_level('ERROR')
    expression += adj_src + gradient_update
    op = Operator(expression, subs=model.spacing_map, dse='advanced', dle='advanced',
                  name="Gradient%s" % randint(1e5))
    op(dt=dt)
    clear_cache()

    return gradient.data


import pickle
# import apriori
#
# dataset = apriori.load_dataset('feedback_csr.pkl')
#
# L, support_data = apriori.apriori(dataset, minsupport=0.02)
#
# op = open('tmp.txt', 'wb')
# op.write(str(apriori.generateRules(L, support_data)))

import scipy.sparse as sp
import pprint

clusters ={}
data = pickle.load(open('feedback_csr.pkl', 'rb'))
user_map = {v: k for k, v in pickle.load(open('user_map.pkl', 'rb')).iteritems()}

comps = sp.cs_graph_components(data)[1]

for i in range(len(comps)):
    if comps[i] in clusters:
        clusters[comps[i]].add(user_map[i])
    else:
        clusters[comps[i]] = {user_map[i]}
clusters.pop(-2)
pprint.pprint(clusters)
open('graph_networks.txt', 'wb').write(pprint.pformat(clusters))


# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Learning 2 Learn training."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

from six.moves import xrange
import tensorflow as tf

from tensorflow.contrib.learn.python.learn import monitored_session as ms

import meta
import util
import cPickle as pickle

flags = tf.flags
logging = tf.logging

FLAGS = flags.FLAGS
flags.DEFINE_string("save_path", None, "Path for saved meta-optimizer.")
flags.DEFINE_integer("num_epochs", 10000, "Number of training epochs.")
flags.DEFINE_integer("log_period", 100, "Log period.")
flags.DEFINE_integer("evaluation_period", 10, "Evaluation period.")
flags.DEFINE_integer("evaluation_epochs", 20, "Number of evaluation epochs.")
flags.DEFINE_integer("seed", 1234, "Seed for TensorFlow's RNG.")

flags.DEFINE_string("problem", "simple", "Type of problem.")
flags.DEFINE_integer("num_steps", 100,
                     "Number of optimization steps per epoch.")
flags.DEFINE_integer("unroll_length", 20, "Meta-optimizer unroll length.")
flags.DEFINE_float("learning_rate", 0.001, "Learning rate.")
flags.DEFINE_boolean("second_derivatives", False, "Use second derivatives.")
flags.DEFINE_boolean("load_trained_model", False, "Load trained model.")
flags.DEFINE_string("model_path", "exp/model", "Trained model path.")

def main(_):
  # Configuration.
  if FLAGS.seed:
    tf.set_random_seed(FLAGS.seed)

  num_unrolls = FLAGS.num_steps // FLAGS.unroll_length

  if FLAGS.seed:
    tf.set_random_seed(FLAGS.seed)

  if FLAGS.save_path is not None:
    if os.path.exists(FLAGS.save_path):
      # raise ValueError("Folder {} already exists".format(FLAGS.save_path))
      pass
    else:
      os.mkdir(FLAGS.save_path)

  # Problem.
  problem, net_config, net_assignments = util.get_config(FLAGS.problem)
  loss_op = problem()

  # Optimizer setup.
  optimizer = meta.MetaOptimizer(**net_config)
  minimize = optimizer.meta_minimize(
      problem, FLAGS.unroll_length,
      learning_rate=FLAGS.learning_rate,
      net_assignments=net_assignments,
      second_derivatives=FLAGS.second_derivatives)
  step, update, reset, cost_op, _ = minimize

  if FLAGS.problem == "mnist":
    var_name_mlp = [
        "mlp/linear_0/w:0", "mlp/linear_0/b:0", "mlp/linear_1/w:0",
        "mlp/linear_1/b:0"
    ]
  else:
    var_name_mlp = []

  problem_vars = tf.get_collection(tf.GraphKeys.VARIABLES)

  if var_name_mlp:
    saver_vars = [vv for vv in problem_vars if vv.name in var_name_mlp]
  else:
    saver_vars = problem_vars

  saver = tf.train.Saver(saver_vars)

  process_id = os.getpid()
  exp_folder = os.path.join(FLAGS.save_path, str(process_id))
  writer = tf.summary.FileWriter(exp_folder)

  if not os.path.isdir(exp_folder):
    os.mkdir(exp_folder)

  with ms.MonitoredSession() as sess:
    # a quick hack!
    regular_sess = sess._sess._sess._sess._sess

    # Prevent accidental changes to the graph.
    tf.get_default_graph().finalize()

    # print("Initial loss = {}".format(sess.run(loss_op)))
    # raw_input("wait")

    if FLAGS.load_trained_model == True:
      print("We are loading trained model here!")
      saver.restore(regular_sess, FLAGS.model_path)

    # init_state = regular_sess.run(optimizer.init_state)
    # pickle.dump(init_state, open("init_state.p", "wb"))

    best_evaluation = float("inf")
    total_time = 0
    total_cost = 0
    for e in xrange(FLAGS.num_epochs):
      # Training.
      time, cost = util.run_epoch(sess, cost_op, [update, step], reset,
                                  num_unrolls, e, writer)
      total_time += time
      total_cost += cost
      writer.flush()

      # Logging.
      if (e + 1) % FLAGS.log_period == 0:
        util.print_stats("Epoch {}".format(e + 1), total_cost, total_time,
                         FLAGS.log_period)
        total_time = 0
        total_cost = 0

      # Evaluation.
      if (e + 1) % FLAGS.evaluation_period == 0 or e == 0:
        eval_cost = 0
        eval_time = 0
        for _ in xrange(FLAGS.evaluation_epochs):
          time, cost = util.run_epoch_val(sess, cost_op, [update], reset,
                                      num_unrolls, e, writer)

          eval_time += time
          eval_cost += cost

        util.print_stats("EVALUATION", eval_cost, eval_time,
                         FLAGS.evaluation_epochs)

        if FLAGS.save_path is not None and eval_cost < best_evaluation:
          # print("Removing previously saved meta-optimizer")
          # for f in os.listdir(FLAGS.save_path):
          #   os.remove(os.path.join(FLAGS.save_path, f))
          # print("Saving meta-optimizer to {}".format(FLAGS.save_path))
          # optimizer.save(sess, FLAGS.save_path)
          optimizer.save(sess, exp_folder, e+1)
          best_evaluation = eval_cost

if __name__ == "__main__":
  tf.app.run()


import boto3
import os

ACCESS_KEY = '*'
SECRET_ACCESS_KEY = '*'

client = boto3.client(
    's3',
    aws_access_key_id=ACCESS_KEY,
    aws_secret_access_key=SECRET_ACCESS_KEY
)

print(vars(client))

upload_file_bucket = 'upload-file-bucket-jpd'
for file in os.listdir():
    if '.py' in file:
        upload_file_key = 'fold1/' + str(file)
        client.upload_file(file, upload_file_bucket, upload_file_key)


# - 다중 클래스 이미지 분류를 금방 개발 가능하도록 템플릿을 작성

# - 클래스 수, 특징 추출 모델, 입력층 등을 쉽게 커스텀 및 업그레이드 가능하도록 하고, 고도화.

# - 사용자 입장에서는 사진 폴더별로 이미지만 나눠서 넣으면 쉽게 학습 가능.

# - 여기선 백본을 resnet v2 50으로 둠.

# 0. 라이브러리

# !pip install -q -U tf-hub-nightly
# 설치 및 임포트

import matplotlib.pylab as plt
import tensorflow as tf
import os
import numpy as np
import tensorflow_hub as hub
import pandas as pd
# from tensorflow.keras.preprocessing import image_dataset_from_directory
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from sklearn.model_selection import train_test_split

# 1. 데이터 준비

IMG_SIZE = (224,224)
BATCH_SIZE = 32

TEST_DIR = "LPD_competition/test/tests"
x = np.load("np/train_x.npy",allow_pickle=True)
y = np.load("np/train_y.npy",allow_pickle=True)
print(y.shape)
y = np.argmax(y, axis=-1)

x_train, x_val, y_train, y_val = train_test_split(x, y, train_size = 0.8, shuffle = True, random_state=1)

print(y_train.shape)

# train_datagen = ImageDataGenerator(
#     horizontal_flip=True,
#     vertical_flip=True,
#     width_shift_range=0.1,
#     height_shift_range=0.1,
#     fill_mode='nearest'
# )

# test_datagen = ImageDataGenerator()

# train_set = train_datagen.flow(x_train, y_train, batch_size=BATCH_SIZE)
# valid_set = train_datagen.flow(x_val, y_val, batch_size=BATCH_SIZE)

# print(len(train_set))
# print(len(valid_set))

data_augmentation = tf.keras.Sequential([
  tf.keras.layers.experimental.preprocessing.RandomFlip('horizontal'),
  tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),
])

# AUTOTUNE = tf.data.experimental.AUTOTUNE
# train_set = train_set.cache().prefetch(buffer_size=AUTOTUNE)
# valid_set = valid_set.cache().prefetch(buffer_size=AUTOTUNE)

# 먼저 이미지 분석에 있어서 없어선 안될 데이터 증강을 레이어로 만들어서 추후 모델에 포함시킬것임.

# 각 순전파마다 자동으로 데이터 증강이 되며, 매 학습마다 다른 이미지로 학습하는 결과를 만들어줌.(실전에서는 모델에 포함시키는게 아니라 학습 프로세스를 변경하여 학습시에만 이를 순전파시켜 이미지를 증강시킬것임.)

minmax_layer = tf.keras.layers.experimental.preprocessing.Rescaling(scale=1./255)
# 데이터 스케일링 레이어.

# 이미지 데이터 값의 스케일을 0~255로 할건지 0~1로 할건지를 정함.

# -1~1의 경우는
Standard_layer = tf.keras.layers.experimental.preprocessing.Rescaling(scale=1./127.5, offset=-1)

# 3. 특징 추출 모델 가져오기

# Create the base model from the pre-trained model Resnet v2 50
feature_extractor_url = "https://tfhub.dev/google/imagenet/resnet_v2_50/feature_vector/4"

IMG_SHAPE = IMG_SIZE + (3,)
base_model = hub.KerasLayer(feature_extractor_url,
                                         input_shape=IMG_SHAPE)
# 텐서플로 허브를 이용해서 CNN 특징추출 모델을 가져온 것.

# 텐서플로 허브를 사용해서 위와 같이 URL만 바꿔줘도 되는데,

# output은 확인할것.

# 위의 모델은 확인해보면 특징 추출 출력값이

# (32, 2048)

# 인데, 배치인 32를 제외하면 1x2048임.

# 즉, 만약 (32, 5, 5, 1280)같은 2d 필터 결과가 나오면, 이에 대해

global_average_layer = tf.keras.layers.GlobalAveragePooling2D()
# feature_batch_average = global_average_layer(feature_batch)
# 라는 글로벌 에버리지 풀링 레이어를 CNN 뒤에 붙여줘서, (32, 5, 5, 1280)를 (32, 1280)으로 만들어줘야함.

# 일단 처음 학습시에는 잘 학습된 CNN은 동결해야하니
base_model.trainable = False

# 4. 모델 만들기

# 이제 조립만 남음.
prediction_layer = tf.keras.layers.Dense(1000)
# 이건 이미지 분류이니, 특징 추출맵에서 나온 것을 받아들여 몇개의 클래스로 반환하는 FC레이어를 하나 만듬.
# next_layer는 일단 분류를 위해 xor문제 해결 퍼셉트론에 따라 만들었는데 노드 개수를 정확히 모르겠음. 일단 노멀하게 사용되는 128을 사용.
# 그리고 prediction레이어는 정답데이터가 0~1의 확률값이 아닌 클래스 인덱스라서 소프트맥스는 사용 안함
"""
inputs = tf.keras.Input(shape=IMG_SHAPE)
x = data_augmentation(inputs)
x = minmax_layer(x)
# x = Standard_layer(x)
x = base_model(x, training=False)
x = tf.keras.layers.Dropout(0.2)(x)
outputs = prediction_layer(x)

model = tf.keras.Model(inputs, outputs)
"""

from tensorflow.keras.layers import Conv2D,BatchNormalization,MaxPooling2D,AveragePooling2D,Dense,GlobalAveragePooling2D

inputs = tf.keras.Input(shape=IMG_SHAPE)
x = data_augmentation(inputs)
x = minmax_layer(x)
# x = Standard_layer(x)
x = Conv2D(filters=128, kernel_size=(2,2),padding='same',activation='swish')(x)
x = BatchNormalization()(x)
x = Conv2D(filters=128, kernel_size=(2,2),padding='same',activation='swish')(x)
x = BatchNormalization()(x)
x = Conv2D(filters=64, kernel_size=(2,2),padding='same',activation='swish')(x)
x = BatchNormalization()(x)
x = MaxPooling2D(pool_size=(2,2),padding='same')(x)
x = Conv2D(filters=64, kernel_size=(2,2),padding='same',activation='swish')(x)
x = BatchNormalization()(x)
x = MaxPooling2D(pool_size=(2,2),padding='same')(x)
x = GlobalAveragePooling2D()(x)
# x = base_model(x, training=False)
outputs = prediction_layer(x)

model = tf.keras.Model(inputs, outputs)
# 모두 조립하면 먼저 IMG_SHAPE만큼 입력층을 가지고, 이걸 증강레이어에 넣고, 이걸 리스케일링 레이어에 넣음.
# 여기까진 입력 형태가 변화 없음.
# 이후 특징추출레이어에 넣어주고, 여기서 (batch, 2048)의 데이터가 나올텐데, 이를 flatten할필요 없이 Dropout에 넣어주고, 최종 결정인 5클래스 출력레이어를 끝으로 하고 이 그래프를 모델로 만들어줌.

model.summary() # 로 구조를 대략 파악해도 됨

# 5. 학습
from tensorflow.keras.optimizers import Adam, Adadelta, Adamax, Adagrad
from tensorflow.keras.optimizers import RMSprop, SGD, Nadam

model.compile(optimizer="adam",
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['acc'])
# 컴파일 후 학습
es = EarlyStopping(monitor='val_acc', patience=10, mode='max')
cp = ModelCheckpoint("lotte2.h5", monitor='val_acc', save_best_only=True, mode='max')
lr = ReduceLROnPlateau(monitor='val_loss',patience=10, factor=0.5, verbose=1)
initial_epochs = 100

history = model.fit(x_train, y_train,
                    callbacks=[cp,es,lr],
                    validation_data=(x_val,y_val),
                    epochs=initial_epochs
                  )

loss, acc = model.evaluate(x_val,y_val)
print("acc : ", acc)
model.save("my_model1")

#  6. 시각화

acc = history.history['acc']
val_acc = history.history['val_acc']

loss = history.history['loss']
val_loss = history.history['val_loss']

plt.figure(figsize=(8, 8))
plt.subplot(2, 1, 1)
plt.plot(acc, label='Training acc')
plt.plot(val_acc, label='Validation acc')
plt.legend(loc='lower right')
plt.ylabel('acc')
plt.ylim([min(plt.ylim()),1])
plt.title('Training and Validation acc')

plt.subplot(2, 1, 2)
plt.plot(loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.ylabel('Cross Entropy')
plt.ylim([0,1.0])
plt.title('Training and Validation Loss')
plt.xlabel('epoch')
plt.show()

# ================== predict ==================
result = pd.read_csv("LPD_competition/sample.csv")

a = pd.DataFrame()
b = []
from tensorflow.keras.models import load_model
model= load_model("my_model1")

for _, _, filenames in os.walk(TEST_DIR):
    # Loop over the files
    for filename in range(len(filenames)):
        # Get filepath
        print(filename)
        filepath = os.path.join(TEST_DIR, str(filename) + '.jpg')
        # Get predictions
        raw_img = tf.keras.preprocessing.image.load_img(filepath, target_size=IMG_SIZE)  # Load the image
        img_array = tf.keras.preprocessing.image.img_to_array(raw_img)  # Convert to numpy array
        img_array = tf.expand_dims(img_array, 0)  # Reshaping - create batch axis
        predictions = model.predict(img_array)  # Make predictions
        b.append(np.argmax(predictions,axis=-1)[0])

    predict = pd.Series(b)  # Get the color with max probabilities

predict = pd.concat([a,predict],axis=1)
result.iloc[:,1] = predict.sort_index().values
result.to_csv('sample_resnetv250_1.csv')

# # 7. 파인튜닝
# es = EarlyStopping(monitor='val_loss', patience=10, mode='min')
# cp = ModelCheckpoint("lotte2.h5", monitor='val_loss', save_best_only=True, mode='min')
# lr = ReduceLROnPlateau(monitor='val_loss',patience=10, factor=0.5, verbose=1)

# base_model.trainable = True
# fine_tune_epochs = 100
# total_epochs =  initial_epochs + fine_tune_epochs

# history_fine = model.fit(train_set,
#                          callbacks=[cp,es,lr],
#                          epochs=total_epochs,
#                          initial_epoch=history.epoch[-1],
#                          validation_data=valid_set)
# model.save("my_model2")

# acc += history_fine.history['acc']
# val_acc += history_fine.history['val_acc']

# loss += history_fine.history['loss']
# val_loss += history_fine.history['val_loss']
# plt.figure(figsize=(8, 8))
# plt.subplot(2, 1, 1)
# plt.plot(acc, label='Training acc')
# plt.plot(val_acc, label='Validation acc')
# plt.ylim([0.8, 1])
# plt.plot([initial_epochs-1,initial_epochs-1],
#           plt.ylim(), label='Start Fine Tuning')
# plt.legend(loc='lower right')
# plt.title('Training and Validation acc')

# plt.subplot(2, 1, 2)
# plt.plot(loss, label='Training Loss')
# plt.plot(val_loss, label='Validation Loss')
# plt.ylim([0, 1.0])
# plt.plot([initial_epochs-1,initial_epochs-1],
#          plt.ylim(), label='Start Fine Tuning')
# plt.legend(loc='upper right')
# plt.title('Training and Validation Loss')
# plt.xlabel('epoch')
# plt.show()

# # ================== predict ==================
# result = pd.read_csv("LPD_competition/sample.csv")

# a = pd.DataFrame()
# b = []

# for _, _, filenames in os.walk(TEST_DIR):
#     # Loop over the files
#     for filename in range(len(filenames)):
#         # Get filepath
#         print(filename)
#         filepath = os.path.join(TEST_DIR, str(filename) + '.jpg')
#         # Get predictions
#         raw_img = tf.keras.preprocessing.image.load_img(filepath, target_size=IMG_SIZE)  # Load the image
#         img_array = tf.keras.preprocessing.image.img_to_array(raw_img)  # Convert to numpy array
#         img_array = tf.expand_dims(img_array, 0)  # Reshaping - create batch axis
#         predictions = model.predict(img_array)  # Make predictions
#         b.append(np.argmax(predictions,axis=-1)[0])

#     predict = pd.Series(b)  # Get the color with max probabilities

# predict = pd.concat([a,predict],axis=1)
# result.iloc[:,1] = predict.sort_index().values
# result.to_csv('sample_resnetv250_2.csv')


import nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
from nltk.corpus import stopwords
from nltk import word_tokenize
import string
from nltk.stem.wordnet import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()
punct = set(string.punctuation)
stop_word_list = set(stopwords.words('english'))

def nlp_pipeline(text_list):
    '''
    this pipeline aims to do a series of preprocessing steps:
    tokenizing, punctuations and numbers removal, lemmatization
    :param:text_list
    :type: a list of str
    :return: a list of preprocessed str
    '''
    string_list = []
    for text in text_list:
        text = text.lower()
        text = word_tokenize(text)
        text = [token for token in text if token not in string.punctuation and token.isalpha()]
        text = [token for token in text if token not in stop_word_list]
        text = [lemmatizer.lemmatize(token) for token in text]
        wordstring = ' '.join(text)
        string_list.append(wordstring)
    return string_list


# !/usr/bin/env python
# -*- coding: utf-8 -*-
# author: wangzq <wangzhenqing1008@163.com>

from fixture import SQLAlchemyFixture, DataSet
from oslo_db import options
from oslo_config import cfg
from testtools import TestCase
from qg.db import api as db_api

from itdblib.dal.asset_consume_info_dal import AssetConsumeInfoDal
from itdblib.models.tb_asset_consume_info import AssetConsumeInfoModel
from itdblib.models.base import BASE
import fixtures

CONF = cfg.CONF
CONF.register_opts(options.database_opts, 'database')

class AssetConsumeInfoMock(DataSet):

    class AssetConsumeInfoModel():
        id=1
        asset_type_id = 1
        model_id = 1
        user_name = "abc"
        store_place_id = 1
        store_state_id = 1
        device_state_id = 1
        in_num = 0
        out_num = 0
        in_out_reason_id = 0
        remark = 'remark'

class StorePlaceMock(DataSet):
    class StorePlace():
        id = 1
        ch_name = 'store_place1'

class ModelMock(DataSet):
    class Model():
        id = 1
        provider_id = 1
        name = 'model1'

class AssetTypeMock(DataSet):
    class AssetTypeModel():
        id = 1
        ch_name = 'asset_path1'
        asset_path = 'asset_path1'

class StoreStateMock(DataSet):
    class StoreState():
        id = 1
        ch_name = 'state1'

class DBConnectionFx(fixtures.Fixture):

    def setUp(self):
        super(DBConnectionFx, self).setUp()
        CONF(['--config-file', 'tests/etc/unittest.ini'])

class DBFixture(fixtures.Fixture):
    def setUp(self):
        super(DBFixture, self).setUp()
        self.engine = db_api.get_engine()
        self.engine.connect()
        self.dbfixture = SQLAlchemyFixture(
            engine=self.engine,
            env={
                "AssetConsumeInfoMock": AssetConsumeInfoModel
            }
        )
        self.data = self.dbfixture.data(AssetConsumeInfoMock)
        self.session = db_api.get_session()
        BASE.metadata.create_all(bind=self.engine)
        self.data.setup()
        self.addCleanup(self.data.teardown)

class TestAssetConsumeInfoDal(TestCase):

    def setUp(self):
        super(TestAssetConsumeInfoDal, self).setUp()
        self.useFixture(DBConnectionFx())
        self.dbfixture = self.useFixture(DBFixture())
        self.session = self.dbfixture.session

    def test_store_new_consume_data(self):

        data = {
            'asset_type': 1,  #id, not name
            'model': 1,       #id, not name
            'store_place': 1, #id, not name
            'in_num': 3,
            'remark': 'test',
            'user_name': 'ops'
        }
        AssetConsumeInfoDal().store_new_consume_data(data)
        model = AssetConsumeInfoDal().get_actual_asset_by_type_model_place(
            asset_type=1,
            model_id=1,
            store_place_id=1,
            store_state=1
        )
        self.assertEqual('test', model.remark)
        self.assertEqual('ops', model.user_name)


from mysite.views import *
from django.conf.urls.defaults import *
from django.conf import settings

urlpatterns = patterns('',
    (r'^$', main_page),

    # Login / logout.
    (r'^login/$', 'django.contrib.auth.views.login'),
    (r'^accounts/login/$', 'django.contrib.auth.views.login'),
    (r'^logout/$', logout_page),

    # Web portal.
    (r'^myapp/', include('myapp.urls')),

    # Serve static content.
    (r'^static/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': 'static'}),
)

if settings.DEBUG:
    urlpatterns += patterns('',
        url(r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
)


from SPARQLWrapper import SPARQLWrapper, JSON
import requests
import json

class DomainGetter():

    def query_wikidata(self, entity, limit_articles_number=None):
        articles_list = []
        sparql = SPARQLWrapper("http://query.wikidata.org/sparql")
        query = """
            SELECT ?item ?domain ?article
            WHERE
            {
              wd:%s wdt:P31 ?domain .
              ?item wdt:P31 ?domain .
              ?article schema:about ?item .
              ?article schema:isPartOf <https://en.wikipedia.org/> .
            }
        """ % (entity)
        if limit_articles_number:
            query += " LIMIT " + str(limit_articles_number)
        sparql.setQuery(query)
        sparql.setReturnFormat(JSON)
        results = sparql.query().convert()

        for result in results["results"]["bindings"]:
            articles_list.append(result["article"]["value"])
        return list(set(articles_list))

    def run(self, entity, limit_articles_number=None):
        article_list = self.query_wikidata(entity, limit_articles_number)
        return article_list

class EntityDomainGetter():

    def get_entity_from_keyword(self, keyword):
        url = "https://www.wikidata.org/w/api.php?action=wbsearchentities&language=en&format=json&limit=3&search=" + keyword
        req = requests.get(url)
        data = json.loads(req.text)
        entities = []
        for x in data['search']:
            print(x['label'])
            entities.append(x['id'])
        return entities

    def get_domain_for_entity(self, entity):
        url = 'http://www.wikidata.org/wiki/Special:EntityData/' + entity + '.json'
        req = requests.get(url)
        data = json.loads(req.text)
        domains = set()
        for x in data['entities'][entity]['claims']['P31']:
            domains.add(x['mainsnak']['datavalue']['value']['id'])
        return list(domains)

class EntityGetter():

    def get_entity_from_keyword(self, keyword):
        url = "https://www.wikidata.org/w/api.php?action=wbsearchentities&language=en&format=json&limit=3&search=" + keyword
        req = requests.get(url)
        data = json.loads(req.text)
        entities = []
        for x in data['search']:
            print(x['label'])
            entities.append(x['id'])
        return entities

    def run(self,keyword):
        return self.get_entity_from_keyword(keyword)


# coding:utf-8
"""
create on May 12, 2020 By Wayne YU

Function:

V2：
在研究俄罗斯断网事件的过程，对出口AS网络做了详细的研究
需进一步分析，我国出口AS互联关系中，美国占比的情况
分为两类，一类是全部互联关系；一类是AS TOP 1000的互联关系占比情况

V3:
新需求，需要研究三大运营商出口AS互联关系中，美国占比的情况
分为两类，一类是全部互联关系；一类是AS TOP 1000的互联关系占比情况

经统计三大运营商出口AS(v4部分)为：

电信:4134、4809、4812、4813
联通:4808、4837、9929、17621、17623
移动（含铁通）：9394、9808、24400、56041

V4:
我国互联网网络关键基础设施布局研究-国际通信网络部分数据分析支撑
我国电信运营商AS网络对外互联方向分国家统计图

经统计我国三大运营商主要出口AS为：

电信:4134、4809、4812、4813
联通:4808、4837、9929、17621、17623
移动（含铁通）：9394、9808、24400、56041

V5:
继续支撑我国互联网网络关键基础设施布局研究-国际通信网络部分数据分析支撑
我国电信运营商AS网络对外互联方向分国家统计图

增加输出Excel表，并将国家信息和缩写对应

V6:
继续支撑我国互联网网络关键基础设施布局研究-国际通信网络部分数据分析支撑
限定运营商

1）中国电信，4134（163网）、4809（CN2网）
2）中国联通，4837（169网）、9929（IP承载A网）
3）中国移动，9808（CMNET网）

一次互联，分别统计各运营商出口至全球各国的互联关系
二次互联，分别统计各运营商及其互联单位出口至全球各国的互联关系

V7:
统计4134、4809、4837、9929、9808五张网，分别的出口互联关系（分为一次互联和二次互联）

"""

import time
import csv
import os
import matplotlib.pyplot as plt

def write_to_csv(res_list, des_path):
    """
    把给定的List，写到指定路径的文件中
    :param res_list:
    :param des_path:
    :return: None
    """
    print("write file <%s> ..." % des_path)
    csvFile = open(des_path, 'w', newline='', encoding='gbk')
    try:
        writer = csv.writer(csvFile, delimiter=",")
        for i in res_list:
            writer.writerow(i)
    except Exception as e:
        print(e)
    finally:
        csvFile.close()
    print("write finish!")

def gain_as2country(as_info_file, country):
    """
    根据传入的as info file信息获取AS与国家的对应字典及该国家的所有的AS Info
    :param as_info_file:
    :param country:
    :return country_as_info:
    :return as2country:
    """
    country_as_info = []  # 存储country as 信息
    as2country = {}  # 存储as号到country的映射关系
    file_read = open(as_info_file, 'r', encoding='utf-8')
    # for line in file_read.readlines():
    #     line = line.strip().split("|")
    #     as2country[line[0]] = line[8]  # 生成字典
    #     temp_list = []
    #     if line[8] == country:
    #         temp_list.append(line[0])  # AS Number
    #         temp_list.append(line[1])  # AS All Relationships
    #         temp_list.append(line[5])  # AS Name
    #         temp_list.append(line[7])  # Source
    #         temp_list.append(line[8])  # Country
    #         country_as_info.append(temp_list)

    for line in file_read.readlines():
        line = line.strip().split("\t")
        # print(line)
        as_number = line[0]
        as_name = line[1].strip().split(",")[0].strip()
        as_country = line[1].strip().split(",")[-1].strip()
        as2country[as_number] = as_country
        temp_list = []
        if as_country == country:
            temp_list.append(as_number)
            temp_list.append(as_name)
            temp_list.append(as_country)
            country_as_info.append(temp_list)

    return country_as_info, as2country

def external_as_analysis(country, as2country, as_list, org_name):
    """
    根据输入的国家，统计该国家的出口AS数量及其互联方向的统计分析
    :param country:
    :param as2country:
    :param as_list:
    :param org_name:
    :return external_country_as:
    """
    print(country)
    external_country_as_return = []
    # 获取1998-2020年间全球BGP互联关系的存储文件
    file_path = []
    for root, dirs, files in os.walk("..\\000LocalData\\as_relationships\\serial-1"):
        for file_item in files:
            file_path.append(os.path.join(root, file_item))

    for path_item in file_path[-5:-4]:
        print(path_item)
        # 遍历一次文件，获取该国出口AS的数量
        file_read = open(path_item, 'r', encoding='utf-8')
        external_cnt = 0  # 存储该国出口连边的数量
        external_as_list = []  # 存储出口AS
        external_country_list = []  # 存储该国出口方向的国家
        external_country_as = []  # 存储该国出口互联关系中，国外AS网络
        for line in file_read.readlines():
            if line.strip().find("#") == 0:
                continue
            try:
                if str(line.strip().split('|')[0]) in as_list:
                    if as2country[str(line.strip().split('|')[1])] != country:
                        external_cnt += 1
                        external_as_list.append(str(line.strip().split('|')[0]))
                        external_country_list.append(as2country[str(line.strip().split('|')[1])])
                        external_country_as.append(str(line.strip().split('|')[1]))

                if as2country[str(line.strip().split('|')[0])] != country:
                    if str(line.strip().split('|')[1]) in as_list:
                        external_cnt += 1
                        external_as_list.append(str(line.strip().split('|')[1]))
                        external_country_list.append(as2country[str(line.strip().split('|')[0])])
                        external_country_as.append(str(line.strip().split('|')[0]))
            except Exception as e:
                pass
        external_country_as_return = external_country_as
        # 统计出口AS的排名
        external_as_rank = {}
        for item in list(set(external_as_list)):
            external_as_rank[item] = 0
        for item in external_as_list:
            external_as_rank[item] += 1
        # 将字典转换为列表
        external_as_rank_list = []
        temp_list = []
        for item in external_as_rank.keys():
            temp_list.append(item)
            temp_list.append(external_as_rank[item])
            external_as_rank_list.append(temp_list)
            temp_list = []
        external_as_rank_list.sort(reverse=True, key=lambda elem: int(elem[1]))
        external_as_list = list(set(external_as_list))
        print("All External Edges Count:", external_cnt)
        print("All External AS Count(Abroad):", len(list(set(external_country_as))))
        external_country_as_country = {}  # 统计三家运营商直联国际网络数量的国家分布
        external_country_as_us = []  # 存储美国AS网络
        for item_as in list(set(external_country_as)):
            # print(as2country[item_as])
            if as2country[item_as] == "US":
                external_country_as_us.append(item_as)

            if as2country[item_as] not in external_country_as_country.keys():
                external_country_as_country[as2country[item_as]] = 1
            else:
                external_country_as_country[as2country[item_as]] += 1
        # 将字典转化为列表
        external_country_as_country_list = []  # 存储三家运营商直联国际网络数量的国家分布
        for key in external_country_as_country.keys():
            external_country_as_country_list.append([key, external_country_as_country[key]])
        external_country_as_country_list.sort(reverse=True, key=lambda elem: elem[1])
        # print(external_country_as_country_list)
        # print("All External AS Count(Abroad(US)):", external_country_as_country["US"])
        # print(external_country_as_us)

        print("All External AS Count(Internal):", len(external_as_list))
        # print(external_as_rank_list)
        print("All External Country Count:", len(list(set(external_country_list))))

        # 统计互联国家方向的排名
        external_country_rank = {}
        for item in list(set(external_country_list)):
            external_country_rank[item] = 0
        for item in external_country_list:
            external_country_rank[item] += 1
        # print(len(external_country_rank))

        # 将字典转为列表
        external_country_rank_list = []
        temp_list = []
        for item in external_country_rank.keys():
            temp_list.append(item)
            temp_list.append(external_country_rank[item])
            external_country_rank_list.append(temp_list)
            temp_list = []
        # print(external_country_rank_list)
        external_country_rank_list.sort(reverse=True, key=lambda elem: int(elem[1]))
        gain_external_country_rank_list_plus(external_country_rank_list, country, org_name)
        draw_bar(external_country_rank_list, country, org_name)
    return external_country_as_return

def gain_external_country_rank_list_plus(rank_list, country, org_name):
    """
    根据传入的rank_list，将其国家缩写与国家信息、大洲信息相对应
    :param rank_list:
    :return:
    """
    # print(rank_list)
    geo_file = "..\\000LocalData\\as_geo\\GeoLite2-Country-Locations-zh-CN.csv"
    code2country = {}  # 存储country_iso_code到国家信息的字典
    geo_file_read = open(geo_file, 'r', encoding='utf-8')
    temp_list = []
    for line in geo_file_read.readlines():
        line = line.strip().split(",")
        if line[4]:  # 当国家代码存在
            # print(line)
            temp_list.append(line[5].strip("\""))
            temp_list.append(line[3].strip("\""))
            code2country[line[4]] = temp_list
            temp_list = []
    # print(code2country)
    external_country_rank_list_save = []  # 存储最终输出的带国家详细的排名表
    temp_list = []
    for item in rank_list:
        try:
            # print(item)
            temp_list.append(item[0])
            temp_list.append(item[1])
            temp_list.extend(code2country[item[0]])
            external_country_rank_list_save.append(temp_list)
            temp_list = []
        except Exception as e:
            # print(e)
            external_country_rank_list_save.append(temp_list)
            temp_list = []
    # print(external_country_rank_list_save)
    # 将信息存储到文件中
    save_path = "..\\000LocalData\\RUNet\\External_BGP_Rel" + country + "-ISP(20200201)_" + org_name + ".csv"
    write_to_csv(external_country_rank_list_save, save_path)

def draw_bar(rank_list, country, orgname):
    """
    根据传入的rank_list信息绘制直方图
    :param rank_list:
    :return:
    """
    x_list = []
    y_list = []

    for item in rank_list:
        x_list.append(item[0])
        y_list.append(item[1])
    # 开始绘图
    fig, ax = plt.subplots(1, 1, figsize=(50, 10))
    title_string = "External BGP Relationships Analysis-ISP:" + country + "( Date:20200201 )"
    ax.set_title(title_string)
    color = ['blue']
    plt.bar(x_list, y_list, width=0.7, color=color)
    for x, y in zip(x_list, y_list):
        plt.text(x, y + 0.05, '%.0f' % y, ha='center', va='bottom', fontsize=11)
    ax.set_xlabel('Country')
    ax.set_ylabel('Relationships Nums')
    # plt.grid(True, linestyle=':', color='r', alpha=0.9)
    # plt.show()
    plt.savefig("..\\000LocalData\\RUNet\\External_BGP_Rel" + country + "-ISP(20200201)" + orgname + ".jpg")

if __name__ == "__main__":
    time_start = time.time()  # 记录启动时间
    country = "CN"
    as_info_file_top = '..\\000LocalData\\as_map\\as_core_map_data_new20200201.csv'
    as_info_file_in = '..\\000LocalData\\as_Gao\\asn_info.txt'
    country_as_info, as2country_dict = gain_as2country(as_info_file_in, country)
    as_list = ['4134', '4809', '4837', '9929', '9808']  # 存储需要统计的AS列表
    for item_as in as_list:
        item_as_direct_name = str(item_as) + "_direct"
        external_country_as_direct = external_as_analysis(country,
                                                          as2country_dict,
                                                          [item_as],
                                                          item_as_direct_name)

        item_as_indirect_name = str(item_as) + "_indirect"
        external_country_as_direct.extend([item_as])
        external_country_as_indirect = external_as_analysis(country,
                                                            as2country_dict,
                                                            external_country_as_direct,
                                                            item_as_indirect_name)

    time_end = time.time()
    print("=>Scripts Finish, Time Consuming:", (time_end - time_start), "S")


""" Generating Database and Graphs for Test set """

import os
import cv2
import pickle
import threading
import numpy as np
import matplotlib.pyplot as plt
# User defined Modules
from createDB import createDB
from plots import plot_matches, plot_EER
from classify import classify_matched, thread_helper

############################### Helper Functions ##############################
def display_images(image):
    cv2.imshow('Showing image', image)
    cv2.waitKey()

def storeData(dat, filename):
    dbfile = open(filename, 'wb')
    pickle.dump(dat, dbfile)
    dbfile.close()

def loadData(filename):
    dbfile = open(filename, 'rb')
    return pickle.load(dbfile)
###############################################################################

# Load/Save DB
DB = createDB('../TestSet/Reference/')
storeData(DB, '../Pickles/database_test.pkl')
DB = loadData('../Pickles/database_test.pkl')
threshold = 0.11

# Matched Points Percentage for each type of signature
print("Questioned Signatures")
match = thread_helper('../TestSet/Questioned/', DB, threshold)
genuine_match = match[[48, 51, 65]]
disguise_match = match[[5, 14, 27, 28, 33, 86, 89]]
simulated_match = match[[0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40,
41, 42, 43, 44, 45, 46, 47, 49, 50, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62,
63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83,
84, 85, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]

# Save/Load percentages (for convenience)
storeData([genuine_match, disguise_match, simulated_match], '../Pickles/vals_test.pkl')
genuine_match, disguise_match, simulated_match = loadData('../Pickles/vals_test.pkl')
plot_matches(genuine_match, disguise_match, simulated_match, '_test')

# Calculate FAR and FRR
far = []
frr = []
theta_range = np.arange(0.05, 0.25, 0.005)
for theta in theta_range:
    ind1 = np.where(genuine_match < theta)
    ind2 = np.where(disguise_match < theta)
    ind3 = np.where(simulated_match > theta)
    frr.append((ind1[0].shape[0] + ind2[0].shape[0]) / (genuine_match.shape[0] + disguise_match.shape[0]) * 100)
    far.append(ind3[0].shape[0] / simulated_match.shape[0] * 100)
plot_EER(theta_range, far, frr, threshold)

# Compute Confidence Matrix
eer = 0.224
ind = np.where(genuine_match < eer)
ind1 = np.where(disguise_match < eer)
ind2 = np.where(simulated_match > eer)
print('\t\tGenuine\tFake')
print('Genuine\t\t' + str(genuine_match.shape[0] - ind[0].shape[0]) + '\t' + str(ind[0].shape[0]))
print('Disguise\t' + str(disguise_match.shape[0] - ind1[0].shape[0]) + '\t' + str(ind1[0].shape[0]))
print('Simulated\t' + str(ind2[0].shape[0]) + '\t' + str(simulated_match.shape[0] - ind2[0].shape[0]))


from django.shortcuts import redirect, render, get_object_or_404
from django.contrib.auth.decorators import login_required
from django.utils import timezone
from .models import Recipe, Comment
from .forms import RecipeForm, CommentForm

def index(request):
	return render(request, 'blog/index.html',{})

def recipes(request):
	recipes = Recipe.objects.filter(pub_date__lte=timezone.now()).order_by('-pub_date')
	return render(request, 'blog/recipes.html',{'recipes': recipes})

def recipe(request, pk):
	recipe = get_object_or_404(Recipe, pk=pk)
	return render(request, 'blog/recipe.html',{'recipe': recipe})

def add_comment(request, pk):
	recipe = get_object_or_404(Recipe, pk=pk)
	if request.method == "POST":
		form = CommentForm(request.POST)
		if form.is_valid():
			comment = form.save(commit=False)
			comment.recipe = recipe
			comment.save()
			return redirect('recipe', pk=recipe.pk)
	else:
		form = CommentForm()
	return render(request, 'blog/add_comment.html', {'form': form})

@login_required
def comment_approve(request, pk):
    comment = get_object_or_404(Comment, pk=pk)
    comment.approve()
    return redirect('blog.views.recipe', pk=comment.recipe.pk)

@login_required
def comment_remove(request, pk):
    comment = get_object_or_404(Comment, pk=pk)
    recipe_pk = comment.recipe.pk
    comment.delete()
    return redirect('blog.views.recipe', pk=recipe_pk)

@login_required
def add_recipe(request):
	if request.method == "POST":
		form = RecipeForm(request.POST, request.FILES)
		if form.is_valid():
			recipe = form.save(commit=False)
			recipe.author = request.user
			recipe.save()
			return redirect('recipe', pk=recipe.pk)
	else:
		form = RecipeForm()
	return render(request, 'blog/add_recipe.html', {'form': form})

@login_required
def edit_recipe(request, pk):
	recipe = get_object_or_404(Recipe, pk=pk)
	if request.method == "POST":
		form = RecipeForm(request.POST, request.FILES, instance=recipe)
		if form.is_valid():
			if not recipe.thumbnail:
				recipe.create_thumbnail()
			recipe = form.save(commit=False)
			recipe.author = request.user
			recipe.save()
			return redirect('recipe', pk=recipe.pk)
	else:
		form = RecipeForm(instance=recipe)
	return render(request, 'blog/edit_recipe.html', {'form': form})

@login_required
def drafts(request):
    recipes = Recipe.objects.filter(pub_date__isnull=True).order_by('created_date')
    return render(request, 'blog/drafts.html', {'recipes': recipes})

@login_required
def recipe_publish(request, pk):
    recipe = get_object_or_404(Recipe, pk=pk)
    recipe.publish()
    return redirect('blog.views.recipe', pk=pk)

@login_required
def recipe_delete(request, pk):
    recipe = get_object_or_404(Recipe, pk=pk)
    recipe.delete()
    return redirect('blog.views.recipes')


"""
Delete round transformation
"""

from __future__ import print_function
import random
import os

import torch
import torch.nn.parallel
import torch.backends.cudnn as cudnn
cudnn.benchmark = True
cudnn.fastest = True

import torchvision.transforms as T

from trainer import Trainer
from config import get_config
from dataloader import get_loader

def main(config):
    # make directory
    if config.ckpt_folder is None:
        config.ckpt_folder = 'checkpoints'
    os.system('mkdir {0}'.format(config.ckpt_folder))
    print("[*] Make checkpoints folder!")

    if config.sample_folder is None:
        config.sample_folder = 'samples'
    os.system('mkdir {0}'.format(config.sample_folder))
    print("[*] Make samples folder!")

    config.manual_seed = random.randint(1, 10000)
    print("Random Seed: ", config.manual_seed)

    random.seed(config.manual_seed)
    torch.manual_seed(config.manual_seed)
    torch.cuda.manual_seed_all(config.manual_seed)

    # for faster training
    cudnn.benchmark = True

    # define train/test transformation
    transform = T.Compose([
        T.ToTensor(),
        T.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),
    ])

    # define train/test dataloader
    train_loader, test_loader = get_loader(dataroot=config.dataroot,
                                           batch_size=config.batch_size,
                                           shuffle=config.shuffle,
                                           transform=transform,
                                           num_workers=config.num_workers)

    # define trainer class
    trainer = Trainer(train_loader, test_loader, config)

    if config.mode == 'train':
        trainer.train()
    elif config.mode == 'test':
        trainer.eval()

if __name__ == "__main__":
    config = get_config()
    main(config)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

from itertools import product
from hw3_runner import main
from multiprocessing import Pool

pool = Pool()

def baseline():
    cmd = "--stupid --greedy --context --thresh {0}"
    thresholds = [1, 5, 10, 50, 100, 1000, 10000]
    cmds = [cmd.format(t).split() for t in thresholds]
    results = pool.map(main, cmds)
    with open("baseline.txt", "w") as f:
        for t, r in zip(thresholds, results):
            args = (map("{0}".format, [t, r[0]])
                    + ["{0:.3f}".format(v*100) for v in r[1:]])
            f.write(" & ".join(args) + "\\\\\n")

def trigram():
    cmd = "--stupid --greedy --thresh {0} --lambda2 {1} --lambda3 {2}"
    thresholds = [2, 5, 10]
    lambda2s = [0.2, 0.3, 0.4]
    lambda3s = [0.5, 0.6, 0.7]
    args = [(t, l2, l3)
            for t, l2, l3 in product(thresholds, lambda2s, lambda3s)
            if l2+l3 < 1.0]
    cmds = [cmd.format(*a).split() for a in args]
    results = pool.map(main, cmds)
    with open("trigram.txt", "w") as f:
        for a, r in zip(args, results):
            cols = (["{0}".format(a[0])] + map("{0:.1f}".format, a[1:])
                    + ["{0:.3f}".format(v*100) for v in r[1:]])
            f.write(" & ".join(cols) + "\\\\\n")

def viterbi():
    cmd = "--stupid --thresh {0} --lambda2 {1} --lambda3 {2}"
    thresholds = [2, 5, 10]
    lambda2s = [0.2, 0.3, 0.4]
    lambda3s = [0.5, 0.6, 0.7]
    args = [(t, l2, l3)
            for t, l2, l3 in product(thresholds, lambda2s, lambda3s)
            if l2+l3 < 1.0]
    cmds = [cmd.format(*a).split() for a in args]
    results = pool.map(main, cmds)
    with open("viterbi.txt", "w") as f:
        for a, r in zip(args, results):
            cols = (["{0}".format(a[0])] + map("{0:.1f}".format, a[1:])
                    + ["{0:.3f}".format(v*100) for v in r[1:]])
            f.write(" & ".join(cols) + "\\\\\n")

def unknown():
    cmd = ("--thresh {0} --lambda2 0.2 --lambda3 0.7 --theta1 {1} "
           "--theta2 {2} --theta3 {3}")
    thresholds = [2, 5, 10]
    theta1s = [0.01, 0.05, 0.1]
    theta2s = [0.3, 0.5, 0.7]
    theta3s = [0.3, 0.5, 0.7]
    args = [(t, t1, t2, t3)
            for t, t1, t2, t3 in product(thresholds, theta1s, theta2s, theta3s)
            if t1+t2+t3 < 1.0]
    cmds = [cmd.format(*a).split() for a in args]
    results = pool.map(main, cmds)
    with open("unknown.txt", "w") as f:
        for a, r in zip(args, results):
            cols = (["{0}".format(a[0])] + map("{0:.1f}".format, a[1:])
                    + ["{0:.3f}".format(v*100) for v in r[1:]])
            f.write(" & ".join(cols) + "\\\\\n")

if __name__ == "__main__":
    # baseline()
    # trigram()
    # viterbi()
    unknown()


# importing all required package library
from keras.applications import ResNet50
from keras.preprocessing.image import img_to_array
from keras.applications import imagenet_utils
import numpy as np

# A class for image prediction based on ResNet pre-trained model
class ResNetPredict:
    def __init__(self):
        self.image = None
        self.model = None
        self.result_value = None

    def load_model(self):
        self.model = ResNet50(weights="imagenet")

    def prepare_image(self, image, target):
        if image.mode != "RGB":
            image = image.convert("RGB")
        image = image.resize(target)
        image = img_to_array(image)
        image = np.expand_dims(image, axis=0)
        image = imagenet_utils.preprocess_input(image)
        return image

    def predict(self, image):
        prediction = self.model.predict(image)
        results = imagenet_utils.decode_predictions(prediction)
        return results


import sys
import os
from Bio import SeqIO
import pandas as pd
import random

# ------- INPUT -------
# 1MB of sequence where local matches will be inserted into
query_file = snakemake.input.query
# Local matches sampled with certain error rate
match_file = snakemake.input.matches

# ------- OUTPUT -------
query_with_insertions_file = snakemake.output.query
ground_truth_file = snakemake.output.ground_truth

def check_file_exists(fn):
    if not os.path.exists(fn):
        raise SystemError("Error: File does not exist\n")

check_file_exists(match_file)
local_matches = list(SeqIO.parse(match_file, "fastq"))

check_file_exists(query_file)
query = list(SeqIO.parse(query_file, "fasta"))
assert (len(query) == 1),"Query contains more than one sequence, pick which one to insert to."

# Sort random positions
query_len = len(query[0].seq)
random.seed(snakemake.params.seed)
ran_ind_list = random.sample(range(query_len), len(local_matches))
ran_ind_list.sort()
# Shuffle local matches so that they would appear in the query in a random order
random.shuffle(local_matches)

# Assign random location to each local match
for i in range(len(local_matches)):

    local_matches[i].random_location = ran_ind_list[i]

original_query = query[0].seq

id_list = []
position_list = []
length_list = []
reference_position_list = []
reference_length_list = []
errors_list = []

# Insert local matches into query at random positions
query_with_insertions = original_query[0:local_matches[0].random_location]
insertion_length = 0
for i in range(0, len(local_matches)):
    match = local_matches[i]

    # Gather ground truth
    id_list.append(match.name)

    # converts "l50-3 start_position=935090,length=50,errors=0,reference_id='1',reference_file='ref_0.fasta'"
    # into {'start_position': '935090', 'length': '50', 'errors': '0', reference_id="'reference_id'", reference_file="'ref_0.fasta'"}
    meta_info = dict((key.strip(), value.strip()) for key, value in (element.split('=') for element in match.description.split(" ", 2)[1].split(',')))
    reference_position_list.append(int(meta_info['start_position']))
    reference_length_list.append(int(meta_info['length']))
    errors_list.append(int(meta_info['errors']))

    insertion_position = match.random_location + insertion_length
    position_list.append(insertion_position)    # position in the query
    length_list.append(len(match.seq))

    prefix = query_with_insertions[0:insertion_position]
    insertion = match.seq

    if (i < (len(local_matches) - 1)):
        postfix = original_query[match.random_location:local_matches[i+1].random_location]
    else:
        # Edge case: insert last local match
        postfix = original_query[local_matches[-1].random_location:]

    insertion_length += len(match.seq)
    query_with_insertions = prefix + insertion + postfix

assert (len(query_with_insertions) - len(original_query)) == insertion_length,"Wrong length of sequence inserted."

# Write insertion query file
with open(query_with_insertions_file, 'w') as f:
    f.write('>1\n')
    f.write(str(query_with_insertions))

f.close()

# Create ground truth file
ground_truth = {'id':id_list,
                'position':position_list,
                'length':length_list,
                'reference_position': reference_position_list,
                'reference_length': reference_length_list,
                'errors': errors_list}
ground_truth_df = pd.DataFrame(ground_truth)
ground_truth_df.to_csv(ground_truth_file, index=False, sep='\t')
ground_truth_df.head()


#! /usr/bin/env python
# -*- coding:utf-8 -*-

def main():
    import sys
    import io
    import json
    import re

    # cp932では出力できない文字が含まれているため、出力エンコーディングをUTF-8に変更
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")

    with open("england.json", "r", encoding="utf-8") as f:
        j = json.load(f)

    text = j["text"]

    # 基礎情報を取り出す
    p_info = re.compile('{{基礎情報\s+国(.*?)\n}}\n', re.DOTALL)
    m = p_info.search(text)
    # 基礎情報をstrにする
    info = m.group(1)

    # 全てのフィールドの行頭が|で始まっていることを利用する。
    p_field = re.compile(r'\n\|(.+?)\s+=\s+(((?!\n\|).)*)', re.DOTALL)
    m2 = p_field.findall(info)

    p_de_emphasize = re.compile(r'\'{2,5}')
    p_remove_link = re.compile(r'\[\[(.*?)\]\]')

    # MediaWikiマークアップの除去
    p_ref = re.compile(r'<ref.*?>(.*?)</ref>')
    p_ref_single = re.compile(r'<ref.*?/>')
    p_nl = re.compile(r'<br\s*?/>')

    d = {}
    for p in m2:
        s = p_de_emphasize.sub('', p[1])
        s = p_remove_link.sub(r'\1', s)
        s = p_ref.sub(r'\1', s)
        s = p_ref_single.sub('', s)
        s = p_nl.sub('', s)
        d[p[0]] = s

    for k, v in sorted(d.items()):
        print(k, v)

if __name__ == '__main__':
    main()


import cv2
import numpy as np
image=cv2.VideoCapture(0)
while True:
    ret,frame=image.read()
#now converting  RGB into HSV
    hsv = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
#now we can filter our ouput with our desired color
# example- for blue color
    lower_range=np.array([110,50,50])
    upper_range=np.array([130,255,255])
    filter_img=cv2.inRange(hsv,lower_range,upper_range)
    cv2.imshow("filter output",filter_img)
    cv2.imshow("original output",frame)
    if cv2.waitKey(1)==13:
        break
image.release()
cv2.destroyAllWindows()

open cv/22.color filtering using webcam/color_filtering.py