# -*- coding: utf-8 -*
import os
import re
import numpy as np
import six
from PIL import Image
from itertools import chain
import glob
import time
import matplotlib.pylab as plt
from sklearn.datasets import fetch_mldata
import chainer
import chainer.links as L
import chainer.functions as F
from chainer import Chain, optimizers, Variable, serializers, iterators, training, datasets
from chainer.datasets import LabeledImageDataset, tuple_dataset
from chainer.training import extensions, triggers
from chainer.serializers import npz
from chainer.backends import cuda
from chainer import Function
from chainer import Link, ChainList
from chainer import cuda
from shuffle import shuffle
from augmentation import augmentation
from resize import resize
from excel import change_excel
from edit import edit_excel
from scale_augmentation import scale_data_augmentation
from count_image import count_image
from accuracy import accuracy
import xlwt
import shutil
import openpyxl as xl
from natsort import natsorted
from decimal import *

def image2Train(pathsAndLabels, channels=3):

    allData = []
    count_train_label = np.zeros(G)

    #データの追加(画像ファイル，ラベル)
    for pathsAndLabel in pathsAndLabels:
        path = pathsAndLabel[0]
        label = pathsAndLabel[1]
        imagelist = glob.glob(path + '*')
        count_train_label[int(label)] = len(imagelist)
        for imgName in imagelist:
            allData.append([imgName, label])
    allData = np.random.permutation(allData)#シャッフル

    #チャンネルが1の時は，画像ファイルとラベルデータに追加
    if channels == 1:
        imageData = []
        labelData = []
        for pathAndLabel in allData:
            img = Image.open(pathsAndLabel[0])
            imgData = np.asarray([np.float32(img)/255.0])#画像の正規化
            imageData.append(imgData)
            labelData.append(np.int32(pathsAndLabel[1]))

        threshold = np.int32(len(imageData)/8*7)#データの何割を教師データとテストデータにするか
        train = (imageData[0:threshold], labelData[0:threshold])
        test = (imageData[threshold:], labeData[threshold:])
    #RGBの時の処理
    else:
        imageData = []
        labelData = []
        for pathAndLabel in allData:
            img = Image.open(pathAndLabel[0])

            #新たに追加
            img = np.asarray(img, dtype=np.float32)
            img = img[:, :, ::-1]
            img -= np.array([103.939, 116.779, 123.68], dtype=np.float32)
            #print(img)
            img = img/255.0

            imageData.append(img)
            labelData.append(np.int32(pathAndLabel[1]))

        dataset = {}
        dataset['train_img'] = np.array(imageData[0:N]).transpose(0,3,1,2)
        dataset['train_label'] = np.array(labelData[0:N])

    return (dataset['train_img'], dataset['train_label']), count_train_label

def image2Test(pathsAndLabels, channels=3):

    allData = []
    count_test_label = np.zeros(G)

    #データの追加(画像ファイル，ラベル)
    for pathsAndLabel in pathsAndLabels:
        path = pathsAndLabel[0]
        label = pathsAndLabel[1]
        imagelist = glob.glob(path + '*')
        count_test_label[int(label)] = len(imagelist)
        for imgName in imagelist:
            allData.append([imgName, label])
    allData = np.random.permutation(allData)#シャッフル

    #チャンネルが1の時は，画像ファイルとラベルデータに追加
    if channels == 1:
        imageData = []
        labelData = []
        for pathAndLabel in allData:
            img = Image.open(pathsAndLabel[0])
            imgData = np.asarray([np.float32(img)/255.0])#画像の正規化
            imageData.append(imgData)
            labelData.append(np.int32(pathsAndLabel[1]))

        threshold = np.int32(len(imageData)/8*7)#データの何割を教師データとテストデータにするか
        train = (imageData[0:threshold], labelData[0:threshold])
        test = (imageData[threshold:], labeData[threshold:])
    #RGBの時の処理
    else:
        imageData = []
        labelData = []
        for pathAndLabel in allData:
            img = Image.open(pathAndLabel[0])

            #新たに追加
            img = np.asarray(img, dtype=np.float32)
            img = img[:, :, ::-1]
            img -= np.array([103.939, 116.779, 123.68], dtype=np.float32)
            img = img/255.0
            imageData.append(img)
            labelData.append(np.int32(pathAndLabel[1]))

        dataset = {}

        dataset['test_img'] = np.array(imageData[0:N_test]).transpose(0,3,1,2)
        dataset['test_label'] = np.array(labelData[0:N_test])

    return (dataset['test_img'], dataset['test_label']), count_test_label

class CNN(Chain):
    def __init__(self):
        super(CNN, self).__init__(
            conv1 = L.Convolution2D(3, 40, 15), # 入力:3 , 出力:40。両方記述。filter 5
            conv2 = L.Convolution2D(40, 50, 15), # 入力:40 , 出力:50。両方記述。filter 5
            l1 = L.Linear(None, 500),  # ここの入力の計算は以下の参考HPを参照。
            l2 = L.Linear(500, 500),
            l3 = L.Linear(500, G, initialW=np.zeros((G, 500), dtype=np.float32))#ここの値は設定する出力の数によって変更
        )
    def forward(self, x, ratio = 0.1):
        h = F.max_pooling_2d(F.relu(self.conv1(x)), 2)
        h = F.max_pooling_2d(F.relu(self.conv2(h)), 2)
        h = F.dropout(F.relu(self.l1(h)), ratio)
        h = F.dropout(F.relu(self.l2(h)), ratio)
        h = self.l3(h)
        return h

class VGG(chainer.Chain):
    def __init__(self, class_labels, pretrained_model='VGG_ILSVRC_16_layers.npz'):
        super(VGG, self).__init__()
        with self.init_scope():
            self.base = BaseVGG()
            self.fc6 = L.Linear(None, 793, initialW=chainer.initializers.HeNormal())
            self.fc7 = L.Linear(None, 252, initialW=chainer.initializers.HeNormal())
            self.fc8 = L.Linear(None, class_labels, initialW=chainer.initializers.HeNormal())
        npz.load_npz(pretrained_model, self.base)

    def __call__(self, x, t):
        h = self.predict(x)
        loss = F.softmax_cross_entropy(h,t)
        chainer.report({'loss': loss, 'accuracy': F.accuracy(h,t)}, self)
        return loss

    def predict(self, x):
        h = self.base(x)
        h = F.relu(self.fc6(h))
        h = F.relu(self.fc7(h))
        h = self.fc8(h)
        return h

class BaseVGG(Chain):
    def __init__(self):
        super(BaseVGG, self).__init__()
        with self.init_scope():
            self.conv1_1 = L.Convolution2D(None, 64, 3, 1, 1)
            self.conv1_2 = L.Convolution2D(64, 64, 3, 1, 1)
            self.conv2_1 = L.Convolution2D(64, 128, 3, 1, 1)
            self.conv2_2 = L.Convolution2D(128, 128, 3, 1, 1)
            self.conv3_1 = L.Convolution2D(128, 256, 3, 1, 1)
            self.conv3_2 = L.Convolution2D(256, 256, 3, 1, 1)
            self.conv3_3 = L.Convolution2D(256, 256, 3, 1, 1)
            self.conv4_1 = L.Convolution2D(256, 512, 3, 1, 1)
            self.conv4_2 = L.Convolution2D(512, 512, 3, 1, 1)
            self.conv4_3 = L.Convolution2D(512, 512, 3, 1, 1)
            self.conv5_1 = L.Convolution2D(512, 512, 3, 1, 1)
            self.conv5_2 = L.Convolution2D(512, 512, 3, 1, 1)
            self.conv5_3 = L.Convolution2D(512, 512, 3, 1, 1)

    def __call__(self, x):
        h = F.relu(self.conv1_1(x))
        h = F.relu(self.conv1_2(h))
        h = F.max_pooling_2d(h, ksize=2, stride=2)

        h = F.relu(self.conv2_1(h))
        h = F.relu(self.conv2_2(h))
        h = F.max_pooling_2d(h, ksize=2, stride=2)

        h = F.relu(self.conv3_1(h))
        h = F.relu(self.conv3_2(h))
        h = F.relu(self.conv3_3(h))
        h = F.max_pooling_2d(h, ksize=2, stride=2)

        h = F.relu(self.conv4_1(h))
        h = F.relu(self.conv4_2(h))
        h = F.relu(self.conv4_3(h))
        h = F.max_pooling_2d(h, ksize=2, stride=2)

        h = F.relu(self.conv5_1(h))
        h = F.relu(self.conv5_2(h))
        h = F.relu(self.conv5_3(h))
        h = F.max_pooling_2d(h, ksize=2, stride=2)

        return h

if __name__ == '__main__':
    print('class_labels変更した?')
    #以下パラメータの設定
    k = 10 #試行回数
    n_epoch = 20 #更新回数
    batch_size = 125 #バッチサイズ
    batch_size_predict = 1
    cnt_trial = 1#試行回数ごとの写真フォルダ分けに使用
    h = 90#入力画像の高さ
    w = 78#入力画像の幅
    gpu = 0 #GPUを用いるなら0,CPUなら-1
    N_test = 793#テスト画像枚数(ここの数字は再確認)

    #以下扱うディレクトリのパス設定
    input_dir = 'C:\\Users\\user\\Desktop\\input'#入力教師用画像フォルダ
    input_testing_dir = 'C:\\Users\\user\\Desktop\\input_test'#入力評価用画像フォルダ
    main_dir = 'C:\\Users\\user\\Desktop\\main'#mainフォルダ
    result_dir = 'C:\\Users\\user\\Desktop\\result'#結果フォルダ

    #main_dirがなければ作成
    if not os.path.exists(main_dir):
        os.mkdir(main_dir)

    #result_dirがなければ作成
    if not os.path.exists(result_dir):
        os.mkdir(result_dir)

    #誤判別時の画像出力フォルダ作成
    miss_judge_train_dir = os.path.join(result_dir, 'miss_judge_train')
    miss_judge_test_dir = os.path.join(result_dir, 'miss_judge_test')

    #誤班別時画像出力フォルダがなければ，作成
    if not os.path.exists(miss_judge_train_dir):
        os.mkdir(miss_judge_train_dir)
    if not os.path.exists(miss_judge_test_dir):
        os.mkdir(miss_judge_test_dir)
    for j in range(1, k+1):
        if not os.path.exists(os.path.join(miss_judge_train_dir, str(j)+'_trial')):
            os.mkdir(os.path.join(miss_judge_train_dir, str(j)+'_trial'))
        if not os.path.exists(os.path.join(miss_judge_test_dir, str(j)+'_trial')):
            os.mkdir(os.path.join(miss_judge_test_dir, str(j)+'_trial'))
        for l in range(1, n_epoch+1):
            if not os.path.exists(os.path.join(os.path.join(miss_judge_train_dir, str(j)+'_trial'), str(l))):
                os.mkdir(os.path.join(os.path.join(miss_judge_train_dir, str(j)+'_trial'), str(l)))
            if not os.path.exists(os.path.join(os.path.join(miss_judge_test_dir, str(j)+'_trial'), str(l))):
                os.mkdir(os.path.join(os.path.join(miss_judge_test_dir, str(j)+'_trial'), str(l)))

    #正解時の画像出力フォルダ作成
    correct_judge_train_dir = os.path.join(result_dir, 'correct_judge_train')
    correct_judge_test_dir = os.path.join(result_dir, 'correct_judge_test')

    #正解判別時画像出力フォルダがなければ，作成
    if not os.path.exists(correct_judge_train_dir):
        os.mkdir(correct_judge_train_dir)
    if not os.path.exists(correct_judge_test_dir):
        os.mkdir(correct_judge_test_dir)
    for j in range(1,k+1):
        if not os.path.exists(os.path.join(correct_judge_train_dir, str(j)+'_trial')):
            os.mkdir(os.path.join(correct_judge_train_dir, str(j)+'_trial'))
        if not os.path.exists(os.path.join(correct_judge_test_dir, str(j)+'_trial')):
            os.mkdir(os.path.join(correct_judge_test_dir, str(j)+'_trial'))
        for l in range(1, n_epoch+1):
            if not os.path.exists(os.path.join(os.path.join(correct_judge_train_dir, str(j)+'_trial'), str(l))):
                os.mkdir(os.path.join(os.path.join(correct_judge_train_dir, str(j)+'_trial'), str(l)))
            if not os.path.exists(os.path.join(os.path.join(correct_judge_test_dir, str(j)+'_trial'), str(l))):
                os.mkdir(os.path.join(os.path.join(correct_judge_test_dir, str(j)+'_trial'), str(l)))

    #以下訓練と評価
    for i in range(1, k+1):
        start_time = time.clock()
        book = xlwt.Workbook()
        print(str(i) + '_trial')
        #試行ごとのフォルダ作成
        current_dir = os.path.join(main_dir, str(i)+'_trial')
        if not os.path.exists(current_dir):
            os.mkdir(current_dir)

        #試行ごとの結果フォルダ作成
        current_result_dir = os.path.join(result_dir, str(i)+'_trial')
        if not os.path.exists(current_result_dir):
            os.mkdir(current_result_dir)

        train_dir = os.path.join(current_dir, 'training')
        test_dir = os.path.join(current_dir, 'testing')

        #shuffleにより評価用画像枚数を返す．
        #N_testはテスト枚数,flagが0ならinputdirから教師用とテスト用に8:2でわける．flagが1なら教師用フォルダから教師用に，評価用フォルダから評価用にわける．

        #N_test = shuffle(input_dir, current_dir, flag = 0)
        N_test = shuffle(input_dir, current_dir, input_testing_dir, flag = 1)#N_test:評価用画像枚数

        #scale_augmentationの実行
        scale_data_augmentation(os.path.join(train_dir, 'image1'), os.path.join(train_dir, 'image2'), (h,w))

        #resizeの実行
        resize(os.path.join(train_dir, 'image2'), os.path.join(train_dir, 'image3'), h, w)
        resize(os.path.join(test_dir, 'image1'), os.path.join(test_dir, 'image2'), h, w)

        #data augmentationの実行
        augmentation(os.path.join(train_dir, 'image3'), os.path.join(train_dir, 'image4'), (h,w))

        #count_imageの実行
        #これにより教師用画像枚数とグループ数取得
        N, G = count_image(os.path.join(train_dir, 'image4'))
        print("N:" + str(N))
        print("N_test:" + str(N_test))

        #画像フォルダのパス
        IMG_DIR_train = os.path.join(train_dir, 'image4')
        IMG_DIR_test = os.path.join(test_dir, 'image2')

        #各グループのパス
        dnames_train = glob.glob('{}/*'.format(IMG_DIR_train))
        dnames_train = sorted(dnames_train, key=lambda s: int(re.search(r'\d+', s).group()))
        dnames_test = glob.glob('{}/*'.format(IMG_DIR_test))
        dnames_test = sorted(dnames_test, key=lambda s: int(re.search(r'\d+', s).group()))

        #画像ファイルパス一覧
        fnames_train = [glob.glob('{}/*.jpg'.format(d)) for d in dnames_train]
        fnames_train = list(chain.from_iterable(fnames_train))
        fnames_test = [glob.glob('{}/*.jpg'.format(d)) for d in dnames_test]
        fnames_test = list(chain.from_iterable(fnames_test))

        # それぞれにフォルダ名から一意なIDを付与し、画像を読み込んでデータセット作成
        labels_train = [os.path.basename(os.path.dirname(fn)) for fn in fnames_train]
        dnames_train = [os.path.basename(d) for d in dnames_train]
        labels_train = [dnames_train.index(l) for l in labels_train]
        d_train = LabeledImageDataset(list(zip(fnames_train, labels_train)))

        labels_test = [os.path.basename(os.path.dirname(fn)) for fn in fnames_test]
        dnames_test = [os.path.basename(d) for d in dnames_test]
        labels_test = [dnames_test.index(l) for l in labels_test]
        d_test = LabeledImageDataset(list(zip(fnames_test, labels_test)))

        #VGG用の前処理関数
        def transform(data):
            img, label = data
            img = L.model.vision.vgg.prepare(img, size=(h, w))
            img = img / 255. #正規化する．0〜1に落とし込む
            return img, label

        train = chainer.datasets.TransformDataset(d_train, transform)
        test = chainer.datasets.TransformDataset(d_test, transform)

        model = VGG(class_labels = G)

        #GPUの設定
        if gpu >= 0:
            print('GPUを用います')
            cuda.get_device(0).use()
            model.to_gpu(0)

        optimizer = optimizers.Adam() # 最適化関数
        optimizer.setup(model)
        # model.base.disable_update()

        #ラベルごとのトータル誤判別数
        miss_train_total = np.zeros(G)
        miss_test_total = np.zeros(G)

        #ラベルごとのトータル正解数
        correct_train_total = np.zeros(G)
        correct_test_total = np.zeros(G)

        #何を何と判別したか2次元配列
        miss_train_judge = np.zeros([G,G])#ここも
        miss_test_judge = np.zeros([G,G])

        cnt_train = 0#画像を保存するように使用
        cnt_test = 0#画像を保存するときの数
        cnt_train_correct_judge = 0
        cnt_test_correct_judge =0

        print(str(i)+'_trial')
        #エクセル書き込み
        sheet = book.add_sheet(str(i)+'_trial')
        sheet.write(1, 0, '教師用')
        sheet.write(2, 0, '評価用')
        sheet.write(6, 0, '教師用')
        sheet.write(6, G+3, '評価用')

        #葉齢ごとの判別率
        for l in range(0, G):
            sheet.write(7, l+1, 'stage'+str(l))

        for l in range(0, G):
            sheet.write(7, l+G+4, 'stage'+str(l))

        sheet.write(n_epoch+15, 0, '誤判別のやーつ(教師用)')
        sheet.write(n_epoch+15, 3, '誤判別先')
        for l in range(0,G):
            sheet.write(n_epoch+16, l+1, l)
            sheet.write(n_epoch+l+17, 0, l)

        sheet.write(n_epoch+15, G+3, '誤判別のやーつ(評価用)')
        for l in range(0, G):
            sheet.write(n_epoch+16, l+G+4, l)
            sheet.write(n_epoch+l+17, G+3, l)

        #trainer周りの設定
        train_iter = iterators.SerialIterator(train, batch_size)
        test_iter = iterators.SerialIterator(test, 1, repeat=False, shuffle=False)

        updater = training.StandardUpdater(train_iter, optimizer, device = gpu)
        trainer = training.Trainer(updater, (n_epoch, 'epoch'), out=current_result_dir)

        trainer.extend(extensions.dump_graph('main/loss'))
        #trainer.extend(extensions.snapshot(), trigger=(epoch, 'epoch'))
        trainer.extend(extensions.LogReport())
        trainer.extend(extensions.observe_lr())
        trainer.extend(extensions.PrintReport(
            ['epoch',
            'main/loss',
            'main/accuracy',
            'test/main/loss',
            'test/main/accuracy',
            'elapsed_time',
            'lr',], sheet = sheet, flag = 1))
        trainer.extend(extensions.ProgressBar(update_interval=1))
        trainer.extend(extensions.Evaluator(test_iter, model, device = gpu), name='test')

        trainer.extend(extensions.PlotReport(['main/loss'], 'epoch', file_name='main_loss.png', marker = ''))
        trainer.extend(extensions.PlotReport(['main/accuracy'], 'epoch', file_name='main_accuracy.png'))
        trainer.extend(extensions.PlotReport(['test/main/loss'], 'epoch', file_name='test_loss.png'))
        trainer.extend(extensions.PlotReport(['test/main/accuracy'], 'epoch', file_name='test_accuracy.png'))
        trainer.extend(extensions.PlotReport(['main/loss', 'test/main/loss'], x_key='epoch', file_name='loss.png', marker=""))
        trainer.extend(extensions.PlotReport(['main/accuracy', 'test/main/accuracy'], x_key='epoch', file_name='accuracy.png', marker=""))

        #trainer.extend(extensions.ExponentialShift('lr', lr_drop_ratio),trigger=(lr_drop_epoch, 'epoch'))

        trainer.run()

        paths = []

        for j in range(0, G):
            if j < 10:
                paths.append(np.asarray([os.path.join(os.path.join(test_dir, 'image2'), 'stage0'+str(j)+'\\'), j]))
            else:
                paths.append(np.asarray([os.path.join(os.path.join(test_dir, 'image2'), 'stage'+str(j)+'\\'), j]))

        (x_test, t_test), correct_test_label = image2Test(paths)

        #以下正確なaccuracyを求めるため
        test_loss = 0
        test_accuracy = 0
        miss_train_copy = miss_train_total.copy()
        miss_test_copy = miss_test_total.copy()
        correct_train_copy = correct_train_total.copy()
        correct_test_copy = correct_test_total.copy()

        for i in range(0, N_test, batch_size_predict):
            x = Variable(cuda.to_gpu(x_test[i:i+batch_size_predict]))#評価画像データ
            t = Variable(cuda.to_gpu(t_test[i:i+batch_size_predict]))#評価ラベル
            y = model.predict(x)
            model.zerograds()
            loss = F.softmax_cross_entropy(y, t)

            #こっから誤判別を特定するプログラム
            acc, data, data2, index, pred, corre, correct_index  = accuracy(y, t)
            acc.to_cpu()
            data.to_cpu()
            data2.to_cpu()
            index.to_cpu()
            pred.to_cpu()
            corre.to_cpu()
            correct_index.to_cpu()
            t.to_cpu()
            x.to_cpu()
            for i in range(len(data.data)):
                miss_test_total[data.data[i]] += 1
                miss_test_judge[data.data[i]][data2.data[i]] += 1

            for i in range(len(corre.data)):
                correct_test_total[corre.data[i]] += 1

            #こっからは誤判別した画像を出力するプログラム(最終更新時のみ)
            for inde in index.data:
                e = np.array(x.data[inde])
                b = np.array(e[0])#青色
                g = np.array(e[1])#緑色
                r = np.array(e[2])#赤色
                img = np.dstack((np.dstack((r,g)),b))

                #print(img)
                img = img * 255.0
                img += np.array([123.68, 116.779, 103.939], dtype=np.float32)
                img = img/255.0
                #print(img)
                plt.imsave(os.path.join(miss_judge_test_dir, str(cnt_trial)+'_trial\\'+str(n_epoch)+'\\'+str(cnt_test)+ '_epoch_' +str(n_epoch)+'_'+str(t.data[inde])+'to'+str(pred.data[inde])+'.jpg'), img)
                cnt_test += 1
            for correct_inde in correct_index.data:
                c_e = np.array(x.data[correct_inde])
                c_b = np.array(c_e[0])#青色
                c_g = np.array(c_e[1])#緑色
                c_r = np.array(c_e[2])#赤色
                c_img = np.dstack((np.dstack((c_r,c_g)),c_b))

                c_img = c_img * 255.0
                c_img += np.array([123.68, 116.779, 103.939], dtype=np.float32)
                c_img = c_img/255.0
                plt.imsave(os.path.join(correct_judge_test_dir, str(cnt_trial)+'_trial\\'+str(n_epoch)+'\\'+str(cnt_test_correct_judge)+'_epoch_'+str(n_epoch)+'_'+str(t.data[correct_inde])+'to'+str(pred.data[correct_inde])+'.jpg'), c_img)
                cnt_test_correct_judge += 1

            test_loss += loss.data*len(t.data)
            test_accuracy += acc.data*len(t.data)

            miss_test_each = miss_test_total - miss_test_copy
            correct_test_each = correct_test_total - correct_test_copy

        #ターミナル上に出力
        print("test acc:{:.04f}".format(test_accuracy/N_test))
        print('miss_test_total : ' + str(miss_test_total))
        print('miss_test_each :' + str(miss_test_each))
        print('correct_test_each :' + str(correct_test_each))
        print()

        # sheet.write(0, 1, n_epoch)#単純にエポック数(現時点)
        sheet.write(3, n_epoch, Decimal(test_accuracy/N_test).quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP))#評価用画像の判別率

        sheet.write((n_epoch + 7), 0, n_epoch)
        sheet.write((n_epoch + 7), G+3, n_epoch)
        for i in range(len(correct_test_each)):
            if correct_test_label[i] != 0:
                sheet.write((n_epoch + 7), (i + G+4), Decimal(correct_test_each[i]/correct_test_label[i]).quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP))

        #最終epochの誤判定時における、何が何を予測したのかを表示
        for i in range(0, G):
            for j in range(0,G):
                if j==i:
                    sheet.write((n_epoch+17+i), (i+G+4), correct_test_each[i])
                else:
                    sheet.write((n_epoch+17+i), (j+G+4), miss_test_judge[i][j])

        #ここから先は前後のラベルを含んだ判別率を求める．
        num_zengo = np.zeros(G)#前後を含んだ正解数
        for i in range(0, G):
            if i == 0:
                num_zengo[i] = correct_test_each[i] + miss_test_judge[i][i+1]
            elif i == (G-1):
                num_zengo[i] = correct_test_each[i] + miss_test_judge[i][i-1]
            else:
                num_zengo[i] = correct_test_each[i] + miss_test_judge[i][i-1] + miss_test_judge[i][i+1]

        #num_zengoの出力とラベルごとの前後を含んだ判別率の出力
        for i in range(len(num_zengo)):
            if correct_test_label[i] != 0:
                sheet.write((n_epoch + 8), (i + G + 4), num_zengo[i])
                sheet.write((n_epoch + 9), (i + G + 4), Decimal(num_zengo[i]/correct_test_label[i]).quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP))

        #最後に前後を含んだ評価用画像の判別率を算出し，出力
        num_zengo_total = 0
        for i in range(0, G):
            num_zengo_total += num_zengo[i]
        sheet.write(4, n_epoch, Decimal(num_zengo_total/N_test).quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP))

        cnt_trial+=1
        end_time = time.clock()
        sheet.write(0,0,end_time-start_time)
        print(end_time - start_time)

        serializers.save_npz(os.path.join(current_result_dir, 'model.npz'), model)
        # shutil.rmtree(train_dir)#必要ならData augmentaion後の教師画像フォルダの削除

        book.save(os.path.join(result_dir, 'result'+str(cnt_trial-1)+'_trial.xls'))

        change_excel(os.path.join(result_dir, 'result'+str(cnt_trial-1)+'_trial.xls'))
        os.remove(os.path.join(result_dir, 'result'+str(cnt_trial-1)+'_trial.xls'))

    excel_names = glob.glob(os.path.join(result_dir, "*.xlsx"))
    ordered_excel = natsorted(excel_names)

    outfile = os.path.join(result_dir, 'result.xlsx')
    wb2 = xl.Workbook()
    wb2.save(outfile)
    print(str(ordered_excel))
    for i, inputfile in enumerate(ordered_excel):
        print(str(i))
        sheettitle = str(i+1) + '_trial'
        print(sheettitle)
        print(str(inputfile))

        wb1 = xl.load_workbook(filename = inputfile)
        ws1 = wb1.worksheets[0]

        wb = xl.load_workbook(filename = outfile)
        if i > 0:
            wb.create_sheet()
        ws = wb.worksheets[i]
        ws.title = sheettitle

        for row in ws1:
            for cell in row:
                ws[cell.coordinate].value = cell.value

        wb.save(outfile)


import asyncio
from telegram import Update
from telegram.ext import ApplicationBuilder, ContextTypes, MessageHandler, filters, CommandHandler

TOKEN = '<token>'

ADMIN_IDS = [123, 0, ]

MAIN_ADMIN_ID = 0

async def do_echo(update: Update, context: ContextTypes.DEFAULT_TYPE):
    await update.message.reply_text(
        text="Your ID = {}\n\n{}".format(
            update.message.chat_id, update.message.text)
    )

def admin_access(f):

    async def inner(*args, **kwargs):
        update = args[0]
        if update and hasattr(update, 'message'):
            if update.message.chat_id in ADMIN_IDS:
                print("Access is allowed!")
                return await f(*args, **kwargs)
            else:
                print("Access is not allowed!")
        else:
            print('No update argument')

    return inner

def log_errors(f):

    async def inner(*args, **kwargs):
        try:
            return await f(*args, **kwargs)
        except Exception as e:
            error_message = f'[ADMIN] An error has occurred: {e}'
            print(error_message)

            update = args[0]
            context = args[1]
            if update and hasattr(update, 'message'):
                # Any error message is always sent to the main admin
                if context and hasattr(context, 'bot'):
                    await context.bot.send_message(
                        chat_id=MAIN_ADMIN_ID,
                        text=error_message,
                    )

                # Send an error message only if it happened to you
                # if update.message.chat_id in ADMIN_IDS:
                #     await update.message.reply_text(
                #         text=error_message
                #     )

            raise e

    return inner

@admin_access
@log_errors
async def secret_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
    await update.message.reply_text(text='secret!')
    rrrr -= 1

@log_errors
async def secret_command2(update: Update, context: ContextTypes.DEFAULT_TYPE):
    await update.xxxx.reply_text(text='secret 222!')

async def bot_get_me(app):
    print(await app.bot.get_me())

def main():
    app = ApplicationBuilder().token(TOKEN).connect_timeout(
        0.5).read_timeout(1.0).build()

    asyncio.ensure_future(bot_get_me(app))

    app.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, do_echo))
    app.add_handler(CommandHandler('secret', secret_command))
    app.add_handler(CommandHandler('secret2', secret_command2))

    app.run_polling()

if __name__ == '__main__':
    main()


from django.db import models

class Request(models.Model):
    method = models.CharField(max_length=16, verbose_name='Request Method')
    datetime = models.DateTimeField()
    date = models.DateField(null=True, blank=True)

    def __str__(self):
        return f"{self.method} request at {self.datetime}"

    # set the date field to the date of the datetime field
    def save(self, *args, **kwargs):
        self.date = self.datetime.date()
        super().save(*args, **kwargs)

    class Meta:
        ordering = ['-date']

        verbose_name = 'Request'
        verbose_name_plural = 'Requests'

        indexes = [
            models.Index(fields=['date']),
        ]

class RequestSummary(models.Model):
    date = models.DateField()
    count = models.IntegerField()

    def __str__(self):
        return f"{self.date} - {self.count}"

    class Meta:
        verbose_name = 'Request Summary'
        verbose_name_plural = 'Requests Summary'
        # index date
        indexes = [
            models.Index(fields=['date']),
        ]


import os
import math
import random
import pygame as pg

class Mars(pg.sprite.Sprite):
    """Planet that rotates and projects gravitational field."""
    basedir = os.path.dirname(__file__)
    mars_img_path = os.path.join(basedir, 'assets/mars.png')
    water_img_path = os.path.join(basedir, 'assets/mars_water.png')

    def __init__(self, settings):
        super().__init__()
        self.settings = settings
        self.img_mars = pg.image.load(self.mars_img_path).convert()
        self.img_water = pg.image.load(self.water_img_path).convert()
        self.image_copy = pg.transform.scale(self.img_mars, (100, 100))
        self.image_copy.set_colorkey(self.settings.black)
        self.rect = self.image_copy.get_rect()
        self.image = self.image_copy
        self.mass = 2000
        self.x = 400
        self.y = 320
        self.rect.center = (self.x, self.y)
        self.angle = math.degrees(0)
        self.rotate_by = math.degrees(0.01)

    def rotate(self):
        """Rotate the planet image with each game loop."""
        last_center = self.rect.center
        self.image = pg.transform.rotate(self.image_copy, self.angle)
        self.rect = self.image.get_rect()
        self.rect.center = last_center
        self.angle += self.rotate_by

    def gravity(self, satellite):
        """Calculate impact of gravity on satellite."""
        G = 1.0
        dist_x = self.x - satellite.x
        dist_y = self.y - satellite.y
        distance = math.hypot(dist_x, dist_y)
        # normalize to unit vector
        dist_x /= distance
        dist_y /= distance
        # apply gravity
        force = G * (satellite.mass * self.mass) / (math.pow(distance, 2))
        satellite.dx += (dist_x * force)
        satellite.dy += (dist_y * force)

    def update(self):
        """Rotate the planet."""
        self.rotate()


from __future__ import unicode_literals

import codecs
import json
import os.path
import re

FLAGS = re.UNICODE | re.IGNORECASE

# Some examples dates from the data
# =================================
#
# Birth dates
# ===========
# "* 1122"
# "* um 1480"
# "geb. 10.11.1810"
# "* 3o.9.1859"
#
# Death dates
# ===========
# "† 19.4.1967"
# "+ 4.6.1875"
# "gest. 02.05.1899"
# "hingerichtet: 02.02.1945"
# "gestorben 29.02.1980"
# "+ Januar 1944"
# "+ vermutlich am 9.7.1386"
#
# Special cases
# =============
# "+19.3.1884 Eppingen, + 12.4.1971 Eppingen"
# "* 3o.9.1859"
# "1843-1906"

MONTHS = '(?:Januar|Februar|März|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember)'

DATE_REGEXES = [
    # "23.10.1805", "4.9.1965", "24.12. 1930"
    r'(?:am\s+)?\d\d?\.\s*\d\d?\.\s*(\d\d\d\d?)',

    # "10. Dezember 1815"
    r'(?:am\s+)?\d\d?\.\s*' + MONTHS + r'\s*(\d\d\d\d?)',

    # "1122", "um 1480"
    r'(?:um)?\s*(\d\d\d\d?)',

    # "Januar 1944"
    MONTHS + r'\s*(\d\d\d\d?)',
]

# Allow 'o' as a digit
DATE_REGEXES = [r.replace(r'\d', r'(?:\d|o)') for r in DATE_REGEXES]

def extract_year(s, prefixes=None):
    """
    Try to extract a year from a string.

    ``s`` is the string to search in. ``prefixes`` is an optional list
    of patterns that may occur before the actual date.

    Dates must match one of the regexes in ``DATE_REGEXES`` (in
    combination with one of the prefixes if any are given).

    The year of the first date found is returned. If no matching date
    is found then ``None`` is returned.
    """
    prefixes = [unicode(p) for p in (prefixes or [''])]
    for pattern in DATE_REGEXES:
        for prefix in prefixes:
            p = prefix + r'\s*' + pattern
            m = re.search(p, s, FLAGS)
            if not m:
                continue
            return int(m.groups()[0].replace('o', '0'))

def extract_person_data(info):
    """
    Try to extract person data from info text.

    Returns a tuple containing the person's name and the years of birth
    and death. Each of these may be ``None`` if the extraction failed.
    """
    birth = extract_year(info, prefixes=[r'\*', r'geb\.'])
    death = extract_year(info, prefixes=[r'†', r'\+', r'gest\.', r'hingerichtet:',
                         r'gestorben'])
    name = None
    if birth or death:
        m = re.match(r'^([\w\s\-.]+)', info, FLAGS)
        if m:
            name = m.groups()[0].strip()
            if name.endswith(' geb'):
                # Birth name ("geboren")
                name = name[:-4]
            elif name.endswith(' gen'):
                # Nick name ("genannt")
                name = name[:-4]
            elif name.endswith('.'):
                if not name[-2] in 'IVX':
                    # Not a roman literal
                    name = name[:-1]
    return name, birth, death

def parse_entries(streets):
    """
    Parse street name data to extract additional information.
    """
    for street in streets.itervalues():
        person, birth, death = extract_person_data(street['info'])
        if person or birth or death:
            street['person'] = person
            street['birth'] = birth
            street['death'] = death

_GENDER_EXCEPTIONS = {
    'Blaise': 'm',
    'Edith': 'f',
    'Elisabeth': 'f',
    'Hedwig': 'f',
    'Henri': 'm',
    'Rahel': 'f',
    'Willi': 'm',
    'Willy': 'm',
    'Wiltraut': 'f',
}

def guess_gender(name):
    """
    Given a person's name guess their gender.

    Returns either ``'m'`` or ``'f'``.
    """
    names = name.split()
    if names[0] == 'Sankt':
        first_name = names[1]
    else:
        first_name = names[0]
    try:
        return _GENDER_EXCEPTIONS[first_name]
    except KeyError:
        pass
    if first_name[-1] in 'aeiy':
        return 'f'
    return 'm'

def guess_genders(streets):
    """
    Guess genders of persons.
    """
    for street in streets.itervalues():
        try:
            street['gender'] = guess_gender(street['person'])
        except KeyError:
            # Not a person
            pass

if __name__ == '__main__':
    HERE = os.path.dirname(os.path.abspath(__file__))
    SOURCE = os.path.join(HERE, 'raw_data.json')
    TARGET = os.path.join(HERE, 'names.json')

    with codecs.open(SOURCE, 'r', encoding='utf8') as f:
        streets = json.load(f)

    parse_entries(streets)

    def copy_person(src, dest):
        src_dict = streets[src]
        dest_dict = streets[dest]
        for k in ['birth', 'death', 'person']:
            dest_dict[k] = src_dict[k]

    def a_person(name, person, birth, death):
        streets[name]['person'] = person
        streets[name]['birth'] = birth
        streets[name]['death'] = death

    def not_a_person(name):
        del streets[name]['person']
        del streets[name]['birth']
        del streets[name]['death']

    # Manual fixes and additions
    #
    # The following changes are manual fixes for errors in the data, fixes
    # for special cases that are too rare to be worth being implemented,
    # and manual additions for missing data.
    streets['Englerstraße']['person'] = 'Karl Engler'
    a_person('Guntherstraße', 'Gundahar', None, 436)
    streets['Agathenstraße']['person'] = 'Agathe von Baden-Durlach'
    streets['Gerda-Krüger-Nieland-Straße']['person'] = 'Gerda Krüger-Nieland'
    copy_person('Tullaweg', 'Tullastraße')
    copy_person('Tullaweg', 'Tullaplatz')
    a_person('Petrus-Waldus-Straße', 'Petrus Waldus', None, None)
    a_person('Am Thomashäusle', 'Thomas Dorner', None, None)
    streets['Gritznerstraße']['person'] = 'Max Karl Gritzner'
    copy_person('Weinbrennerstraße', 'Weinbrennerplatz')
    streets['Martinstraße']['person'] = 'Sankt Martin'
    streets['Nikolausstraße']['person'] = 'Sankt Nikolaus'
    streets['Kaiserstraße']['person'] = 'Wilhelm I.'
    copy_person('Kaiserstraße', 'Kaiserpassage')
    copy_person('Kaiserstraße', 'Kaiserplatz')
    copy_person('Kaiserstraße', 'Kaiserallee')
    streets['Sepp-Herberger-Weg']['person'] = 'Joseph Herberger'
    streets['Mendelssohnplatz']['person'] = 'Moses Mendelssohn'
    copy_person('Martin-Luther-Straße', 'Martin-Luther-Platz')
    streets['Rolandplatz']['person'] = 'Roland'
    streets['Karlstraße']['person'] = 'Karl Ludwig Friedrich von Baden'
    not_a_person('Curjel-und-Moser-Straße')
    copy_person('Baumeisterstraße', 'Reinhard-Baumeister-Platz')
    streets['Laurentiusstraße']['person'] = 'Sankt Laurentius'
    not_a_person('Winkler-Dentz-Straße')
    not_a_person('Eichrodtweg')
    not_a_person('Bernhardstraße')
    a_person('Bernhardusplatz', 'Bernhard II. von Baden', 1428, 1458)
    not_a_person('Geschwister-Scholl-Straße')
    streets['Jakob-Dörr-Straße']['birth'] = 1884
    streets['Jakob-Dörr-Straße']['death'] = 1971
    a_person('Besoldgasse', 'Christoph Besold', None, None)
    streets['Ostendorfstraße']['death'] = 1915
    streets['Rudolfstraße']['person'] = 'Rudolf I. von Baden'
    not_a_person('Gebrüder-Bachert-Straße')
    copy_person('Friedrichsplatz', 'Alte Friedrichstraße')
    not_a_person('Haid-und-Neu-Straße')
    copy_person('Karl-Wilhelm-Straße', 'Karl-Wilhelm-Platz')
    copy_person('Fritz-Haber-Straße', 'Fritz-Haber-Weg')
    streets['Markusstraße']['person'] = 'Sankt Markus'
    streets['Luisenstraße']['person'] = 'Luise Marie Elisabeth von Preußen'
    copy_person('Hildastraße', 'Nördliche Hildapromenade')
    copy_person('Hildastraße', 'Südliche Hildapromenade')
    streets['Ernst-Friedrich-Straße']['person'] = 'Ernst Friedrich'
    streets['Margarethenstraße']['person'] = 'Margarethe Margräfin von Baden'
    streets['Ludwig-Wilhelm-Straße']['person'] = 'Ludwig Wilhelm Prinz von Baden'
    copy_person('Werderstraße', 'Werderplatz')
    streets['Ada-Lovelace-Straße']['person'] = 'Ada Lovelace'
    not_a_person('Bertholdstraße')
    streets['Karl-Friedrich-Straße']['person'] = 'Karl Friedrich von Baden'
    streets['Stephanstraße']['person'] = 'Heinrich von Stephan'
    copy_person('Stephanstraße', 'Stephanplatz')
    streets['Huttenstraße']['person'] = 'Ulrich Reichsritter von Hutten'
    a_person('Schultheiß-Kiefer-Straße','Erhard Kiefer', None, None)
    copy_person('Lützowstraße', 'Lützowplatz')
    streets['Sankt-Florian-Straße']['person'] = 'Sankt Florian'
    a_person('Blankenhornweg', 'Adolph Blankenhorn', 1843, 1906)
    copy_person('Karlstraße', 'Karlstor')
    copy_person('Brahmsstraße', 'Brahmsplatz')
    copy_person('Hermann-Löns-Weg', 'Lönsstraße')
    not_a_person('Gebrüder-Grimm-Straße')
    streets['Hubertusallee']['person'] = 'Hubrtus von Lüttich'
    streets['Philippstraße']['person'] = 'Philipp I. von Baden'
    streets['Sankt-Valentin-Platz']['person'] = 'Sankt Valentin von Terni'
    streets['Gebhardstraße']['person'] = 'Gebhard III. von Zähringen'
    a_person('Rosalienberg', 'Rosalie Lichtenauer', None, None)
    streets['Charlottenstraße']['person'] = 'Anna Charlotte Amalie von Nassau-Dietz-Oranien'
    streets['Sankt-Barbara-Weg']['person'] = 'Sankt Barbara'
    a_person('Graf-Konrad-Straße', 'Konrad I. von Kärnten', 975, 1011)
    not_a_person('Christofstraße')
    streets['Hennebergstraße']['person'] = 'Berthold von Hohenberg'
    a_person('Winkelriedstraße', 'Arnold von Winkelried', None, 1386)
    streets['Viktoriastraße']['person'] = 'Viktoria von Baden'
    a_person('Schultheißenstraße', 'Bernhard Metz', None, None)
    streets['Sophienstraße']['person'] = 'Sophie Wilhelmine von Schleswig-Holstein-Gottorf'
    streets['Augustastraße']['person'] = 'Augusta von Sachsen-Weimar-Eisenach'
    streets['Karolinenstraße']['person'] = 'Karoline von Baden'
    copy_person('Bismarckstraße', 'Kanzlerstraße')
    not_a_person('Friedenstraße')
    streets['Marie-Alexandra-Straße']['person'] = 'Marie Alexandra von Baden'
    streets['Erbprinzenstraße']['person'] = 'Karl Ludwig von Baden'
    streets['Hauckstraße']['year'] = 1950
    streets['Goethestraße'].update({
        'person': 'Johann Wolfgang von Goethe',
        'birth': 1749,
        'death': 1832,
    })
    streets['Ernststraße']['person'] = 'Ernst I. von Baden-Durlach'
    streets['Kronprinzenstraße']['person'] = 'Friedrich Wilhelm Victor August Ernst von Preußen'

    guess_genders(streets)

    # TODO: The following streets are named after several people. The current
    # data format cannot describe that properly:
    #
    #    Eichrodtweg
    #    Winkler-Dentz-Straße
    #    Curjel-und-Moser-Straße
    #    Bernhardstraße
    #    Geschwister-Scholl-Straße
    #    Gebrüder-Bachert-Straße
    #    Bertholdstraße
    #    Peter-und-Paul-Platz
    #    Engler-Bunte-Ring
    #    Gebrüder-Grimm-Straße
    #    Christofstraße

    with codecs.open(TARGET, 'w', encoding='utf8') as f:
        json.dump(streets, f, sort_keys=True, indent=4, separators=(',', ': '))


import socket
import os
import datetime
import time
HOST = '192.168.2.50'
PORT = 8485
ip_port=(HOST,PORT)

filename = "ghijkl"

socket1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket1.connect((HOST, PORT))

socket1.sendto(filename.encode(encoding='utf-8'),ip_port)
source=""
with open('test.txt', 'wb') as file_to_write:
    while True:
        data = socket1.recv(1024)
        if not data:
            break
        file_to_write.write(data)
        time.sleep(1)
        print(data)
        source+=data.decode()
    file_to_write.close()
    print(datetime.datetime.now())
socket1.close()


# coding: UTF-8
"""
Created on 2021/11/27

@author: Mark Hsu
"""
from shop.models import Item, Image, Category
from shop.Serializer import ItemSerializer
from rest_framework import status
from rest_framework.views import APIView
from rest_framework.response import Response
from django.http import Http404, HttpResponse
import PIL.Image as PImage
import copy

def save_images(request, item):
    def is_valid_image(f):
        try:
            with PImage.open(f):
                return True
        except IOError:
            return False

    response_message = {'images': []}
    files = request.FILES.getlist('images')

    PImage.init()
    for file in files:
        content_type = file.content_type.split('/')
        f = copy.deepcopy(file)
        if not (content_type[0] == 'image' and
                content_type[1].upper() in PImage.EXTENSION.values() and
                is_valid_image(f)):
            response_message['images'].append({file.name: 'Invalid file type.'})
        else:
            img = Image.objects.create(item=item, image=bytes(file.read()))
            img.save()
            response_message['images'].append({file.name: 'Successful uploaded.'})
    return response_message

def check_category(request):
    response_message = {}
    category = None
    try:
        category = Category.objects.get(name=request.data['category'])
    except Category.DoesNotExist:
        response_message.update({'category': ['Category not found.']})
    except KeyError:
        response_message.update({'category': ['This field is required.']})

    return category, response_message

class ItemList(APIView):
    def get(self, request):
        items = Item.objects.all()
        serializer = ItemSerializer(items, many=True)
        return Response(serializer.data)

    def post(self, request):
        if not request.user.is_staff:
            return Response({'Permission': 'Denied'}, status.HTTP_401_UNAUTHORIZED)

        item_serializer = ItemSerializer(data=request.data)
        category, response_message = check_category(request)

        if item_serializer.is_valid():
            if category is not None:
                item = item_serializer.save(category=category)
                response_message.update(dict(item_serializer.data))
                response_message.update(save_images(request, item))
            return Response(response_message, status=status.HTTP_201_CREATED)
        response_message.update(item_serializer.errors)
        return Response(response_message, status=status.HTTP_400_BAD_REQUEST)

class ItemSpecific(APIView):
    def get_object(self, item_id):
        try:
            return Item.objects.get(id=item_id)
        except Item.DoesNotExist:
            raise Http404

    def get(self, request, item_id):
        item = self.get_object(item_id)
        serializer = ItemSerializer(item)
        return Response(serializer.data)

    def put(self, request, item_id):
        if not request.user.is_staff:
            return Response({'Permission': 'Denied'}, status.HTTP_401_UNAUTHORIZED)
        item = self.get_object(item_id)
        serializer = ItemSerializer(item, data=request.data, partial=True)
        if serializer.is_valid():
            category, msg = check_category(request)
            if category is None:
                serializer.save()
            else:
                serializer.save(category=category)
            response_message = dict(serializer.data)
            response_message.update(save_images(request, item))
            if msg:
                response_message.update(msg)
            return Response(response_message)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, item_id):
        if not request.user.is_staff:
            return Response({'Permission': 'Denied'}, status.HTTP_401_UNAUTHORIZED)
        item = self.get_object(item_id)
        item.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

class ImageSpecific(APIView):
    def get(self, request, image_id):
        image = Image.objects.get(id=image_id)
        return HttpResponse(bytes(image.image), content_type='image/png')

    def delete(self, request, image_id):
        if not request.user.is_staff:
            return Response({'Permission': 'Denied'}, status.HTTP_401_UNAUTHORIZED)
        image = Image.objects.get(id=image_id)
        image.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)


import pygraphviz as pgv

import gol

gol.init()
TENSOR_GRAPH_ATTR = dict(fontname="Times-Roman", fontsize=14, shape="polygon",
                         style="rounded", color="black",
                         fixedsize=False)
PARA_GRAPH_ATTR = dict(fontname="Times-Roman", fontsize=14, shape="polygon",
                       style="filled", fillcolor="#B4E7B7",
                       fixedsize=False)

OP_GRAPH_ATTR = dict(fontname="Times-Roman", fontsize=14, shape="circle",
                     style="filled", fillcolor="#BBE4FF",
                     fixedsize=False)

G = pgv.AGraph(directed=True, rankdir="RL", overlap=False, normlized=True,
               encoding='UTF-8')
TENSOR_DICT = {}
PARA_DICT = {}
OP_DICT = {}
# GRAPH_FLAG = True

gol.set_value('TENSOR_GRAPH_ATTR', TENSOR_GRAPH_ATTR)
gol.set_value('PARA_GRAPH_ATTR', PARA_GRAPH_ATTR)
gol.set_value('OP_GRAPH_ATTR', OP_GRAPH_ATTR)
gol.set_value('G', G)
gol.set_value('TENSOR_DICT', TENSOR_DICT)
gol.set_value('PARA_DICT', PARA_DICT)
gol.set_value('OP_DICT', OP_DICT)
gol.set_value('GRAPH_FLAG', True)

import argparse
import os

import numpy as np
import nni
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader
from torchvision import transforms
from tqdm import trange
import np_autograd as anp
from dataloader import Mnist

from utils import AverageMeter

from model_mlp_my_impl import MLP

import sys

sys.setrecursionlimit(3000)

def train(train_loader: DataLoader, network: anp.Layer, criterion: anp.Layer, optimizer: anp.Optimizer) -> float:
    # 损失函数平均器
    losses = AverageMeter()

    # 模型设置为训练模式
    network.train()
    # 迭代训练集
    for data, label in train_loader:
        # 将数据转换为张量
        data = anp.Tensor(data.reshape(-1, 28 * 28).numpy(), requires_grad=True, name='data')
        label = anp.Tensor(label.numpy(), requires_grad=True, name='label')

        # 前向传播
        output = network(data)
        loss = criterion(output, label)
        # 更新损失函数平均器
        losses.update(loss.data.mean())
        if gol.get_value('GRAPH_FLAG'):
            G.layout()
            G.draw('graph.png')

        # 反向传播
        loss.backward(np.ones_like(loss.data))

        # for k, v in TENSOR_DICT.items():
        #     if v.grad is not None:
        #         G.add_node(f'{v.id}_grad', label=f'grad: {v.grad}', **TENSOR_GRAPH_ATTR)
        #         G.add_edge(v.id, f'{v.id}_grad', label=f'grad', **TENSOR_GRAPH_ATTR)
        #         # G.add_edge(v.id, f'{v.id}_grad')

        # 调用优化器，更新模型参数
        optimizer.step()
        # 清空梯度
        optimizer.zero_grad()

        gol.set_value('GRAPH_FLAG', False)

    # print(f'losses: {losses.avg}')
    # 返回损失函数平均值
    return losses.avg

def valid(val_loader:DataLoader, network: anp.Layer) -> float:
    avg_acc = AverageMeter()

    network.eval()

    for data, label in val_loader:
        data = anp.Tensor(data.reshape(-1, 28 * 28).numpy())
        label = anp.Tensor(label.numpy())

        output = network(data)

        acc = np.mean(np.argmax(output, axis=-1) == label)

        avg_acc.update(acc)

    return avg_acc.avg

if __name__ == '__main__':
    # 定义超参数
    epochs = 100
    batch_size = 512
    train_size = 1024
    test_size = 256
    lr = 0.001
    # momentum = 0.2
    eval_freq = 1
    # save_dir = './saved_models'
    num_workers = 0

    # 定义数据集
    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])

    # train_set = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)
    # test_set = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform)
    # transform = transforms.Compose([transforms.ToTensor()])
    # train_set = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)
    # test_set = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform)
    train_set = Mnist(root='./data', train=True, download=True, transform=transform, size=train_size)
    test_set = Mnist(root='./data', train=False, download=True, transform=transform, size=test_size)

    # 定义数据加载器
    train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, drop_last=True, num_workers=num_workers)
    test_loader = DataLoader(test_set, batch_size=test_size, shuffle=True, num_workers=num_workers)

    # 定义模型
    model = MLP(784, 10)
    print(model.__repr__())

    # 定义损失函数
    criterion = anp.CrossEntropyLoss()
    # optimizer = anp.SGD(list(model.parameters()), lr=lr, momentum=momentum)

    # 定义优化器
    optimizer = anp.Adam(list(model.parameters()), lr=lr)

    # 定义训练过程
    writer = SummaryWriter(log_dir=f'logs/my_impl/adam_bs_{batch_size}_lr_{lr}')
    best_acc = 0.
    for epoch in trange(epochs):
        # 训练
        loss = train(train_loader, model, criterion, optimizer)

        writer.add_scalar('train_loss', loss, epoch)

        # 评估
        if epoch % eval_freq == 0:
            avg_acc = valid(test_loader, model)

            writer.add_scalar('valid_acc', avg_acc, epoch)

            nni.report_intermediate_result(float(avg_acc))
            # 更新最优精度
            if avg_acc > best_acc:
                best_acc = avg_acc

            writer.add_scalar('best_acc', best_acc, epoch)
    nni.report_final_result(float(best_acc))


import mplcursors
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt, cm
from matplotlib.colors import Normalize
from pyod.models.knn import KNN
from sklearn import preprocessing
from sklearn.metrics import mean_squared_error

df = pd.read_csv("data/3D_spatial_network.csv").sample(n=500).reset_index(drop=True)
x = df.drop('alt', axis=1)
y = df.alt
min_max_scaler = preprocessing.MinMaxScaler()
x_train_scaled = min_max_scaler.fit_transform(x)
grid = np.linspace(0, 1, 80)
x_, y_ = np.meshgrid(grid, grid)
x_test_scaled = np.vstack([x_.ravel(), y_.ravel()]).T

clf = KNN(n_neighbors=10)
clf.fit(x_train_scaled)
distance, neighbours = clf.get_neighbours(x_test_scaled)
y_test_scores = clf.decision_function(x_test_scaled)

f3_liberal = []
f3_conservative = []
uncertainty = []
outlierness = []

for idx, val in enumerate(x_test_scaled):
    neighborhood = [y[i] for i in neighbours[idx]]
    avg = np.ones(10) * np.mean(neighborhood)
    f1 = 10 * mean_squared_error(neighborhood, avg)
    f2 = y_test_scores[idx]
    # f3_conservative.append(f1 + f2 - (f1 * f2))
    # f3_liberal.append(f1 * f2)
    uncertainty.append(f1)
    outlierness.append(f2)

norm = np.linalg.norm(uncertainty)
uncertainty = uncertainty / norm
norm = np.linalg.norm(outlierness)
outlierness = outlierness / norm
f1 = uncertainty
f2 = outlierness

# f1 = stats.zscore(f1)
# f2 = stats.zscore(f2)
# print(f1)
# print(f2)

for i in range(len(uncertainty)):
    f3_conservative.append(f1[i] + f2[i] - (f1[i] * f2[i]))
    f3_liberal.append(f1[i] * f2[i])

# Plot
minima = np.min(f3_liberal)
maxima = np.max(f3_liberal)
norm = Normalize(vmin=minima, vmax=maxima, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap=cm.RdYlGn_r)

color = []
for v in f3_liberal:
    color.append(mapper.to_rgba(v))
fig, axs = plt.subplots(1, 2, figsize=(12, 6))
scatter = axs[0].scatter([item[0] for item in x_test_scaled], [item[1] for item in x_test_scaled],
                         color=[item for item in color], s=2)
axs[0].set_aspect('equal', adjustable='box')
axs[0].title.set_text("Liberal")
cursor = mplcursors.cursor(scatter, hover=True)

@cursor.connect("add")
def on_add(sel):
    sel.annotation.set(text=f3_liberal[sel.target.index])

minima = np.min(f3_conservative)
maxima = np.max(f3_conservative)
norm = Normalize(vmin=minima, vmax=maxima, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap=cm.RdYlGn_r)
color = []
for v in f3_conservative:
    color.append(mapper.to_rgba(v))
scatter = axs[1].scatter([item[0] for item in x_test_scaled], [item[1] for item in x_test_scaled],
                         color=[item for item in color], s=2)
axs[1].set_aspect('equal', adjustable='box')
axs[1].title.set_text("Conservative")
cursor = mplcursors.cursor(scatter, hover=True)

@cursor.connect("add")
def on_add(sel):
    sel.annotation.set(text=f3_conservative[sel.target.index])

plt.show()

minima = np.min(outlierness)
maxima = np.max(outlierness)
norm = Normalize(vmin=minima, vmax=maxima, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap=cm.RdYlGn_r)

color = []
for v in outlierness:
    color.append(mapper.to_rgba(v))
fig, axs = plt.subplots(1, 2, figsize=(12, 6))
scatter = axs[0].scatter([item[0] for item in x_test_scaled], [item[1] for item in x_test_scaled],
                         color=[item for item in color], s=2)
axs[0].set_aspect('equal', adjustable='box')
axs[0].title.set_text("Only Outlierness")
cursor = mplcursors.cursor(scatter, hover=True)

@cursor.connect("add")
def on_add(sel):
    sel.annotation.set(text=outlierness[sel.target.index])

minima = np.min(uncertainty)
maxima = np.max(uncertainty)
norm = Normalize(vmin=minima, vmax=maxima, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap=cm.RdYlGn_r)
color = []
for v in uncertainty:
    color.append(mapper.to_rgba(v))
scatter = axs[1].scatter([item[0] for item in x_test_scaled], [item[1] for item in x_test_scaled],
                         color=[item for item in color], s=2)
axs[1].set_aspect('equal', adjustable='box')
axs[1].title.set_text("Only Uncertainty")
cursor = mplcursors.cursor(scatter, hover=True)

@cursor.connect("add")
def on_add(sel):
    sel.annotation.set(text=uncertainty[sel.target.index])

plt.show()

plt.scatter(x_train_scaled[:, 0], x_train_scaled[:, 1], s=10)
plt.show()


import pandas as pd
from nltk import re

df = pd.read_csv("Dataset/covid19_tweets.csv", usecols=['text'])
df = df.sample(frac = 0.001, replace = False, random_state=42)      # Take %0.5 of total reviews
print(df.describe())

for tweet in df['text']:
    print("\n", tweet)
    tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))', '', tweet)  # Remove links
    tweet = re.sub('((@[^\s]+)|(#[^\s]+))', '', tweet)              # Remove Mension and Hashtag
    tweet = re.sub('[\s]+', ' ', tweet)                             # Remove additional white spaces
    # tweet = tweet.strip('\'"')     # trim
    print(tweet)


# YOLOv5 ðŸš€ by Ultralytics, GPL-3.0 license
import tkinter
import tkinter.messagebox
import customtkinter
from PIL import Image,ImageTk
import os
import tkinter as tk
from tkinter import filedialog
import cv2
import time
from speednotification import *
import tkinter
import tkinter.messagebox
import customtkinter
from PIL import Image,ImageTk
import os
import tkinter as tk
from tkinter import filedialog
"""
Run inference on images, videos, directories, streams, etc.

Usage - sources:
    $ python path/to/detect.py --weights yolov5s.pt --source 0              # webcam
                                                             img.jpg        # image
                                                             vid.mp4        # video
                                                             path/          # directory
                                                             path/*.jpg     # glob
                                                             'https://youtu.be/Zgi9g1ksQHc'  # YouTube
                                                             'rtsp://example.com/media.mp4'  # RTSP, RTMP, HTTP stream

Usage - formats:
    $ python path/to/detect.py --weights yolov5s.pt                 # PyTorch
                                         yolov5s.torchscript        # TorchScript
                                         yolov5s.onnx               # ONNX Runtime or OpenCV DNN with --dnn
                                         yolov5s.xml                # OpenVINO
                                         yolov5s.engine             # TensorRT
                                         yolov5s.mlmodel            # CoreML (macOS-only)
                                         yolov5s_saved_model        # TensorFlow SavedModel
                                         yolov5s.pb                 # TensorFlow GraphDef
                                         yolov5s.tflite             # TensorFlow Lite
                                         yolov5s_edgetpu.tflite     # TensorFlow Edge TPU
"""

import argparse
from audioop import bias
import os
import sys
from pathlib import Path
import numpy as np
from pandas import concat

# os.chdir("./yolov5")

# print(os.listdir())

# print(os.getcwd())
import torch
import torch.backends.cudnn as cudnn

FILE = Path(__file__).resolve()
ROOT = FILE.parents[0]  # YOLOv5 root directory
if str(ROOT) not in sys.path:
    sys.path.append(str(ROOT))  # add ROOT to PATH
ROOT = Path(os.path.relpath(ROOT, Path.cwd()))  # relative

from models.common import DetectMultiBackend
from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadStreams
from utils.general import (LOGGER, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2,
                           increment_path, non_max_suppression, print_args, scale_coords, strip_optimizer, xyxy2xywh)
from utils.plots import Annotator, colors, save_one_box
from utils.torch_utils import select_device, time_sync
#from PyQt5.QtGui import *

@torch.no_grad()
class Detector:
    def __init__(self):
        self.weights="weights/best.pt"  # model.pt path(s)
        self.source='vid.mp4' # file/dir/URL/glob, 0 for webcam
        # print(type(frame))
        # self.source = frame
        # self.frame = frame
        # self

        # ball
        # bat
        # wicket
        # pitch
        # shoe
        self.currBallXAxis = 0
        self.data='data/coco128.yaml'  # dataset.yaml path
        self.imgsz=(640, 480)  # inference size (height, width)
        self.conf_thres=0.25 # confidence threshold
        self.iou_thres=0.45 # NMS IOU threshold
        self.max_det=1000  # maximum detections per image
        self.device= 0   # cuda device, i.e. 0 or 0,1,2,3 or cpu
        self.view_img=True  # show results
        self.save_txt=False  # save results to *.txt
        self.save_conf=False  # save confidences in --save-txt labels
        self.save_crop=False   # save cropped prediction boxes
        self.nosave=False  # do not save images/videos
        self.classes=None  # filter by class: --class 0, or --class 0 2 3
        self.agnostic_nms=False  # class-agnostic NMS
        self.augment=False  # augmented inference
        self.visualize=False  # visualize features
        self.update=False  # update all models
        self.project=ROOT / 'runs/detect'  # save results to project/name
        self.name='exp'  # save results to project/name
        self.exist_ok=False  # existing project/name ok, do not increment
        self.line_thickness=3  # bounding box thickness (pixels)
        self.hide_labels=False  # hide labels
        self.hide_conf=False  # hide confidences
        self.half = False  # use FP16 half-precision inference
        # print(self.half)
        self.dnn=False  # use OpenCV DNN for ONNX inference

        self.stride = None
        self.pt = None
        self.model= None
        self.bs = None
        self.dataset = None
        self.names = None
        self.pred = None

        self.classifcation_point = {}
        self.ballFrames = 0
        self.FlagFrameStart = False
        self.FlagFrameStop = False
        self.previou = 0
        self.videoCapFlag = False
        self.videoCapCounter = 0

        self.ballspeed = 0

    def extractDetectorInformation(self, detections):
        self.classifcation_point = {}

        # print(detections)
        # print(detections.shape[1] == 6)
        # print(len(detections.size()))
        if detections.shape[1] == 6:
            # detections = detections.cpu().detach().numpy()

            for x1, y1, x2, y2, cls_conf, cls_pred in detections:

                cls_pred = int(cls_pred)

                if cls_pred not in self.classifcation_point.keys():
                    self.classifcation_point[cls_pred] = []

                self.classifcation_point[int(cls_pred)].append([x1, y1, x2, y2])

    def load_model(self,video_player_object):

        vidcap = cv2.VideoCapture(video_player_object.current_file_path)
        self.fps = vidcap.get(cv2.CAP_PROP_FPS)

        print(self.fps)
        self.device = select_device(self.device)

        self.model = DetectMultiBackend(self.weights, device=self.device, dnn=self.dnn, data=self.data, fp16=self.half)
        self.stride, self.names, self.pt = self.model.stride, self.model.names, self.model.pt
        self.imgsz = check_img_size(self.imgsz, s=self.stride)  # check image size

    def data_loader(self):
        # self.dataset = LoadImages(self.source, img_size=self.imgsz, stride=self.stride, auto=self.pt)
        self.bs = 1  # batch_size
        vid_path, vid_writer = [None] * self.bs, [None] * self.bs
        self.model.warmup(imgsz=(1, 3, *self.imgsz))

    def inference(self,frame,video_player_obj):
        # self.model.warmup(imgsz=(1, 3, *self.imgsz))  # warmup
        seen, windows, dt = 0, [], [0.0, 0.0, 0.0]
        temp=frame
        self
        # for path, im, im0s, vid_cap, s in self.dataset:

        # self.frame.size()
        t1 = time_sync()
        frame = cv2.resize(frame,(640,480))
        im = torch.from_numpy(frame).to(self.device)
        im = im.half() if self.model.fp16 else im.float()  # uint8 to fp16/32
        im /= 255  # 0 - 255 to 0.0 - 1.0
        if len(im.shape) == 3:
            im = im[None]  # expand for batch dim
        t2 = time_sync()
        dt[0] += t2 - t1

        # Inference
        # cv2.resize(im,(640,480))
        # visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False

        im = im.permute(0,3,1,2)
        # print(im.shape)
        self.pred = self.model(im, augment=self.augment, visualize=self.visualize)

        t3 = time_sync()

        dt[1] += t3 - t2
        # NMS

        self.pred = non_max_suppression(self.pred, self.conf_thres, self.iou_thres, self.classes, self.agnostic_nms, max_det=self.max_det)
        im0 = frame.copy()
        ballspeed = self.logic()

        if ballspeed > 1:
            self.ballspeed = round(ballspeed)
            # self.ballspeed = round()
            self.videoCapFlag = True

        if self.videoCapFlag:
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(temp,"Ball Speed="+str(self.ballspeed),(100, 50),font, 1,(0, 0, 255),1,cv2.LINE_4)
            video_player_obj.ball_speed_label=customtkinter.CTkButton(video_player_obj,width=5,text="Ball Speed = "+str(self.ballspeed)+' km/hr',font=customtkinter.CTkFont(size=20, weight="bold"))
            video_player_obj.ball_speed_label.grid(row=3, column=2,columnspan=3,padx=(20,20),pady=(20,20))
            #speed_notification=Notification(video_player_obj,str(self.ballspeed),timeout=5000)

            self.videoCapCounter += 1

        if self.videoCapCounter == 30:
            self.videoCapCounter = 0
            self.videoCapFlag= False

        dt[2] += time_sync() - t3
        # Second-stage classifier (optional)
        # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s)

        # Process predictions
        for i, det in enumerate(self.pred):  # per image
            seen += 1

            # p, im0, frame = path, im0s.copy(), getattr(self.dataset, 'frame', 0)

            # p = Path(p)  # to Path
            # save_path = str(save_dir / p.name)  # im.jpg
            # txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}')  # im.txt
            # s += '%gx%g ' % im.shape[2:]  # print string
            gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]  # normalization gain whwh
            imc = im0.copy() if self.save_crop else im0  # for save_crop
            annotator = Annotator(im0, line_width=self.line_thickness, example=str(self.names))
            if len(det):
                # Rescale boxes from img_size to im0 size
                det[:, :4] = scale_coords(im.shape[2:], det[:, :4], im0.shape).round()
                # Write results
                for *xyxy, conf, cls in reversed(det):

                    c = int(cls)  # integer class
                    label = None if self.hide_labels else (self.names[c] if self.hide_conf else f'{self.names[c]} {conf:.2f}')
                    annotator.box_label(xyxy, label, color=colors(c, True))

            # Stream results
            im0 = annotator.result()
            #im0 = cv2.resize(im0,(640,480))
           #cv2.imshow("Cricket Frames", im0)
            #cv2.waitKey(1)
            frame = cv2.resize(im0, (750, 550))
            img = cv2.cvtColor(temp, cv2.COLOR_BGR2RGB)
            img = Image.fromarray(img)
            img = ImageTk.PhotoImage(img)
            video_player_obj.inference_label.configure(image=img)
            video_player_obj.inference_label.image = img
            video_player_obj.update()
            cv2.imwrite("frame.jpg",temp)
            #video_player_obj.pixmap = QPixmap('frame.jpg')
            #video_player_obj.label.setPixmap(video_player_obj.pixmap)
            cv2.waitKey(1)

    def logic(self):
        pred = self.pred[0].detach().cpu().numpy()
        self.extractDetectorInformation(pred)
        # print(self.classifcation_point)
        # print("##")

        pitchLengthInKM = 0.0201168
        # timeInHOURS = sec/360

        # print(c3d_pred)
        if 3 in self.classifcation_point.keys():

            # print("here in logic")
            if 0 in self.classifcation_point.keys():
                ballBox = self.classifcation_point[0][0]
            else:
                ballBox = [0,0,0,0]
            self.prevBallXAxis = self.currBallXAxis
            self.currBallXAxis= ballBox[0]

            # print(self.currBallXAxis," AND ", self.prevBallXAxis,self.ballFrames)
            pitchBox = self.classifcation_point[3][0]
            # print(ballBox)
            # print(ballBox)
            # self.ballFrames += 1
            iou = self.handAndObjectIOUWithRespecToHandObject(ballBox,pitchBox)
            Ballspeed = 0
            # print(iou)
            # print(self.previou, ":::",iou)
            if self.previou == 0 and iou > 0.5:
                self.FlagFrameStart = True
                # self.ballFrames += 1
            elif self.previou > 0.5 and iou == 0:
                # print("in ELIFF")
                self.FlagFrameStart= False
                # self.ballFrames = 0

            # print(self.ballFrames)
            if self.ballFrames > 32:
                self.ballFrames = 0
            # if self.FlagFrameStart:
            if self.FlagFrameStart and abs(self.currBallXAxis - self.prevBallXAxis) < 10:
                self.ballFrames += 1
            else:
                if self.ballFrames != 0 and self.ballFrames>8:

                    # if self.ballFrames == 11:
                    #     self.ballFrames += 1
                    # if self.ballFrames == 10:
                    #     self.ballFrames += 2

                        # print(self.ballFrames)
                    timeInSec = (self.ballFrames+3)/self.fps
                    # print(timeInSec)
                    timeInHours = timeInSec/3600
                    # print(timeInHours)
                    Ballspeed = pitchLengthInKM/timeInHours

                    print(Ballspeed , " Kmp/hr")
                    self.ballFrames = 0
                # else:
                #     self.ballFrames = 0

            self.previou = self.handAndObjectIOUWithRespecToHandObject(ballBox,pitchBox)
            return Ballspeed

        return 0

        # for cls in self.classifcation_point.keys():
        #     if

    def handAndObjectIOUWithRespecToHandObject(self, hand, object):
        xA = max(hand[0], object[0])
        yA = max(hand[1], object[1])
        xB = min(hand[2], object[2])
        yB = min(hand[3], object[3])

        inter_area = max(0, xB - xA + 1) * max(0, yB - yA + 1)

        hand_area = (hand[2] - hand[0] + 1) * (hand[3] - hand[1] + 1)
        object_area = (object[2] - object[0] + 1) * (object[3] - object[1] + 1)

        iou = inter_area / float(hand_area)
        return iou

def run_detector(video_player_obj):
    Detect = Detector()
    Detect.load_model(video_player_obj)
    Detect.data_loader()
    cap = cv2.VideoCapture(video_player_obj.current_file_path)

    while(cap.isOpened()):
        suc,frame = cap.read()
        if suc:
            Detect.inference(frame,video_player_obj)
        else:
            break
    # print("")
    # Detect.logic()

#main(None)

# if __name__ == "__main__":
#     # opt = parse_opt()
#     main(video_player_obj)


from pathlib import Path
import json
import logging
import os
from typing import Dict, Iterator
from composer.efile.xmlio import JsonTranslator
from concurrent.futures import ProcessPoolExecutor

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

root_dir: str = "/Volumes/Bulk/efile"
xml_dir: str = os.path.join(root_dir, "xml")
json_dir: str = os.path.join(root_dir, "json")

translate: JsonTranslator = JsonTranslator()

# This works fast. Why does filings.py work slow?
def convert(xml_path: Path):
    xml_fn: str = str(xml_path)
    json_relpath: str = "/".join(xml_fn.split("/")[-3:]).split(".")[0] + ".json"
    json_fn: str = os.path.join(json_dir, json_relpath)
    json_dirpath: str = "/".join(json_fn.split("/")[:-1])
    os.makedirs(json_dirpath, exist_ok=True)
    with open(xml_fn) as xml_fh:
        raw_xml = xml_fh.read()
        if raw_xml.strip() == "":
            logging.error("%s is empty. Skipping." % xml_fn)
            return
    with open(json_fn, "w") as json_fh:
        as_json: Dict = translate(raw_xml)
        json.dump(as_json, json_fh)

xml_paths: Iterator[Path] = Path(xml_dir).glob("**/*.xml")

with ProcessPoolExecutor() as executor:
    executor.map(convert, xml_paths)


# -*- coding: utf-8 -*-
"""
Created on Thu Nov 28 15:11:05 2019

@author: Allen
"""
import os
#import tensorflow as tf
import numpy as np
#import random
import csv
import math
#import matplotlib.pyplot as ma
#import keras
#from keras.utils import np_utils
#from keras.models import Sequential
#from keras.layers import Dense,Dropout,Activation,Flatten
#from keras.layers import Conv1D,MaxPooling1D,ZeroPadding1D,normalization
import pywt
import itertools
import tkinter as tk
from tkinter import filedialog
import tkinter.messagebox
import matplotlib.pyplot as plt
import pandas as pd
import time
import TimeDomain as TD

def open_file():
    file_path = filedialog.askdirectory()
    if file_path:
        file_path_text.set(''+file_path)
        list_file(file_path)
    else:
        return None

def list_file(path):
    if path != '':
        listbox.delete(0,tk.END)
        list_files = os.listdir(path)
        for file in list_files:
            listbox.insert(tk.END, file)
    else:
        tk.messagebox.showerror(title = 'Oops', message = 'Please select a file path')

def export_file():
    return 0

def check_parameter(input_path):
    if input_path == '':
        tk.messagebox.showerror(title = 'Oops', message = 'Please select a file path')
    elif output_label_str.get() == '':
        tk.messagebox.showerror(title='Oops', message = 'Please enter label number')

    elif output_label_str.get():
        try:
            int(output_label_str.get())
        except ValueError:
            tk.messagebox.showerror(title='Oops', message = 'Please enter label number')
            return False
        else:
            if sampling_length_str.get() == '':
                tk.messagebox.showerror(title='Oops', message = 'Please enter sampling length')
                return False
            elif sampling_length_str.get():
                try:
                    int(sampling_length_str.get())
                except ValueError:
                    tk.messagebox.showerror(title='Oops', message = 'Please enter sampling length')
                    return False
                else:
                    if int(sampling_length_str.get()) < 2:
                        tk.messagebox.showerror(title='Oops', message = 'Please enter sampling length')
                        return False
                    elif output_decomposition_str.get() == '':
                        tk.messagebox.showerror(title='Oops', message = 'Please enter decomposition level number')
                    elif output_decomposition_str.get():
                        try:
                            int(output_decomposition_str.get())
                        except ValueError:
                            tk.messagebox.showerror(title='Oops', message = 'Please enter decomposition level number')
                            return False
                        else:
                            if int(output_decomposition_str.get()) < 0:
                                tk.messagebox.showerror(title='Oops', message = 'Please enter positive number')
                                return False
                            else:
                                return True
    else:
        return False

def enable_time_domain():
    filemenu.entryconfig("Time domain plot", state = "normal")

def disable_time_domain():
    filemenu.entryconfig("Time domain plot", state = "disabled")

def time_domain_plot():
    try :
        label = output_label_str.get()
        plt.savefig("Time Domain" + label + ".jpg")
        plt.show(block = True)
        tk.messagebox.showinfo(title = "Time domain plot", message = "Image Save!!")
        disable_time_domain()
    except:
        tk.messagebox.showerror(title = "Oops", message = "Save failed")
"""
def time_domain(data):
    time_series = pd.Series(data, index = np.arange(0, len(data)/10240, 1/10240))
    time_series = pd.to_numeric(time_series)
    plt.figure().set_size_inches(14, 10)
    time_series.plot()
    plt.title("Time domain",fontsize=18)
    plt.xticks(fontsize=10)
    plt.yticks(fontsize=10)
    plt.ylabel("Accelaration (g)")
    plt.xlabel('Time (s)')
    #plt.savefig("Time Domain.jpg")
    #plt.show()

    #print(time_series)
"""

def data_sampling(length, data):
    section_data = []
    if length > (len(data) / 2):
        tk.messagebox.showerror(title= 'Oops', message = 'Your sampling length is out of range')
    else:
        if (len(data) / length) > int(len(data) / length):
            section_num = int(len(data) / length) + 1
        else:
            section_num = int(len(data) / length)
        for i in range(section_num):
            section_data.append( data[length*i:length*(i+1)] )

    return section_data

def WPT():
    path = file_path_text.get()
    if check_parameter(path):
        input_data = []
        sampling_length = int(sampling_length_str.get())
        files = os.listdir(path)

        for file in files:
            if os.path.splitext(file)[1] == '.csv':
                tem_input_data = list(csv.reader(open(path + "/" + file,'r')))
                del tem_input_data[0]
                for i in range(len(tem_input_data)):
                    tem_input_data[i].pop(0)
                tem_input_data = list(itertools.chain.from_iterable(tem_input_data))

                ## delete impulse signal
                input_data.extend(TD.DataThreshold(tem_input_data, 2))
                #input_data.extend(tem_input_data)

                    #tk.messagebox.showerror(title = 'Oops', message = path)
                #wp = 0
        TD.TimePlot(input_data, 10240)
        sec_data = data_sampling(sampling_length, input_data)
############################
## Coefficient Initialize ##
        wp_node = []
        coef=[]
        reC = []
        reS = []
        decomposition_level = int(output_decomposition_str.get())
#####################################
## Decompostion and Reconstruction ##
        for i in range(len(sec_data)):
            coef.append(np.zeros((pow(2,decomposition_level),1)))
            start_time = time.time()
            wpt = pywt.WaveletPacket(data = sec_data[i], maxlevel = decomposition_level, wavelet = 'db10', mode = 'symmetric')
            end_time = time.time()
            elapsed_time = end_time - start_time
            print(elapsed_time)
            wp_node = [node.path for node in wpt.get_level(decomposition_level,'freq')]
            index = 0
            for i in range(len(wp_node)):
                reC.append(np.zeros((1,1)))
            new_wp = pywt.WaveletPacket(None,'db10')
            for i in wp_node:
                new_wp[i] = wpt[i]
                reC[index] = new_wp.reconstruct()
                index+=1
                new_wp = pywt.WaveletPacket(None, 'db10')
            reS.append(reC)
            reC=[]
                        #reC.append(np.zeros((len(wp_node), len(new_wp.reconstruct()))))
        temp_energy = []
        energy = []
        energy_ss = []

        for i in range(len(reS)):
            for j in range(len(wp_node)):
                temp_energy.append(np.zeros((1,1)))
                temp_energy[j] = sum(map(lambda x:x*x, reS[i][j]))
            energy.append(temp_energy)
            temp_energy = []
            energy_ss.append(np.zeros((1,1)))
        for i in range(len(energy)):
            energy_ss[i] = math.sqrt(sum(map(lambda x:x*x, energy[i])))
            energy[i] = [x / energy_ss[i] for x in energy[i]]

        label = int(output_label_str.get())
        for i in range(len(energy)):
            energy[i].append(label)

#################
## File output ##
        with open(output_DOE_str.get() + '_Energy_Spectrum_' + '0' + output_decomposition_str.get() + '_' + output_label_str.get() + '.csv', 'w', newline='') as csvfile:
            writer = csv.writer(csvfile)
            for i in range(len(energy)):
                writer.writerow(energy[i])
        tk.messagebox.showinfo(title='WPT', message = 'Decomposition done !!')
        enable_time_domain()

tem_input_data = []
input_data = []
decomposition_level = 0
image_plot = False

window = tk.Tk()
window.title('WPT App')
window.geometry('800x600')

menubar = tk.Menu(window)
filemenu = tk.Menu(menubar, tearoff = 0)
menubar.add_cascade(label = 'File', menu = filemenu)
filemenu.add_command(label = 'Time domain plot', command = time_domain_plot)

input_frame = tk.Frame(window)
input_frame.pack()

title = tk.Label(input_frame, text = 'Wavelet Packet Decomposition')
title.config(font=('Lolita' , 20))
title.pack()

folder_label = tk.Label(input_frame, text = 'Folder Directory')
folder_label.config(font = ('Lolita', 12))
folder_label.pack()

open_dict = tk.Button(input_frame, text= 'Open', command = open_file)
open_dict.pack()

file_path_text = tk.StringVar(window)
tk.Label(input_frame, textvariable = file_path_text).pack()

listbox_frame = tk.Frame(window, bg = "bisque")
listbox_frame.pack()

listbox = tk.Listbox(listbox_frame, width = 80)
listbox.pack(side = tk.LEFT)
scroller = tk.Scrollbar(listbox_frame)
scroller.config(command = listbox.yview)
scroller.pack(side = tk.LEFT, fill = tk.Y)
listbox.config(yscrollcommand = scroller.set)

output_frame = tk.Frame(window, width=50, height=50)
output_frame.pack(pady = 10)

output_frame_left = tk.Frame(output_frame, width=50, height=50)
output_frame_left.pack(side = tk.LEFT, fill = tk.Y)

output_frame_right = tk.Frame(output_frame, width=50, height=50)
output_frame_right.pack(side = tk.RIGHT, fill = tk.Y, padx = 10)

#output_frame_bottom = tk.Frame(output_frame, width=50, height=50, background="#b15165")
#output_frame_bottom.pack(side = tk.BOTTOM, fill = tk.X)

output_DOE_label = tk.Label(output_frame_left, text = 'DOE')
output_DOE_label.config(font = ('Lolita', 12))
output_DOE_label.pack()
output_DOE_str=tk.StringVar()
output_DOE_entry = tk.Entry(output_frame_right, textvariable = output_DOE_str).pack(pady = 3)

output_label_label = tk.Label(output_frame_left, text = 'Label')
output_label_label.config(font = ('Lolita', 12))
output_label_label.pack()
output_label_str=tk.StringVar()
output_label_entry = tk.Entry(output_frame_right, textvariable=output_label_str).pack(pady = 3)

sampling_length_label = tk.Label(output_frame_left, text = 'Sampling Length')
sampling_length_label.config(font = ('Lolita', 12))
sampling_length_label.pack()
sampling_length_str = tk.StringVar()
sampling_length_entry = tk.Entry(output_frame_right, textvariable = sampling_length_str).pack(pady = 3.5)

output_decomposition_label = tk.Label(output_frame_left, text = 'Decomposition Level')
output_decomposition_label.config(font = ('Lolita', 12))
output_decomposition_label.pack()
output_decomposition_str = tk.StringVar()
output_decomposition_entry = tk.Entry(output_frame_right, textvariable = output_decomposition_str).pack(pady = 3.5)

output_button_frame = tk.Frame(window, width=50, height=50)
output_button_frame.pack()

wpd_btn = tk.Button(output_button_frame, text='Start', command = WPT, bg='red').pack(pady = 10)

window.config(menu = menubar)
disable_time_domain()

window.mainloop()

"""
raw_data_direction = "C:/Users/allen/Desktop/Python Code/raw_data/20190927_台灣晶技_機台運轉行為/EXP001_研磨機_位置1/01_靜止"
tem_input_datas = []
input_datas = []
decomposed_level = 6
for file in os.listdir(raw_data_direction):
    tem_input_datas = list(csv.reader(open(raw_data_direction + "/" + file,'r')))
    del tem_input_datas[0]
    for i in range(len(tem_input_datas)):
        tem_input_datas[i].pop(0)
    tem_input_datas = list(itertools.chain.from_iterable(tem_input_datas))
    input_datas.append(tem_input_datas)

wp = 0
wp_node = []
############################
## Coefficient Initialize ##
coef=[]
reC = []
reS = []

for i in range(len(input_datas)):
    coef.append(np.zeros((pow(2,decomposed_level),1)))

    wpt = pywt.WaveletPacket(data = input_datas[i], maxlevel = decomposed_level, wavelet = 'db10', mode = 'symmetric')
    wp_node = [node.path for node in wpt.get_level(6,'freq')]
    index = 0
    for i in range(len(wp_node)):
        reC.append(np.zeros((1,1)))
    index = 0
    new_wp = pywt.WaveletPacket(None,'db10')
    for i in wp_node:
        new_wp[i] = wpt[i]
        reC[index] = new_wp.reconstruct()
        index+=1
        new_wp = pywt.WaveletPacket(None, 'db10')
    reS.append(reC)
    reC=[]
        #reC.append(np.zeros((len(wp_node), len(new_wp.reconstruct()))))

#print(reS[0][0])

#    for j in range(len(wp_node)):
#        coef[i][j] = wpt[wp_node[j]].data

###################
## Reconstuction ##
#for i in range(len(wp_node)):
#    coef[i]= wpt[wp_node[i]].data
#for i in range(len(wp_node)):
#    reC.append(np.zeros((1,1)))
#index = 0
#for i in wp_node:
#    new_wp[i] = wpt[i]
#    reC[index] = new_wp.reconstruct()
#    index+=1
#    new_wp = pywt.WaveletPacket(None, 'db10')

#####################
## Energy Spectrum ##
temp_energy = []
energy = []
energy_ss = []
a=[]
for i in range(64):
    a.append(np.zeros((1,1)))
    a[i]=sum(map(lambda x:x*x, reS[7][i]))
b = sum(a)

for i in range(len(reS)):
    for j in range(len(wp_node)):
        temp_energy.append(np.zeros((1,1)))
        temp_energy[j] = sum(map(lambda x:x*x, reS[i][j]))
    energy.append(temp_energy)
    temp_energy = []
    energy_ss.append(np.zeros((1,1)))
for i in range(len(energy)):
    energy_ss[i] = math.sqrt(sum(map(lambda x:x*x, energy[i])))
    energy[i] = [x / energy_ss[i] for x in energy[i]]

label = 1
for i in range(len(energy)):
    energy[i].append(label)

#A = new_wp.reconstruct(update = False)
#index=0
#for i in wp_node:
#    new_wp[i] = wpt[i]
#    reC[index] = new_wp.reconstruct(update = False)
#    index+=1
#reC = new_wp.reconstruct(update=False)

#arr, coeff_slices = pywt.coeffs_to_array((wpt[wp_node[0]].data).tolist())
#coeffs_from_arr = pywt.array_to_coeffs(arr,coeff_slices,output_format = 'wavedec')
#wpr = pywt.waverec(coeffs_from_arr, wavelet = 'db10')
"""


import sys
sys.path.insert(0, './utils')
import numpy as np
import matplotlib.pyplot as plt
import math
from cliffwalk import CliffWalk
import time
import sys
import tracemalloc

def policy_evaluation(P, R, policy, gamma=0.9, tol=1e-2):
    """
    Args:
        P: np.array
            transition matrix (NsxNaxNs)
        R: np.array
            reward matrix (NsxNa)
        policy: np.array
            matrix mapping states to action (Ns)
        gamma: float
            discount factor
        tol: float
            precision of the solution
    Return:
        value_function: np.array
            The value function of the given policy
    """
    Ns, Na = R.shape
    # ====================================================
	# YOUR IMPLEMENTATION HERE
    ## Compute matrix P_pi with policy pi ie P_pi(s', s) = P(s, pi(s), s')
    P_pi = P[range(Ns), policy, :]
    R_pi = R[range(Ns), policy]
    ## We use solve to resolve the linear system instead of inverting the matrix
    value_function = np.linalg.solve(np.eye(Ns) - gamma*P_pi, R_pi)
    # ====================================================
    return value_function

def policy_iteration(P, R, gamma=0.9, tol=1e-3):
    """
    Args:
        P: np.array
            transition matrix (NsxNaxNs)
        R: np.array
            reward matrix (NsxNa)
        gamma: float
            discount factor
        tol: float
            precision of the solution
    Return:
        policy: np.array
            the final policy
        V: np.array
            the value function associated to the final policy
    """
    Ns, Na = R.shape
    V = np.zeros(Ns)
    policy = np.zeros(Ns, dtype=np.int)
    # ====================================================
	# YOUR IMPLEMENTATION HERE
    iterating = True
    while iterating :
        V = policy_evaluation(P, R, policy, gamma)
        before_max = R + gamma*P.dot(V) # Shape (NsxNa)
        new_policy = np.argmax(before_max, axis = 1)
        if np.allclose(new_policy, policy, atol = tol, rtol = 0):
            iterating = False
        else :
            policy = new_policy
    # ====================================================
    return policy, V

def value_iteration(P, R, gamma=0.9, tol=1e-3):
    """
    Args:
        P: np.array
            transition matrix (NsxNaxNs)
        R: np.array
            reward matrix (NsxNa)
        gamma: float
            discount factor
        tol: float
            precision of the solution
    Return:
        Q: final Q-function (at iteration n)
        greedy_policy: greedy policy wrt Qn
        Qfs: all Q-functions generated by the algorithm (for visualization)
    """
    Ns, Na = R.shape
    Q = np.zeros((Ns, Na))
    Qfs = [Q]
    # ====================================================
	# YOUR IMPLEMENTATION HERE
    iterating = True
    while iterating :
        V = np.max(Q, axis = 1)
        Q_new = R + gamma * P.dot(V)
        V_new =  np.max(Q_new, axis = 1)
        if np.linalg.norm(V_new - V, ord = np.inf) < tol :
            iterating = False
        else :
            Qfs.append(Q_new)
            Q = Q_new
    greedy_policy = np.argmax(Q_new, axis = 1)
    # ====================================================
    return Q, greedy_policy, Qfs

# Edit below to run policy and value iteration on different environments and
# visualize the resulting policies in action!
# You may change the parameters in the functions below
if __name__ == "__main__":
    tol =1e-5
    env = CliffWalk(proba_succ=0.6)
    env.render()

    # run value iteration to obtain Q-values
    t1 = time.time()
    VI_Q, VI_greedypol, all_qfunctions = value_iteration(env.P, env.R, gamma=env.gamma, tol=tol)
    # compute the value function of the greedy policy using matrix inversion
    greedy_V = np.zeros((env.Ns, env.Na))
    # ====================================================
	# YOUR IMPLEMENTATION HERE
    # compute value function of the greedy policy
    greedy_V = policy_evaluation(env.P, env.R, VI_greedypol, gamma=env.gamma)
    # ====================================================
    t2 = time.time()
    print("TIME TO DO THE VALUE ITERATION : ", t2 - t1)

    # render the policy
    #print("[VI]Greedy policy: ")
    #env.render_policy(VI_greedypol)

    # show the error between the computed V-functions and the final V-function
    # (that should be the optimal one, if correctly implemented)
    # as a function of time
    norms = [ np.linalg.norm(q.max(axis=1) - greedy_V) for q in all_qfunctions]
    plt.plot(norms)
    plt.xlabel('Iteration')
    plt.ylabel('Error')
    plt.title("Value iteration: convergence")

    #### POLICY ITERATION ####
    t3 = time.time()
    PI_policy, PI_V = policy_iteration(env.P, env.R, gamma=env.gamma, tol=tol)
    print("\n[PI]final policy: ")
    t4= time.time()
    print("TIME TO DO THE POLICY ITERATION : ", t4 - t3)
    env.render_policy(PI_policy)

    # control that everything is correct
    assert np.allclose(PI_policy, VI_greedypol),\
        "You should check the code, the greedy policy computed by VI is not equal to the solution of PI"
    assert np.allclose(PI_V, greedy_V),\
        "Since the policies are equal, even the value function should be"

    # for visualizing the execution of a policy, you can use the following code
    # state = env.reset()
    # env.render()
    # for i in range(15):
    #     action = VI_greedypol[state]
    #     state, reward, done, _ = env.step(action)
    #     env.render()
    plt.show()

### I used the following code to generate the plot with different prob_succ:

    # plt.show()
    # tol =1e-5
    # for prob_succ in [0.2, 0.4, 0.6, 0.8, 1]:
    #     env = CliffWalk(proba_succ=prob_succ)
    #     env.render()
    #
    #     # run value iteration to obtain Q-values
    #     VI_Q, VI_greedypol, all_qfunctions = value_iteration(env.P, env.R, gamma=env.gamma, tol=tol)
    #
    #     # render the policy
    #     print("[VI]Greedy policy: ")
    #     #env.render_policy(VI_greedypol)
    #
    #     # compute the value function of the greedy policy using matrix inversion
    #     greedy_V = np.zeros((env.Ns, env.Na))
    #     # ====================================================
    #     # YOUR IMPLEMENTATION HERE
    #     # compute value function of the greedy policy
    #     greedy_V = policy_evaluation(env.P, env.R, VI_greedypol, gamma=env.gamma)
    #     # ====================================================
    #
    #     # show the error between the computed V-functions and the final V-function
    #     # (that should be the optimal one, if correctly implemented)
    #     # as a function of time
    #     norms = [ np.linalg.norm(q.max(axis=1) - greedy_V) for q in all_qfunctions]
    #     plt.plot(norms, label='Prob success = %s' % prob_succ)
    #     plt.xlabel('Iteration')
    #     plt.ylabel('Error')
    #     plt.title("Value iteration: convergence")
    #
    #     #### POLICY ITERATION ####
    #     PI_policy, PI_V = policy_iteration(env.P, env.R, gamma=env.gamma, tol=tol)
    #     print("\n[PI]final policy: ")
    #     #env.render_policy(PI_policy)
    #
    #     # control that everything is correct
    #     assert np.allclose(PI_policy, VI_greedypol),\
    #         "You should check the code, the greedy policy computed by VI is not equal to the solution of PI"
    #     assert np.allclose(PI_V, greedy_V),\
    #         "Since the policies are equal, even the value function should be"
    #
    #
    #     # # for visualizing the execution of a policy, you can use the following code
    #     # state = env.reset()
    #     # env.render()
    #     # for i in range(15):
    #     #     action = VI_greedypol[state]
    #     #     state, reward, done, _ = env.step(action)
    #     #     env.render()
    #
    # plt.legend()
    # plt.show()