import numpy as np
import tensorflow as tf
import math, os, random, time
from scipy.linalg import hadamard
import scipy.misc
from PIL import Image
from TRAIN import BATCH_SIZE
from TRAIN import PATCH_SIZE
import struct
import re

import time , threading
from UTILS import *

class Reader (threading.Thread):
    def __init__(self,file_name,id,input_list,gt_list):
        super(Reader,self).__init__()
        self.file_name=file_name
        self.id = id
        self.input_list=input_list
        self.gt_list=gt_list
    def run(self):
        input_image = c_getYdata(self.file_name[0])
        gt_image = c_getYdata(self.file_name[1])
        #print(input_image[0])
        in_ =[]
        gt_ =[]
        for j in range(BATCH_SIZE//8):
            input_imgY, gt_imgY = crop(input_image, gt_image, PATCH_SIZE[0], PATCH_SIZE[1], "ndarray")
            input_imgY = normalize(input_imgY)
            gt_imgY = normalize(gt_imgY)

            in_.append(input_imgY)
            gt_.append(gt_imgY)

        #print(np.shape(in_))
        self.input_list[self.id] = in_
        self.gt_list[self.id] = gt_

def normalize(x):
    x = x / 255.
    return truncate(x, 0., 1.)

def denormalize(x):
    x = x * 255.
    return truncate(x, 0., 255.)

def truncate(input, min, max):
    input = np.where(input > min, input, min)
    input = np.where(input < max, input, max)
    return input

def remap(input):
    input = 16+219/255*input
    #return tf.clip_by_value(input, 16, 235).eval()
    return truncate(input, 16.0, 235.0)

def deremap(input):
    input = (input-16)*255/219
    #return tf.clip_by_value(input, 0, 255).eval()
    return truncate(input, 0.0, 255.0)

def load_file_list(directory):
    list = []
    # for filename in [y for y in os.listdir(directory) if os.path.isfile(os.path.join(directory,y))]:
    #     print(filename)
    #     if filename.split(".")[-1]=="yuv":
    #         list.append(os.path.join(directory,filename))
    for root,dirs,files in os.walk(directory):
        for file in files:
            file_name =os.path.join(root,file)
            if file_name.split(".")[-1]=="yuv":
                list.append(file_name)
    return sorted(list)

def get_train_list(lowList, highList):
    assert len(lowList) % len(highList)==0, "low:%d, high:%d"%(len(lowList), len(highList))
    train_list = []
    for i in range(len(lowList)):
        # print(lowList[i])
        qp=lowList[i].split("\\")[-2].split("qp")[1]
        if 37<=int(qp)<=46:
            train_list.append([lowList[i], highList[i%len(highList)]])
    return train_list
def get_train_list1(lowList, highList):
    # assert len(lowList) == len(highList), "low:%d, high:%d"%(len(lowList), len(highList))
    train_list = []
    for i in range(len(lowList)):
        # print(lowList[i],type(lowList[i]))
        #idx=int(lowList[i].split("\\")[-1].split("_")[-2])-1
        # qp = int(lowList[i].split("\\")[-2].split("q")[1])
        # print(lowList[i], idx,qp)
        # if(qp==53):
        # if(lowList[i].split("_")[-3]==highList[i].split("_")[-3]):
        train_list.append([lowList[i], highList[i%len(highList)]])
        # else:
        #     print(lowList[i],highList[i])
    return train_list
#
# def prepare_nn_data(train_list):
#     batchSizeRandomList = random.sample(range(0,len(train_list)), 8)
#     #print(batchSizeRandomList)
#     input_list = []
#     gt_list = []
#
#     for i in batchSizeRandomList:
#         #print(train_list[i][0])
#         input_image = c_getYdata(train_list[i][0])
#         gt_image = c_getYdata(train_list[i][1])
#
#         for j in range(BATCH_SIZE//8):
#             input_imgY, gt_imgY = crop(input_image, gt_image, PATCH_SIZE[0], PATCH_SIZE[1], "ndarray")
#
#             input_imgY = normalize(input_imgY)
#             gt_imgY = normalize(gt_imgY)
#
#             input_list.append(input_imgY)
#             gt_list.append(gt_imgY)
#
#     input_list = np.reshape(input_list, (BATCH_SIZE, PATCH_SIZE[0], PATCH_SIZE[1], 1))
#     gt_list = np.reshape(gt_list, (BATCH_SIZE, PATCH_SIZE[0], PATCH_SIZE[1], 1))
#
#     return input_list, gt_list

def prepare_nn_data(train_list):
    thread_num =8
    #random_init = random.randint(0, len(train_list)-thread_num)
    batchSizeRandomList = random.sample(range(0,len(train_list)), thread_num)
    #
    #print(batchSizeRandomList)
    input_list = [0 for i in range(thread_num)]
    gt_list = [0 for i in range(thread_num)]
    t = []
    for i in range(thread_num):
        t.append(Reader(train_list[batchSizeRandomList[i]],i,input_list,gt_list))
        #t.append(Reader(train_list[random_init+i], i, input_list, gt_list))
    for i in range(thread_num):
        t[i].start()
    for i in range(thread_num):
        t[i].join()
    #print(input_list)
    input_list = np.reshape(input_list, (BATCH_SIZE, PATCH_SIZE[0], PATCH_SIZE[1], 1))
    gt_list = np.reshape(gt_list, (BATCH_SIZE, PATCH_SIZE[0], PATCH_SIZE[1], 1))

    return input_list, gt_list

'''
def prepare_nn_data(train_list, idx_img=None):
    i = np.random.randint(len(train_list)) if (idx_img is None) else idx_img
    input_image  = c_getYdata(train_list[i][0])
    gt_image = c_getYdata(train_list[i][1])

    input_list = []
    gt_list = []
    inputcbcr_list = []

    for idx in range(BATCH_SIZE):

        #crop images to the disired size.
        input_imgY, gt_imgY = crop(input_image, gt_image, PATCH_SIZE[0], PATCH_SIZE[1], "ndarray")

        #input_imgY = Greying(input_imgY, PATCH_SIZE[0])
        #scipy.misc.imsave('/home/chenjs/relHM/tFromPython/%s.png'%(time.time()), input_imgY)
        #scipy.misc.imsave('/home/chenjs/relHM/tFromPython/%s.png'%(time.time()), gt_imgY)

        #normalize
        input_imgY = normalize(input_imgY)
        gt_imgY = normalize(gt_imgY)

        input_list.append(input_imgY)
        gt_list.append(gt_imgY)
        #inputcbcr_list.append(input_imgCbCr)

    input_list = np.reshape(input_list, (BATCH_SIZE, PATCH_SIZE[0], PATCH_SIZE[1], 1))
    gt_list = np.reshape(gt_list, (BATCH_SIZE, PATCH_SIZE[0], PATCH_SIZE[1], 1))
    #inputcbcr_list = np.resize(inputcbcr_list, (BATCH_SIZE, PATCH_SIZE[0], PATCH_SIZE[1], 2))

    return input_list, gt_list, inputcbcr_list
'''
def getLdata(path):
    img = Image.open(path)
    return  np.asarray(img, dtype='uint8')

def Greying(block, patch_size):
    newBlock = [[0 for i in range(patch_size)] for j in range(patch_size)]
    for i in range(patch_size):
        for j in range(patch_size):
            if (i >= patch_size // 2 and j >= patch_size // 2):
                newBlock[i][j] = 128
                continue
            newBlock[i][j] = block[i][j]
    return np.asarray(newBlock)

def getWH1(yuvfileName):   # Test
    w_included , h_included = os.path.splitext(os.path.basename(yuvfileName))[0].split('x')
    w = w_included.split('_')[-1]
    h = h_included.split('_')[0]
    return int(w), int(h)

def getWH(yuvfileName):   # Train
    #print(yuvfileName)
    deyuv=re.compile(r'(.+?)\.')
    deyuvFilename=deyuv.findall(yuvfileName)[0] #去yuv后缀的文件名
    # print(deyuvFilename)
    if os.path.basename(deyuvFilename).split("_")[0].isdigit():
        wxh = os.path.basename(deyuvFilename).split('_')[1]
    else:
        wxh = os.path.basename(deyuvFilename).split('_')[1]
    w, h = wxh.split('x')
   # print(deyuvFilename,w,h)
    return int(w), int(h)

def getYdata(path, size):
    w= size[0]
    h=size[1]
    #print(w,h)
    Yt = np.zeros([h, w], dtype="uint8", order='C')
    with open(path, 'rb') as fp:
        fp.seek(0, 0)
        Yt = fp.read()
        tem = Image.frombytes('L', [w, h], Yt)

        Yt = np.asarray(tem, dtype='float32')

        # for n in range(h):
        #     for m in range(w):
        #         Yt[n, m] = ord(fp.read(1))

    return Yt

def c_getYdata(path):
    #print(path)
    return getYdata(path, getWH(path))

def calcHADLoss(ori, cur):
    ori = tf.squeeze(ori)
    cur = tf.squeeze(cur)
    resi = tf.subtract(ori, cur)

    transfMatrix = hadamard(8, dtype='float32')

    assert resi.shape[0] == BATCH_SIZE
    resiList = [resi[a] for a in range(BATCH_SIZE)]

    totalSATD = 0
    for i in range(BATCH_SIZE):
        resiPatch = tf.reshape(resiList[i], [1024])

        for col in range(4):
            for row in range(4):
                start = col*8*PATCH_SIZE[0]+row*8
                qt = [[0 for i in range(8)] for j in range(8)]
                for i in range(8):
                    for j in range(8):
                        qt[i][j] = resiPatch[start + i * PATCH_SIZE[0] + j]
                transfromed = tf.matmul(transfMatrix, qt)
                transfromed = tf.matmul(transfromed, transfMatrix)

                totalSATD += tf.reduce_sum(tf.abs(transfromed))

    return totalSATD

def img2y(input_img):
    if np.asarray(input_img).shape[2] == 3:
        input_imgY = input_img.convert('YCbCr').split()[0]
        input_imgCb, input_imgCr = input_img.convert('YCbCr').split()[1:3]

        input_imgY = np.asarray(input_imgY, dtype='float32')
        input_imgCb = np.asarray(input_imgCb, dtype='float32')
        input_imgCr = np.asarray(input_imgCr, dtype='float32')

        #Concatenate Cb, Cr components for easy, they are used in pair anyway.
        input_imgCb = np.expand_dims(input_imgCb,2)
        input_imgCr = np.expand_dims(input_imgCr,2)
        input_imgCbCr = np.concatenate((input_imgCb, input_imgCr), axis=2)

    elif np.asarray(input_img).shape[2] == 1:
        print("This image has one channal only.")
        #If the num of channal is 1, remain.
        input_imgY = input_img
        input_imgCbCr = None
    else:
        print("The num of channal is neither 3 nor 1.")
        exit()
    return input_imgY, input_imgCbCr

def crop(input_image, gt_image, patch_width, patch_height, img_type):
    assert type(input_image) == type(gt_image), "types are different."
    #return a ndarray object
    if img_type == "ndarray":
        in_row_ind   = random.randint(0,input_image.shape[0]-patch_width)
        in_col_ind   = random.randint(0,input_image.shape[1]-patch_height)

        input_cropped = input_image[in_row_ind:in_row_ind+patch_width, in_col_ind:in_col_ind+patch_height]
        gt_cropped = gt_image[in_row_ind:in_row_ind+patch_width, in_col_ind:in_col_ind+patch_height]

    #return an "Image" object
    elif img_type == "Image":
        in_row_ind   = random.randint(0,input_image.size[0]-patch_width)
        in_col_ind   = random.randint(0,input_image.size[1]-patch_height)

        input_cropped = input_image.crop(box=(in_row_ind, in_col_ind, in_row_ind+patch_width, in_col_ind+patch_height))
        gt_cropped = gt_image.crop(box=(in_row_ind, in_col_ind, in_row_ind+patch_width, in_col_ind+patch_height))

    return input_cropped, gt_cropped

def save_images(inputY, inputCbCr, size, image_path):
    """Save mutiple images into one single image.

    Parameters
    -----------
    images : numpy array [batch, w, h, c]
    size : list of two int, row and column number.
        number of images should be equal or less than size[0] * size[1]
    image_path : string.

    Examples
    ---------
    >>> images = np.random.rand(64, 100, 100, 3)
    >>> tl.visualize.save_images(images, [8, 8], 'temp.png')
    """
    def merge(images, size):
        h, w = images.shape[1], images.shape[2]
        img = np.zeros((h * size[0], w * size[1], 3))
        for idx, image in enumerate(images):
            i = idx % size[1]
            j = idx // size[1]
            img[j*h:j*h+h, i*w:i*w+w, :] = image
        return img

    inputY = inputY.astype('uint8')
    inputCbCr = inputCbCr.astype('uint8')
    output_concat = np.concatenate((inputY, inputCbCr), axis=3)

    assert len(output_concat) <= size[0] * size[1], "number of images should be equal or less than size[0] * size[1] {}".format(len(output_concat))

    new_output = merge(output_concat, size)

    new_output = new_output.astype('uint8')

    img = Image.fromarray(new_output, mode='YCbCr')
    img = img.convert('RGB')
    img.save(image_path)

def get_image_batch(train_list,offset,batch_size):
    target_list = train_list[offset:offset+batch_size]
    input_list = []
    gt_list = []
    inputcbcr_list = []
    for pair in target_list:
        input_img = Image.open(pair[0])
        gt_img = Image.open(pair[1])

        #crop images to the disired size.
        input_img, gt_img = crop(input_img, gt_img, PATCH_SIZE[0], PATCH_SIZE[1], "Image")

        #focus on Y channal only
        input_imgY, input_imgCbCr = img2y(input_img)
        gt_imgY, gt_imgCbCr = img2y(gt_img)

        #input_imgY = normalize(input_imgY)
        #gt_imgY = normalize(gt_imgY)

        input_list.append(input_imgY)
        gt_list.append(gt_imgY)
        inputcbcr_list.append(input_imgCbCr)

    input_list = np.resize(input_list, (batch_size, PATCH_SIZE[0], PATCH_SIZE[1], 1))
    gt_list = np.resize(gt_list, (batch_size, PATCH_SIZE[0], PATCH_SIZE[1], 1))

    return input_list, gt_list, inputcbcr_list

def save_test_img(inputY, inputCbCr, path):
    assert len(inputY.shape) == 4, "the tensor Y's shape is %s"%inputY.shape
    assert inputY.shape[0] == 1, "the fitst component must be 1, has not been completed otherwise.{}".format(inputY.shape)

    inputY = np.squeeze(inputY, axis=0)
    inputY = inputY.astype('uint8')

    inputCbCr = inputCbCr.astype('uint8')

    output_concat = np.concatenate((inputY, inputCbCr), axis=2)
    img = Image.fromarray(output_concat, mode='YCbCr')
    img = img.convert('RGB')
    img.save(path)

def psnr(hr_image, sr_image, max_value=255.0):
    eps = 1e-10
    if((type(hr_image)==type(np.array([]))) or (type(hr_image)==type([]))):
        hr_image_data = np.asarray(hr_image, 'float32')
        sr_image_data = np.asarray(sr_image, 'float32')

        diff = sr_image_data - hr_image_data
        mse = np.mean(diff*diff)
        mse = np.maximum(eps, mse)
        return float(10*math.log10(max_value*max_value/mse))
    else:
        assert len(hr_image.shape)==4 and len(sr_image.shape)==4
        diff = hr_image - sr_image
        mse = tf.reduce_mean(tf.square(diff))
        mse = tf.maximum(mse, eps)
        return 10*tf.log(max_value*max_value/mse)/math.log(10)

# 将filename对应的图片文件读入，并缩放到统一的大小
def _parse_function(filename, label):
  # 读取图像文件内容编码为字符串
  image_contents = tf.read_file(filename)
  # 根据图像编码后的字符串解码为uint8的tensor
  image_decoded = tf.image.decode_image(image_contents)
  # 修改图像尺寸
  image_resized = tf.image.resize_images(image_decoded, [28, 28])
  return image_resized, label

def divide_img(img, block_size, pading_size):
    h = img.shape[1]
    w = img.shape[2]
    boh = math.ceil(h / block_size)
    bow = math.ceil(w / block_size)
    sub_imgs = []
    for i in range(boh):
        for j in range(bow):
            if i != boh - 1 and j != bow - 1:
                if i == 0 and j == 0:
                    sub_img = img[:, 0:block_size + pading_size, 0:block_size + pading_size, :]
                elif i == 0 and j != 0:
                    sub_img = img[:, 0:block_size + pading_size,
                              block_size * (j) - pading_size:block_size * (j + 1) + pading_size, :]
                elif i != 0 and j == 0:
                    sub_img = img[:, block_size * i - pading_size:block_size * (i + 1) + pading_size,
                              0:block_size + pading_size, :]
                else:
                    sub_img = img[:, block_size * i - pading_size:block_size * (i + 1) + pading_size,
                              block_size * (j) - pading_size:block_size * (j + 1) + pading_size, :]
            elif i == boh - 1 and j != bow - 1:
                if j != 0:
                    sub_img = img[:, -(block_size + pading_size):,
                              block_size * (j) - pading_size:block_size * (j + 1) + pading_size, :]
                else:
                    sub_img = img[:, -(block_size + pading_size):,
                              0:block_size * (j + 1) + pading_size, :]
            elif i != boh - 1 and j == bow - 1:
                if i != 0:
                    sub_img = img[:, block_size * i - pading_size:block_size * (i + 1) + pading_size,
                              -(block_size + pading_size):, :]
                else:
                    sub_img = img[:, 0:block_size * (i + 1) + pading_size,
                              -(block_size + pading_size):, :]
            else:
                sub_img = img[:, -(block_size + pading_size):,
                          -(block_size + pading_size):, :]

            sub_imgs.append(sub_img)

    #         plt.subplot(boh, bow, i  * bow + j+1 )
    #         #plt.title("subImg : "+ str(j)+" "+str(i))
    #         plt.axis("off")
    #         plt.imshow(sub_img[0,:,:,0],cmap='gray')
    # plt.show()
    return sub_imgs

def compose_img(img, sub_imgs, block_size, pading_size):
    h = img.shape[1]
    w = img.shape[2]
    rec = np.zeros([h, w], int)
    boh = math.ceil(h / block_size)
    bow = math.ceil(w / block_size)
    #plt.figure(figsize=(24, 7))
    for i in range(boh):
        for j in range(bow):
            if i != boh - 1 and j != bow - 1:
                if i == 0 and j == 0:
                    rec[0:block_size, 0:block_size] = np.mat(sub_imgs[ i  * bow + j])[0:block_size, 0:block_size]
                elif i == 0 and j != 0:
                    rec[ :block_size, block_size * (j):block_size * (j + 1)] = np.mat(sub_imgs[ i  * bow + j])[
                                                                                :block_size,
                                                                                pading_size:block_size + pading_size]
                elif i != 0 and j == 0:
                    rec[ block_size * i:block_size * (i + 1), :block_size ] = np.mat(sub_imgs[
                                                                                i  * bow + j])[
                                                                                pading_size:block_size + pading_size,
                                                                                :block_size]
                else:
                    rec[ block_size * i:block_size * (i + 1), block_size * (j):block_size * (j + 1)] = np.mat(sub_imgs[ i  * bow + j])[
                                                                                pading_size:block_size + pading_size,
                                                                                pading_size:block_size + pading_size]
            elif i == boh - 1 and j != bow - 1:
                if j != 0:
                    rec[ -block_size:, block_size * (j):block_size * (j + 1)] = np.mat(sub_imgs[ i  * bow + j])[
                                                                                pading_size:block_size + pading_size,
                                                                                pading_size:block_size + pading_size]
                else:
                    rec[ -block_size:, 0:block_size * (j + 1)] = np.mat(sub_imgs[ i  * bow + j])[
                                                                      pading_size:block_size + pading_size,
                                                                      0:block_size]
            elif i != boh - 1 and j == bow - 1:
                if i != 0:
                    rec[ block_size * i:block_size * (i + 1), - block_size:] = np.mat(sub_imgs[ i  * bow + j])[
                                                                                pading_size:block_size + pading_size,
                                                                                pading_size:block_size + pading_size]
                else:
                    rec[ 0:block_size * (i + 1) , -block_size :] = np.mat(sub_imgs[ i  * bow + j])[
                                                                                0:block_size,
                                                                                pading_size:block_size + pading_size]
            else:
                rec[ -block_size: , -block_size :] = np.mat(sub_imgs[ i  * bow + j])[
                                                                                -block_size: , -block_size :]
    return rec

def saveImg(file,inp):

    h, w = inp[0], inp[1]
    # tem = np.asarray(inp, dtype='uint8')
    # #np.save(r"H:\KONG\cnn_2K%f" % time.time(),tem)
    # tem = Image.fromarray(tem, 'L')
    # tem.save("D:/rec/FromPython%f.jpg" % time.time())
    with open(file,'wb') as fp:
        for line in inp:
            for i in line:
                #print(i)
                fp.write(struct.pack('B',i))

    print('image saved')


from flask import Flask, abort, request
import json
from kafka import KafkaProducer
import configparser

app = Flask(__name__)
config = configparser.ConfigParser();
config.read('config.ini')
producer = KafkaProducer(bootstrap_servers=config['KAFKA']['bootstrap_servers'])

@app.route('/api', methods=['POST'])
def foo():
    if not request.json:
        abort(400)
    producer.send(config['KAFKA']['topic'], bytes(str(request.get_data()).encode()))
    return ('', 200)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=80, debug=True)


import numpy as np
from keras.utils import to_categorical, normalize
from matplotlib import pyplot as plt
from keras.datasets import fashion_mnist
from sklearn.model_selection import train_test_split
from keras.models import Sequential,Input,Model
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.layers.advanced_activations import LeakyReLU
from keras.losses import categorical_crossentropy
from keras.optimizers import Adam

(train_x , train_y) , (test_x , test_y) = fashion_mnist.load_data()

print('Shape of training data: ',train_x.shape,train_y.shape)
print('shape of testing data: ',test_x.shape,test_y.shape)

train_x = normalize(train_x, axis=1)
test_x = normalize(test_x, axis=1)
classes = np.unique(train_y)
nclasses = len(classes)
print('No of unique classes: ',nclasses)
print('Output Classes: ',classes)

plt.figure(figsize=[5,5])

plt.subplot(121)
plt.imshow(test_x[1,:,:],cmap='gray')
plt.title('Ground Truth: {}'.format(test_y[1]))

plt.subplot(122)
plt.imshow(test_x[0,:,:],cmap='gray')
plt.title('Ground Truth: {}'.format(test_y[0]))
plt.figure()
plt.show()

train_x = train_x.reshape(-1,28,28,1)
test_x = test_x.reshape(-1,28,28,1)
train_x = train_x.astype('float32')
test_x = test_x.astype('float32')
train_x = train_x / 255
test_x = test_x / 255
print(train_x.shape , test_x.shape)
train_y_one_hot = to_categorical(train_y)
test_y_one_hot = to_categorical(test_y)

print('original label: ', train_y[0])
print('one hot: ', train_y_one_hot[0])

train_x, valid_x, train_label, valid_label = train_test_split(train_x, train_y_one_hot, test_size=0.2, random_state=13)
print(train_x.shape , valid_x.shape, train_label.shape, valid_label.shape)

batch_size = 64
epochs = 20
num_classes = 10

fashion_model = Sequential()
fashion_model.add(Conv2D(32, kernel_size=(3,3), activation='linear', input_shape=(28,28,1), padding='same'))
fashion_model.add(LeakyReLU(alpha=0.1))
fashion_model.add(MaxPooling2D(pool_size=(2,2), padding='same'))
fashion_model.add(Dropout(0.25))
fashion_model.add(Conv2D(128, kernel_size=(3,3), activation='linear', padding='same'))
fashion_model.add(LeakyReLU(alpha=0.1))
fashion_model.add(MaxPooling2D(pool_size=(2,2), padding='same'))
fashion_model.add(Dropout(0.25))
fashion_model.add(Conv2D(128,kernel_size=(3,3), activation='linear', padding='same'))
fashion_model.add(LeakyReLU(alpha=0.1))
fashion_model.add(MaxPooling2D(pool_size=(2,2), padding='same'))
fashion_model.add(Dropout(0.4))
fashion_model.add(Flatten())
fashion_model.add(Dense(128, activation='linear'))
fashion_model.add(LeakyReLU(alpha=0.1))
fashion_model.add(Dropout(0.3))
fashion_model.add(Dense(num_classes, activation='softmax'))

fashion_model.summary()

fashion_model.compile(loss= categorical_crossentropy, optimizer=Adam(), metrics=['accuracy'])

fashion_train = fashion_model.fit(train_x, train_label, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(valid_x,valid_label))

fashion_model.save('fashion_model_dropout_1.h5py')

test_eval = fashion_model.evaluate(test_x,test_y_one_hot,verbose=1)
print('Test Loss, Test Acccuracy',test_eval[0], test_eval[1])
#plt.clf()
accuracy = fashion_train.history['acc']
val_accuracy = fashion_train.history['val_acc']
loss = fashion_train.history['loss']
val_loss = fashion_train.history['val_loss']
epochs = range(len(accuracy))
plt.plot(epochs, accuracy, 'bo', label='Training accuracy')
plt.plot(epochs, val_accuracy, 'b', label='validation accuracy')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()
plt.show()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.figure()
plt.show()

predicted_classes = fashion_model.predict(test_x)
predicted_classes = np.argmax(np.round(predicted_classes), axis=1)
print(predicted_classes.shape, test_y.shape)
correct = np.where(predicted_classes==test_y)[0]
print('found correct labels:', len(correct))
# for i, correct in enumerate(correct[:9]):
#     plt.subplot(3,3,i+1)
#     plt.imshow(test_x[correct].reshape(28,28), cmap='gray', interpolation='none')
#     plt.title('predicted {}, class {}'.format(predicted_classes[correct], test_y[correct]))
#     plt.tight_layout()


# coding=utf-8
"""Site model definition.

"""

import logging
import requests
import json

from django.core.exceptions import ValidationError
from django.contrib.gis.db import models
from django.dispatch import receiver
from django.contrib.postgres.fields import JSONField
from django.conf import settings

from bims.models.location_type import LocationType
from bims.utils.get_key import get_key
from bims.models.document_links_mixin import DocumentLinksMixin
from bims.models.search_process import SearchProcess
from bims.enums.geomorphological_zone import GeomorphologicalZoneCategory

LOGGER = logging.getLogger(__name__)

class LocationSite(DocumentLinksMixin):
    """Location Site model."""

    __original_centroid = None
    geocontext_url_format = '{geocontext_url}/api/v1/geocontext/value/' \
                            'collection/{longitude}/{latitude}/' \
                            '{geocontext_collection_key}'

    name = models.CharField(
        max_length=300,
        blank=False,
    )
    site_description = models.CharField(
        max_length=500,
        blank=True,
        default=''
    )
    site_code = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )
    legacy_site_code = models.CharField(
        max_length=100,
        blank=True,
        default=''
    )
    location_type = models.ForeignKey(
        LocationType,
        models.CASCADE,
        null=False,
    )
    refined_geomorphological = models.CharField(
        blank=True,
        null=True,
        max_length=200,
        choices=[(g.value, g.value) for g in GeomorphologicalZoneCategory],
        help_text='Would be used in preference to the one discovered '
                  'in geocontext',
    )
    geometry_point = models.PointField(
        null=True,
        blank=True,
    )
    geometry_line = models.LineStringField(
        null=True,
        blank=True,
    )
    geometry_polygon = models.PolygonField(
        null=True,
        blank=True,
    )
    geometry_multipolygon = models.MultiPolygonField(
        null=True,
        blank=True,
    )
    map_reference = models.CharField(
        null=True,
        blank=True,
        max_length=200
    )
    land_owner_detail = models.TextField(
        null=True,
        blank=True,
    )
    river = models.ForeignKey(
        'sass.River',
        null=True,
        blank=True,
    )

    location_context_document = JSONField(
        verbose_name='Document for location context as JSON.',
        help_text='This document is generated from GeoContext by using '
                  'management command or changing the geometry.',
        null=True,
        blank=True
    )

    location_context = JSONField(
        verbose_name='Formatted location_context_document',
        help_text='This is intended for filtering',
        null=True,
        blank=True
    )

    additional_data = JSONField(
        verbose_name='Additional json data',
        null=True,
        blank=True
    )

    latitude = models.FloatField(
        blank=True,
        help_text='Latitude of location site',
        null=True
    )

    longitude = models.FloatField(
        blank=True,
        help_text='Longitude of location site',
        null=True
    )
    original_geomorphological = models.CharField(
        blank=True,
        null=True,
        max_length=200,
        help_text='Original geomorphological zone from spatial layer'
    )

    creator = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        models.SET_NULL,
        blank=True,
        null=True,
    )

    def location_context_group_values(self, group_name):
        """
        Returns location group data by group name
        :param group_name: string of group name, e.g. geomorphological_group
        :return: dict of group data
        """
        try:
            location_context = json.loads(self.location_context)
        except ValueError:
            return {}
        try:
            return (
                location_context
                ['context_group_values']
                [group_name]
                ['service_registry_values']
            )
        except KeyError:
            return {}

    def get_centroid(self):
        """ Getting centroid of location site """

        if self.geometry_point:
            return self.geometry_point
        else:
            if self.get_geometry():
                return self.get_geometry().centroid
            else:
                return None

    def get_geometry(self):
        """Function to get geometry."""
        geometry = None
        validation_error = ValidationError('Only one geometry allowed.')

        if self.geometry_point:
            if geometry:
                raise validation_error
            geometry = self.geometry_point

        if self.geometry_polygon:
            if geometry:
                raise validation_error
            geometry = self.geometry_polygon

        if self.geometry_multipolygon:
            if geometry:
                raise validation_error
            geometry = self.geometry_multipolygon

        if self.geometry_line:
            if geometry:
                raise validation_error
            geometry = self.geometry_line

        return geometry

    def get_geocontext_group_data(self, group_key):
        LOGGER.debug('get_location_group_data for ' + group_key)
        geocontext_url = get_key('GEOCONTEXT_URL')
        if not geocontext_url:
            message = (
                'Can not update location context document because geocontext '
                'url is None. Please set it.')
            return False, message
        if not self.get_centroid():
            message = (
                'Can not update location context document because centroid is '
                'None. Please set it.')
            return False, message
        longitude = self.get_centroid().x
        latitude = self.get_centroid().y

        geocontext_group_url_format = (
            '{geocontext_url}/api/v1/geocontext/value/group/'
            '{longitude}/{latitude}/{geocontext_group_key}')
        # build url
        url = geocontext_group_url_format.format(
            geocontext_url=geocontext_url,
            longitude=longitude,
            latitude=latitude,
            geocontext_group_key=group_key,
        )
        LOGGER.info('Request url : {}'.format(url))

        r = requests.get(url)
        if r.status_code != 200:
            message = (
                    'Request to url %s got %s [%s], can not update location '
                    'context document.' % (url, r.status_code, r.reason))
            LOGGER.info(message)
            return None

        return json.dumps(r.json())

    def add_context_group(self, group_key):
        old_location_context_string = self.location_context_document
        old_location_context = None
        if old_location_context_string:
            try:
                old_location_context = json.loads(old_location_context_string)
            except (ValueError, TypeError):
                if isinstance(old_location_context_string, dict):
                    old_location_context = old_location_context_string
                else:
                    LOGGER.info('Not a JSON Object')
        new_data = self.get_geocontext_group_data(group_key)
        if not new_data:
            return False, "Error from GeoContext"
        if not old_location_context:
            old_location_context_string = (
                '{"context_group_values":[%s]}' % new_data)
            self.location_context_document = old_location_context_string
            return True, "Group values added to empty document"
        new_context_data = json.loads(new_data)
        key = new_context_data['key']
        data_exists = False
        return_message = ''
        for context_group in old_location_context['context_group_values']:
            if context_group['key'] == key:
                return_message = 'Group value updated'
                data_exists = True
                context_group['service_registry_values'] = (
                    new_context_data['service_registry_values']
                )
        if not data_exists:
            return_message = 'Group values added'
            old_location_context['context_group_values'].append(
                new_context_data
            )
        self.location_context_document = json.dumps(old_location_context)
        return True, return_message

    def clear_location_context_document(self):
        self.location_context_document = ""
        return True, "Document cleared"

    def update_location_context_document(self):
        """Update location context document."""
        LOGGER.debug('update_location_context_document')
        geocontext_url = get_key('GEOCONTEXT_URL')
        geocontext_collection_key = get_key('GEOCONTEXT_COLLECTION_KEY')
        if not geocontext_url:
            message = (
                'Can not update location context document because geocontext '
                'url is None. Please set it.')
            return False, message
        if not geocontext_collection_key:
            message = (
                'Can not update location context document because geocontext '
                'collection key is None. Please set it.')
            return False, message
        if not self.get_centroid():
            message = (
                'Can not update location context document because centroid is '
                'None. Please set it.')
            return False, message
        longitude = self.get_centroid().x
        latitude = self.get_centroid().y

        # build url
        url = self.geocontext_url_format.format(
            geocontext_url=geocontext_url,
            longitude=longitude,
            latitude=latitude,
            geocontext_collection_key=geocontext_collection_key,
        )

        r = requests.get(url)
        if r.status_code != 200:
            message = (
                'Request to url %s got %s [%s], can not update location '
                'context document.' % (url, r.status_code, r.reason))
            return False, message

        self.location_context_document = json.dumps(r.json())
        return True, 'Successfully update location context document.'

    # noinspection PyClassicStyleClass
    class Meta:
        """Meta class for project."""
        app_label = 'bims'

    def __unicode__(self):
        return u'%s' % self.name

    def validate_json_field(self, field):
        max_allowed = 10
        attempt = 0
        is_dictionary = False

        while not is_dictionary and attempt < max_allowed:
            if not field:
                break
            if isinstance(field, dict):
                is_dictionary = True
            else:
                field = json.loads(
                    field
                )
                attempt += 1
        return field

    def save(self, *args, **kwargs):
        lon_lat_changed = False

        self.additional_data = self.validate_json_field(
            self.additional_data
        )
        self.location_context_document = self.validate_json_field(
            self.location_context_document
        )
        self.location_context = self.validate_json_field(
            self.location_context
        )

        """Check if one of geometry is not null."""
        if self.geometry_point or self.geometry_line or \
                self.geometry_polygon or self.geometry_multipolygon:
            # Check if geometry is allowed
            if isinstance(self.get_geometry(),
                          self.location_type.get_allowed_geometry_class()):
                if (
                        self.get_centroid().x != self.__original_longitude or
                        not self.longitude
                ):
                    self.longitude = self.get_centroid().x
                    lon_lat_changed = True
                if (
                        self.get_centroid().y != self.__original_latitude or
                        not self.latitude
                ):
                    self.latitude = self.get_centroid().y
                    lon_lat_changed = True

                # Check if latitude or longitude changed manually
                if (
                        self.latitude != self.__original_latitude or
                        self.longitude != self.__original_longitude
                ):
                    if not lon_lat_changed:
                        self.geometry_point.x = self.longitude
                        self.geometry_point.y = self.latitude
                super(LocationSite, self).save(*args, **kwargs)
                self.__original_centroid = self.get_centroid()
            else:
                raise ValidationError('Geometry is not allowed.')
        else:
            raise ValidationError('At least one geometry need to be filled.')

        if (
                self.__original_refined_geomorphological !=
                self.refined_geomorphological):
            SearchProcess.objects.filter(
                finished=True
            ).delete()

    def __init__(self, *args, **kwargs):
        super(LocationSite, self).__init__(*args, **kwargs)
        self.__original_centroid = self.get_centroid()
        self.__original_refined_geomorphological = (
            self.refined_geomorphological
        )
        self.__original_latitude = self.latitude
        self.__original_longitude = self.longitude

@receiver(models.signals.post_save)
def location_site_post_save_handler(sender, instance, **kwargs):
    """
    Update cluster when location site saved
    """
    from bims.utils.location_context import format_location_context
    if not issubclass(sender, LocationSite):
        return
    format_location_context(instance.id)


from .group import Group
from .node import Node
from .topic import Topic
from .task import Task
from . import errors
from . import element

from networkx.algorithms.dag import descendants
from networkx import DiGraph

class Soup:
    def __init__(self):
        self._elements = set()
        self._root_group = None
        self._worldgraph = None

    def __len__(self):
        # well this is a bit dumb
        return len(list(self._root_group.iter_recursive()))

    def iter_topics(self):
        self._assert_committed()
        for name, elem in self._root_group.iter_recursive():
            if isinstance(elem, Topic):
                yield elem
            elif isinstance(elem, Task) or isinstance(elem, Group):
                # possibly generalize that; pass class as argument
                continue
            else:
                assert False, elem

    def add_element(self, element):
        self._assert_uncommitted()
        if self._worldgraph:
            raise errors.TopicError(f'cannot add {element}: graph already made')
        self._elements.add(element)

    def commit(self):
        if self._root_group is not None:
            return

        self._root_group = Group(title='Root', path=(), docname='')
        self._make_hierarchy()
        self._add_nodes_to_groups()
        assert len(self._elements) == 0
        del self._elements

    @property
    def root(self):
        return self._root_group

    def element_by_path(self, path):
        return self._root_group.element_by_path(path)

    def worldgraph(self):
        self._assert_committed()
        return self._make_worldgraph()

    def subgraph(self, entrypoints):
        '''Given entrypoints, compute a subgraph of the world graph that
        contains the entrypoints and all their descendants.

        entrypoints is an iterable of element paths or elements (can
        be mixed)
        '''

        self._assert_committed()
        world = self._make_worldgraph()

        entrypoint_elems = set()
        for e in entrypoints:
            try:
                element.verify_is_path(e)
            except errors.TopicError:   # not a path; must be element
                entrypoint_elems.add(e)
            else:
                entrypoint_elems.add(self.element_by_path(e))

        topics = set()
        for topic in entrypoint_elems:
            topics.add(topic)
            topics.update(descendants(world, topic))
        return world.subgraph(topics)

    def _make_worldgraph(self):
        if self._worldgraph is not None:
            return self._worldgraph

        self._worldgraph = DiGraph()
        for name, elem in self._root_group.iter_recursive():
            if not isinstance(elem, Node):
                continue
            self._worldgraph.add_node(elem)
            for target_path in elem.dependencies:
                try:
                    target_topic = self.element_by_path(target_path)
                except errors.PathNotFound:
                    raise errors.TopicError(f'{elem.docname} ({elem}): dependency {target_path} not found')
                self._worldgraph.add_edge(elem, target_topic)

        return self._worldgraph

    def _make_hierarchy(self):
        level = 1
        while True:
            all_groups = [g for g in self._elements if isinstance(g, Group)]
            if not all_groups:   # no more groups
                break
            level_groups = [g for g in all_groups if len(g._requested_path) == level]
            for g in level_groups:
                self._root_group.add_element(g)
                self._elements.remove(g)
            level += 1

    def _add_nodes_to_groups(self):
        nodes = [n for n in self._elements if isinstance(n, Node)]
        for n in nodes:
            self._root_group.add_element(n)
            self._elements.remove(n)

    def _assert_committed(self):
        if self._root_group is None:
            raise errors.NotCommitted('soup not committed')

    def _assert_uncommitted(self):
        if self._root_group is not None:
            raise errors.AlreadyCommitted('soup already committed')


import sys
import xml.dom.minidom
import os
import random
import datetime

filter_categories = ['800-8400','800-8300','800-8200','800-8000','700-7851','700-7850','700-7600','700-7450','700-7300','700-7000','600-6950','600-6900','600-6800','600-6700','600-6600','600-6500','600-6400','600-6300','600-6200','600-6100','600-6000','550-5520','500-5100','500-5000','300-3200','300-3100','300-3000','200-2300','200-2200','200-2100','200-2000','100-1100','100-1000']

country_sample_numbers = [
    # EEU
    {'country':'TUR', 'counts': [13,13,33,33,33]},
    {'country':'POL', 'counts': [5,5,10,10,10]},
    {'country':'RUS', 'counts': [3,3,8,8,8]},
    {'country':'ROU', 'counts': [2,2,7,7,7]},
    {'country':'GRC', 'counts': [2,2,5,5,6]},
    {'country':'CZE', 'counts': [1,2,4,4,4]},
    {'country':'HUN', 'counts': [1,2,4,4,4]},
    {'country':'HRV', 'counts': [1,1,2,3,3]},
    {'country':'BGR', 'counts': [1,1,2,3,3]},
    {'country':'SVK', 'counts': [1,1,1,1,1]},
    {'country':'KAZ', 'counts': [1,1,1,1,1]},
    # WEU
    {'country':'NLD', 'counts': [2,2,7,7,7]},
    {'country':'SWE', 'counts': [2,2,7,7,7]},
    {'country':'BEL', 'counts': [1,1,4,4,5]},
    {'country':'PRT', 'counts': [1,1,4,4,5]},
    {'country':'AUT', 'counts': [1,1,2,3,3]},
    {'country':'DNK', 'counts': [1,1,2,3,3]},
    {'country':'NOR', 'counts': [1,1,2,3,3]},
    {'country':'FIN', 'counts': [1,1,2,3,3]},
    {'country':'CHE', 'counts': [1,1,2,3,3]},
    {'country':'IRL', 'counts': [1,1,1,1,1]},
    {'country':'LUX', 'counts': [1,1,1,1,1]},
    {'country':'MLT', 'counts': [1,1,1,1,1]},
    {'country':'ISL', 'counts': [1,1,1,1,1]},
    # APAC
    {'country':'THA', 'counts': [4,10,10,12,12]},
    {'country':'IDN', 'counts': [4,4,9,9,9]},
    {'country':'VNM', 'counts': [2,2,2,7,7]},
    {'country':'MYS', 'counts': [1,2,4,4,4]},
    {'country':'PHL', 'counts': [1,1,2,3,3]},
    {'country':'TWN', 'counts': [1,1,2,3,3]},
    {'country':'SGP', 'counts': [1,1,2,3,3]},
    {'country':'HKG', 'counts': [1,1,1,1,1]},
    {'country':'IND', 'counts': [15,15,35,35,35]},
    {'country':'BGD', 'counts': [1,1,1,1,1]},
    {'country':'NPL', 'counts': [1,1,1,1,1]},
    {'country':'LKA', 'counts': [1,1,1,1,1]},
    # NA
    {'country':'CAN', 'counts': [12,12,32,32,32]},
    {'country':'BHS', 'counts': [2,2,2,2,2]},
    {'country':'CYM', 'counts': [1,1,1,1,1]},
    {'country':'JAM', 'counts': [2,2,2,2,2]},
    {'country':'VGB', 'counts': [1,1,1,1,1]},
    # LAM
    {'country':'BRA', 'counts': [7,8,20,20,20]},

    {'country':'MEX', 'counts': [3,3,8,8,8]},
    {'country':'ARG', 'counts': [2,2,2,2,2]},
    {'country':'COL', 'counts': [1,1,1,1,1]},
    {'country':'CHL', 'counts': [1,1,1,1,1]},
    {'country':'PER', 'counts': [1,1,1,1,1]},
    {'country':'ECU', 'counts': [1,1,1,1,1]},
    {'country':'PRY', 'counts': [1,1,1,1,1]},
    {'country':'PAN', 'counts': [1,1,1,1,1]},
    {'country':'VEN', 'counts': [1,1,1,1,1]},
    # BIG 7
    {'country':'AUS', 'counts': [15,15,40,40,40]},
    {'country':'DEU', 'counts': [15,15,40,40,40]},
    {'country':'ESP', 'counts': [15,15,40,40,40]},
    {'country':'GBR', 'counts': [15,15,20,20,20]},
    {'country':'ITA', 'counts': [15,15,40,40,40]},
    {'country':'FRA', 'counts': [15,15,40,40,40]},
    {'country':'USA', 'counts': [15,15,40,40,40]},
]

def main(country, region, kick_off_date):
    randomed_list = read_samples(country)
    csv_list = create_csv_list(country, region, randomed_list)
    save_csv_list(csv_list, country, region, kick_off_date)

def read_samples(country):
    randomed_list = []
    dir_path = os.path.dirname(os.path.realpath(__file__))
    fw = open (os.path.join(dir_path, "xml", country+"_tmp.csv"), 'rb')
    for line in fw:
        randomed_list.append(line)
    fw.close()
    os.remove(os.path.join(dir_path, "xml", country+"_tmp.csv"))
    return randomed_list

def create_csv_list(country, region, randomed_list):
    print("sampling: "+country)
    csv_list = []
    for i in range(0,3):
        random.shuffle(randomed_list)
        csv_list = []
        givenCountry = [x for x in country_sample_numbers if x['country']==country][0]
        current_counts = {
            'r1': givenCountry['counts'][0],
            'r2': givenCountry['counts'][1],
            'r3': givenCountry['counts'][2],
            'r4': givenCountry['counts'][3],
            'r5': givenCountry['counts'][4]
        }
        if region == "EEU":
            csv_list.append(
                "place_id,name,phone,house_number,street_basename,street_type,city,state,country,postal,nt_cat_id,nt_cat_name,pd_cat_id,pd_cat_name,lat,lon,isplace,isopen,isname,isaddr,isphone,reality_score,full_house_number,district"
            )
        else:
            csv_list.append(
                "place_id,name,phone,house_number,street_basename,street_type,city,state,country,postal,nt_cat_id,nt_cat_name,pd_cat_id,pd_cat_name,lat,lon,isplace,isopen,isname,isaddr,isphone,reality_score"
            )
        for item in randomed_list:
            doc = xml.dom.minidom.parseString(item)
            placeid = ""
            name = ""
            phone = ""
            house_number = ""
            street_basename = ""
            street_type = ""
            city = ""
            state = ""
            postal = ""
            lcms_category_id = ""
            lcms_category_name = ""
            poi_category_id = ""
            poi_category_name = ""
            lat = ""
            lon = ""
            isplace = "-0.1"
            isopen = "-0.1"
            isname = "-0.1"
            isaddr = "-0.1"
            isphone = "-0.1"
            reality_score = ""
            full_house_number = "NULL"
            district = ""

            # category lcms ---------------------------------------------
            categories = doc.getElementsByTagName('Category')
            for category in categories:
                if category.hasAttribute('primaryFlag') and category.getAttribute('primaryFlag') == "true" and category.getAttribute('categorySystem') == "navteq-lcms":
                    categoryIdNode = category.getElementsByTagName('CategoryId')
                    if categoryIdNode:
                        lcms_category_id = categoryIdNode[0].firstChild.data
                    categoryNameNode = category.getElementsByTagName('Text')
                    if categoryNameNode:
                        lcms_category_name = categoryNameNode[0].firstChild.data
                    break

            # Qyality scoring ----------------------------------------
            attributes = doc.getElementsByTagName('Attribute')
            for attr in attributes:
                if attr.hasAttribute('key') and attr.getAttribute('key') == "isPlace":
                    isplace = attr.firstChild.data
                if attr.hasAttribute('key') and attr.getAttribute('key') == "isOpen":
                    isopen = attr.firstChild.data
                if attr.hasAttribute('key') and attr.getAttribute('key') == "isNameCorrect":
                    isname = attr.firstChild.data
                if attr.hasAttribute('key') and attr.getAttribute('key') == "isAddressCorrect":
                    isaddr = attr.firstChild.data
                if attr.hasAttribute('key') and attr.getAttribute('key') == "isPhoneCorrect":
                    isphone = attr.firstChild.data
                if attr.hasAttribute('key') and attr.getAttribute('key') == "reality_score":
                    reality_score = attr.firstChild.data

            if region != "LAM":
                if not lcms_category_id[:8] in filter_categories:
                    continue
            if check_counts(current_counts, float(reality_score)) == False:
                continue
            # place id --------------------------------------------------
            placeIdNode = doc.getElementsByTagName('PlaceId')
            if placeIdNode:
                placeid = placeIdNode[0].firstChild.data
            # name --------------------------------------------------------
            names = doc.getElementsByTagName('Name')
            for n in names:
                if n.hasAttribute('primaryFlag') and n.getAttribute('primaryFlag') == "true":
                    baseTextNodes = n.getElementsByTagName("BaseText")
                    for baseText in baseTextNodes:
                        name = baseTextNodes[0].firstChild.data
                        break
            # phone -------------------------------------------------------
            contacts = doc.getElementsByTagName("Contact")
            for contact in contacts:
                if contact.hasAttribute('type') and contact.getAttribute('type') == "PHONE":
                    phone = contact.getElementsByTagName('ContactString')[0].firstChild.data
                    break

            # house number ------------------------------------------------
            houseNumberNode = doc.getElementsByTagName('HouseNumber')
            if houseNumberNode:
                house_number = houseNumberNode[0].firstChild.data

            # street base name --------------------------------------------------
            streetNameNode = doc.getElementsByTagName('StreetName')
            if streetNameNode:
                street_basenameNode = streetNameNode[0].getElementsByTagName('BaseName')
                if street_basenameNode:
                    street_basename = street_basenameNode[0].firstChild.data
                street_typeNode = street_basenameNode[0].getElementsByTagName('StreetType')
                if street_typeNode:
                    street_type = street_typeNode[0].firstChild.data
            # city ----------------------------------------------------------------
            level4 = doc.getElementsByTagName('Level4')
            if level4:
                city = level4[0].firstChild.data
            # state ------------------------------------------------------------------
            level2 = doc.getElementsByTagName('Level2')
            if level2:
                state = level2[0].firstChild.data
            # postal -----------------------------------------------------------------
            postalNode = doc.getElementsByTagName('PostalCode')
            if postalNode:
                postal = postalNode[0].firstChild.data

            # category poi ---------------------------------------------
            categories = doc.getElementsByTagName('Category')
            for category in categories:
                if category.getAttribute('categorySystem') == "navteq-poi":
                    categoryIdNode = category.getElementsByTagName('CategoryId')
                    if categoryIdNode:
                        poi_category_id = categoryIdNode[0].firstChild.data
                    categoryNameNode = category.getElementsByTagName('Text')
                    if categoryNameNode:
                        poi_category_name = categoryNameNode[0].firstChild.data
                    break
            # coordinates ---------------------------------------------------
            geoPositoins = doc.getElementsByTagName('GeoPosition')
            for geoPosition in geoPositoins:
                if geoPosition.hasAttribute('type') and geoPosition.getAttribute('type') == "ROUTING":
                    latNode = geoPosition.getElementsByTagName('Latitude')
                    if latNode:
                        lat = latNode[0].firstChild.data
                    lonNode = geoPosition.getElementsByTagName('Longitude')
                    if lonNode:
                        lon = lonNode[0].firstChild.data
                    break
            # full house number ----------------------------------------------
            additionalDataNodes = doc.getElementsByTagName('AdditionalData')
            for additData in additionalDataNodes:
                if additData.hasAttribute('key') and additData.getAttribute('key') == "FullHouseNumber":
                    full_house_number = additData.firstChild.data
            if region == "EEU":
                csv_list.append(
                    placeid
                    +","+fix_quote(name)
                    +","+fix_quote(phone)
                    +","+fix_quote(house_number)
                    +","+fix_quote(street_basename)
                    +","+fix_quote(street_type)
                    +","+fix_quote(city)
                    +","+fix_quote(state)
                    +","+fix_quote(country)
                    +","+fix_quote(postal)
                    +","+fix_quote(country)
                    +","+fix_quote(poi_category_id)
                    +","+fix_quote(poi_category_name)
                    +","+fix_quote(lcms_category_id+"|"+lcms_category_name)
                    +","+lat
                    +","+lon
                    +","+isplace
                    +","+isopen
                    +","+isname
                    +","+isaddr
                    +","+isphone
                    +","+reality_score
                    +","+full_house_number
                    +","+district
                )
            else:
                csv_list.append(
                    placeid
                    +","+fix_quote(name)
                    +","+fix_quote(phone)
                    +","+fix_quote(house_number)
                    +","+fix_quote(street_basename)
                    +","+fix_quote(street_type)
                    +","+fix_quote(city)
                    +","+fix_quote(state)
                    +","+fix_quote(country)
                    +","+fix_quote(postal)
                    +","+fix_quote(country)
                    +","+fix_quote(poi_category_id)
                    +","+fix_quote(poi_category_name)
                    +","+fix_quote(lcms_category_id+"|"+lcms_category_name)
                    +","+lat
                    +","+lon
                    +","+isplace
                    +","+isopen
                    +","+isname
                    +","+isaddr
                    +","+isphone
                    +","+reality_score
                )

            if check_counts_finished(current_counts):
                break
        if check_counts_finished(current_counts):
            break

    return csv_list

def check_counts(current_counts, score):
    if score < 0.21:
        if current_counts['r1']>0:
            current_counts['r1'] = current_counts['r1'] - 1
            return True
        else:
            return False
    if score >= 0.21 and score < 0.28:
        if current_counts['r2']>0:
            current_counts['r2'] = current_counts['r2'] - 1
            return True
        else:
            return False
    if score >= 0.28 and score < 0.55:
        if current_counts['r3']>0:
            current_counts['r3'] = current_counts['r3'] - 1
            return True
        else:
            return False
    if score >= 0.55 and score < 0.8:
        if current_counts['r4']>0:
            current_counts['r4'] = current_counts['r4'] - 1
            return True
        else:
            return False
    if score >= 0.8:
        if current_counts['r5']>0:
            current_counts['r5'] = current_counts['r5'] - 1
            return True
        else:
            return False

def check_counts_finished(current_counts):
    c1 = current_counts['r1']
    c2 = current_counts['r2']
    c3 = current_counts['r3']
    c4 = current_counts['r4']
    c5 = current_counts['r5']
    if c1 == 0 and c2 == 0 and c3 == 0 and c4 == 0 and c5 == 0:
        return True
    return False

def save_csv_list(csv_list, country, region, kick_off_date):
    dir_path = os.path.dirname(os.path.realpath(__file__))
    file_name = os.path.join(dir_path, kick_off_date,region+"_"+country+"_"+kick_off_date+".csv")
    if region == "BIG7":
        file_name = os.path.join(dir_path, kick_off_date, country+"_"+kick_off_date+".csv")
    fo = open(file_name, 'wb')
    for line in csv_list:
        fo.write(line.encode('UTF-8')+"\r\n")
        # fo.write(line+"\t\n")

def fix_quote(input):
    if "," in input:
        return "\""+input+"\""
    return input

if __name__ == "__main__":
    main(sys.argv[1], sys.argv[2], sys.argv[3])


"""Runway config file module."""
import logging
import os
import sys
from typing import Any, Dict, List, Optional, Union, Iterator  # pylint: disable=unused-import
import yaml

LOGGER = logging.getLogger('runway')

class ConfigComponent(object):
    """Base class for Runway config components."""

    def get(self, key, default=None):
        # type: (str, Any) -> Any
        """Implement evaluation of get."""
        return getattr(self, key, getattr(self, key.replace('-', '_'), default))

    def __getitem__(self, key):
        # type: (str) -> Any
        """Implement evaluation of self[key]."""
        return getattr(self, key, getattr(self, key.replace('-', '_')))

    def __setitem__(self, key, value):
        # type: (str, Any) -> None
        """Implement evaluation of self[key] for assignment."""
        setattr(self, key, value)

    def __len__(self):
        # type: () -> int
        """Implement the built-in function len()."""
        return len(self.__dict__)

    def __iter__(self):
        # type: () -> Iterator[Any]
        """Return iterator object that can iterate over all attributes."""
        return iter(self.__dict__)

class ModuleDefinition(ConfigComponent):  # pylint: disable=too-many-instance-attributes
    """A module defines the directory to be processed and applicable options.

    It can consist of `CloudFormation`_ (using `Stacker`_),
    `Terraform`_, `Serverless Framework`_, `AWS CDK`_, or `Kubernetes`_.
    It is recommended to place the appropriate extension on each directory
    for identification (but it is not required). See
    :ref:`Repo Structure<repo-structure>` for examples of a module
    directory structure.

    +------------------+-----------------------------------------------+
    | Suffix/Extension | IaC Tool/Framework                            |
    +==================+===============================================+
    | ``.cdk``         | `AWS CDK`_                                    |
    +------------------+-----------------------------------------------+
    | ``.cfn``         | `CloudFormation`_                             |
    +------------------+-----------------------------------------------+
    | ``.sls``         | `Serverless Framework`_                       |
    +------------------+-----------------------------------------------+
    | ``.tf``          | `Terraform`_                                  |
    +------------------+-----------------------------------------------+
    | ``.k8s``         | `Kubernetes`_                                 |
    +------------------+-----------------------------------------------+

    A module is only deployed if there is a corresponding env/config
    present. This can take the form of either a file in the module folder
    or the ``environments`` option being defined. The naming format
    varies per-module type. See
    :ref:`Module Configurations<module-configurations>` for acceptable
    env/config file name formats.

    Modules can be defined as a string or a mapping. The minimum
    requirement for a module is a string that is equal to the name of
    the module directory. Providing a string is the same as providing a
    value for ``path`` in a mapping definition.

    Example:
      .. code-block:: yaml

        deployments:
          - modules:
              - my-module.cfn  # this
              - path: my-module.cfn  # is the same as this

    Using a map to define a module provides the ability to specify
    per-module ``options``, environment values, tags, and even a custom
    class for processing the module. The options that can be used with
    each module vary. For detailed information about module-specific
    options, see :ref:`Module Configurations<module-configurations>`.

    Example:
      .. code-block:: yaml

        deployments:
          - modules:
              - name: my-module
                path: my-module.tf
                environments:
                  dev:
                    image_id: ami-1234
                tags:
                  - app:example
                  - my-tag
                options:
                  terraform_backend_config:
                    region: us-east-1
                  terraform_backend_cfn_outputs:
                    bucket: StackName::OutputName
                    dynamodb_table: StackName::OutputName

    One special map keyword, ``parallel``, indicates a list of child
    modules that will be executed in parallel (simultaneously) if the
    ``CI`` :ref:`environment variable is set<non-interactive-mode>`.

    Example:
      .. code-block:: yaml

        deployments:
          - modules:
            - backend.tf
            - parallel:
              - servicea.cfn  # any normal module option can be used here
              - path: serviceb.cfn
              - path: servicec.cfn
                environments:
                  dev:
                    count: 1
                  prod:
                    count: 3
            - frontend.tf

    (in that ^ example, backend.tf will be deployed, the services will be all
    be deployed simultaneously, followed by frontend.tf)

    """

    def __init__(self,  # pylint: disable=too-many-arguments
                 name,  # type: str
                 path,  # type: str
                 class_path=None,  # type: Optional[str]
                 environments=None,  # type: Optional[Dict[str, Dict[str, Any]]]
                 env_vars=None,  # type: Optional[Dict[str, Dict[str, Any]]]
                 options=None,  # type: Optional[Dict[str, Any]]
                 tags=None,  # type: Optional[Dict[str, str]]
                 child_modules=None  # type: Optional[List[Union[str, Dict[str, Any]]]]
                 # pylint only complains for python2
                 ):  # pylint: disable=bad-continuation
        # type: (...) -> None
        """.. Runway module definition.

        Keyword Args:
            name (str): Name of the module. Used to more easily identify
                where different modules begin/end in the logs.
            path (str): Path to the module relative to the Runway config
                file. This cannot be higher than the Runway config file.
            class_path (Optional[str]): Path to custom Runway module class.
                Also used for static site deployments. See
                :ref:`Module Configurations<module-configurations>` for
                detailed usage.
            environments (Optional[Dict[str, Dict[str, Any]]]): Mapping for
                variables to environment names. When run, the variables
                defined here are merged with those in the
                ``.env``/``.tfenv``/environment config file. If this is
                defined, ``.env`` files can be omitted and the module
                will still be processed.
            env_vars (Optional[Dict[str, Dict[str, Any]]]): A mapping of
                OS environment variable overrides to apply when processing
                modules in the deployment. Can be defined per environment
                or for all environments using ``"*"`` as the environment
                name. Takes precendence over values set at the deployment-
                level.
            options (Optional[Dict[str, Any]]): Module-specific options.
                See :ref:`Module Configurations<module-configurations>`
                for detailed usage.
            tags (Optional[Dict[str, str]]): Module tags used to select
                which modules to process using CLI arguments.
                (``--tag <tag>...``)
            child_modules (Optional[List[Union[str, Dict[str, Any]]]]):
                Child modules that can be executed in parallel

        References:
            - `AWS CDK`_
            - `CloudFormation`_
            - `Serverless Framework`_
            - `Stacker`_
            - `Troposphere`_
            - `Terraform`_
            - `Kubernetes`_
            - :ref:`Module Configurations<module-configurations>` -
              detailed module ``options``
            - :ref:`Repo Structure<repo-structure>` - examples of
              directory structure
            - :ref:`command-deploy`
            - :ref:`command-destroy`
            - :ref:`command-plan`

        """
        self.name = name
        self.path = path
        self.class_path = class_path
        self.environments = environments or {}
        self.env_vars = env_vars or {}
        self.options = options or {}
        self.tags = tags or {}
        self.child_modules = child_modules

    @classmethod
    def from_list(cls, modules):
        """Instantiate ModuleDefinition from a list."""
        results = []
        for mod in modules:
            if isinstance(mod, str):
                results.append(cls(mod, mod, {}))
                continue
            if mod.get('parallel'):
                name = 'parallel_parent'
                child_modules = ModuleDefinition.from_list(mod.pop('parallel'))
                path = '[' + ', '.join([x.path for x in child_modules]) + ']'
                if mod:
                    LOGGER.warning(
                        'Invalid keys found in parallel module config have been ignored: %s',
                        ', '.join(mod.keys())
                    )
            else:
                name = mod.pop('name', mod['path'])
                child_modules = None
                path = mod.pop('path')
            results.append(cls(name,
                               path,
                               class_path=mod.pop('class_path', None),
                               environments=mod.pop('environments', {}),
                               env_vars=mod.pop('env_vars', {}),
                               options=mod.pop('options', {}),
                               tags=mod.pop('tags', {}),
                               child_modules=child_modules))
            if mod:
                LOGGER.warning(
                    'Invalid keys found in module %s have been ignored: %s',
                    name, ', '.join(mod.keys())
                )
        return results

class DeploymentDefinition(ConfigComponent):  # pylint: disable=too-many-instance-attributes
    """A deployment defines modules and options that affect the modules.

    Deployments are processed during a ``deploy``/``destroy``/``plan``
    action. If the processing of one deployment fails, the action will
    end.

    During a ``deploy``/``destroy`` action, the user has the option to
    select which deployment will run unless the ``CI`` environment
    variable is set, the ``--tag <tag>...`` cli option was provided, or
    only one deployment is defined.

    Example:
      .. code-block:: yaml

        deployments:
          - modules:  # minimum requirements for a deployment
              # "./" can alternatively be used for the module name to indicate
              # the current directory
              - my-module.cfn
            regions:
              - us-east-1
          - name: detailed-deployment  # optional
            modules:
              - path: my-other-modules.cfn
            regions:
              - us-east-1
            account_id:  # optional
              dev: 0000
              prod: 1111
            assume_role:  # optional
              dev: arn:aws:iam::0000:role/role-name
              prod: arn:aws:iam::1111:role/role-name
            environments:  # optional
              dev:
                region: us-east-1
                image_id: ami-abc123
            env_vars:  # optional environment variable overrides
              dev:
                AWS_PROFILE: foo
                APP_PATH:  # a list will be treated as components of a path on disk
                  - myapp.tf
                  - foo
              prod:
                AWS_PROFILE: bar
                APP_PATH:
                  - myapp.tf
                  - foo
              "*":  # applied to all environments
                ANOTHER_VAR: foo

    """

    def __init__(self, deployment):
        # type: (Dict[str, Any]) -> None
        """.. Runway deployment definition.

        Keyword Args:
            account_alias (Optional[Dict[str, str]]): A mapping of
                ``$environment: $alias`` that, if provided, is used to
                verify the currently assumed role or credentials.
            account_id (Optional[Dict[str, Union[str, int]]]): A mapping
                of ``$environment: $id`` that, if provided, is used to
                verify the currently assumed role or credentials.
            assume_role (Optional[Dict[str, Union[str, Dict[str, str]]]]):
                A mapping of ``$environment: $role`` or
                ``$environment: {arn: $role, duration: $int}`` to assume
                a role when processing a deployment. ``arn: $role`` can
                be used to apply the same role to all environment.
                ``post_deploy_env_revert: true`` can also be provided to
                revert credentials after processing.
            environments (Optional[Dict[str, Dict[str, Any]]]): Mapping for
                variables to environment names. When run, the variables
                defined here are merged with those in the
                ``.env``/``.tfenv``/environment config file and
                environments section of each module.
            env_vars (Optional[Dict[str, Dict[str, Any]]]): A mapping of
                OS environment variable overrides to apply when processing
                modules in the deployment. Can be defined per environment
                or for all environments using ``"*"`` as the environment
                name.
            modules (Optional[List[Dict[str, Any]]]): A list of modules
                to be processed in the order they are defined.
            module_options (Optional[Dict[str, Any]]): Options that are
                shared among all modules in the deployment.
            name (str): Name of the deployment. Used to more easily
                identify where different deployments begin/end in the logs.
            regions (List[str]): AWS region names where modules will be
                deployed/destroyed. Can optionally define as a map with
                ``parallel`` as the key and a list of regions as the value.
                See **parallel_regions** for more info.
            parallel_regions: Can be defined in place of ``regions.parallel[]``.
                This will cause all modules in the deployment to be executed
                in all provided regions in parallel (at the same time).
                Only takes effect when the ``CI`` environment variable is set,
                enabling non-interactive mode, as prompts will not be able
                to be presented. If ``CI`` is not set, the regions will be
                processed one at a time. This can be used in tandom with
                **parallel modules**. ``assume_role.post_deploy_env_revert``
                will always be ``true`` when run in parallel.

        References:
            - :class:`module<runway.config.ModuleDefinition>`
            - :ref:`command-deploy`
            - :ref:`command-destroy`
            - :ref:`command-plan`

        """
        self.account_alias = deployment.pop(
            'account_alias', deployment.pop('account-alias', {})
        )  # type: Optional[Dict[str, str]]
        self.account_id = deployment.pop(
            'account_id', deployment.pop('account-id', {})
        )  # type: Optional[Dict[str, Union[str, int]]]
        self.assume_role = deployment.pop(
            'assume_role', deployment.pop('assume-role', {})
        )  # type: Optional[Dict[str, Union[str, Dict[str, str]]]]
        self.environments = deployment.pop(
            'environments', {}
        )  # type: Optional[Dict[str, Dict[str, Any]]]
        self.env_vars = deployment.pop(
            'env_vars', deployment.pop('env-vars', {})
        )  # type: Optional[Dict[str, Dict[str, Any]]]
        if deployment.pop('current_dir', False):
            # Deprecated in 1.0 (late 2019). Retain for at least a major version.
            LOGGER.warning('DEPRECATION WARNING: The "current_dir" option has '
                           'been deprecated in favor of a "./" module '
                           'definition. Please update your config.')
            modules = ['.' + os.sep]
        else:
            if not deployment.get('modules'):
                LOGGER.error('No modules have been defined in your Runway '
                             'deployment.')
                sys.exit(1)
            modules = deployment.pop('modules')
        self.modules = ModuleDefinition.from_list(
            modules
        )  # type: List[ModuleDefinition]
        self.module_options = deployment.pop(
            'module_options', deployment.pop('module-options', {})
        )  # type: Optional(Dict[str, Any])
        self.name = deployment.pop('name')  # type: str
        self.regions = deployment.pop(
            'regions', []
        )  # type: Union[List[str], Dict[str, List[str]]]

        if self.regions and deployment.get('parallel_regions'):
            LOGGER.error('Found "regions" and "parallel_regions" in '
                         'deployment "%s"; only one can be defined',
                         self.name)
            sys.exit(1)
        if isinstance(self.regions, dict) and self.regions.get('parallel'):
            self.parallel_regions = self.regions.pop('parallel')
            self.regions = []
        else:
            self.parallel_regions = deployment.pop('parallel_regions', [])

        if deployment:
            LOGGER.warning(
                'Invalid keys found in deployment %s have been ignored: %s',
                self.name, ', '.join(deployment.keys())
            )

    @classmethod
    def from_list(cls, deployments):
        """Instantiate DeploymentDefinitions from a list."""
        results = []
        for i, deployment in enumerate(deployments):
            if not deployment.get('name'):
                deployment['name'] = 'deployment_{}'.format(str(i + 1))
            results.append(cls(deployment))
        return results

class TestDefinition(ConfigComponent):
    """Tests can be defined as part of the Runway config file.

    This is to remove the need for complex Makefiles or scripts to initiate
    test runners. Simply define all tests for a project in the Runway
    config file and use the ``runway test`` :ref:`command<command-test>`
    to execute them.

    Example:
      .. code-block:: yaml

        tests:
          - name: my-test
            type: script
            required: false
            args:
              commands:
                - echo "Hello World!"

    """

    def __init__(self,
                 name,  # type: str
                 test_type,  # type: str
                 args=None,  # type: Optional[Dict[str, Any]]
                 required=True  # type: bool
                 # pylint only complains for python2
                 ):  # pylint: disable=bad-continuation
        # type: (...) -> None
        """.. Runway test definitions.

        Keyword Args:
            name (str): Name of the test. Used to more easily identify
                where different tests begin/end in the logs.
            type (str): The type of test to run. See
                :ref:`Build-in Test Types<built-in-test-types>`
                for supported test types.
            args (Optional[Dict[str, Any]]): Arguments to be passed to
                the test. Supported arguments vary by test type. See
                :ref:`Build-in Test Types<built-in-test-types>` for the
                list of arguments supported by each test type.
            required (bool):  If false, testing will continue if the test
                fails. *(default: true)*

        References:
            - :ref:`Build-in Test Types<built-in-test-types>` - Supported
              test types and their
              arguments
            - :ref:`test command<command-test>`

        """
        self.name = name
        self.type = test_type
        self.args = args or {}
        self.required = required

    @classmethod
    def from_list(cls, tests):
        # type: (List[Dict[str, Any]]) -> List[TestDefinition]
        """Instantiate TestDefinitions from a list."""
        results = []

        for index, test in enumerate(tests):
            name = test.pop('name', 'test_{}'.format(index + 1))
            results.append(cls(name, test.pop('type'),
                               test.pop('args', {}),
                               test.pop('required', False)))

            if test:
                LOGGER.warning(
                    'Invalid keys found in test %s have been ignored: %s',
                    name, ', '.join(test.keys())
                )
        return results

class Config(ConfigComponent):
    """The Runway config file is where all options are defined.

    It contains definitions for deployments, tests, and some global
    options that impact core functionality.

    The Runway config file can have two possible names, ``runway.yml``
    or ``runway.yaml``. It must be stored at the root of the directory
    containing all modules to be deployed.

    Example:
        .. code-block:: yaml

            ---
            # See full syntax at https://github.com/onicagroup/runway
            ignore_git_branch: true
            tests:
              - name: example
                type: script
                args:
                  commands:
                    - echo "Hello world"
            deployments:
              - modules:
                  - path: my-modules.cfn
                regions:
                  - us-east-1

    """

    accepted_names = ['runway.yml', 'runway.yaml']

    def __init__(self,
                 deployments,  # type: List[Dict[str, Any]]
                 tests=None,  # type: List[Dict[str, Any]]
                 ignore_git_branch=False  # type: bool
                 # pylint only complains for python2
                 ):  # pylint: disable=bad-continuation
        # type: (...) -> None
        """.. Top-level Runway config file.

        Keyword Args:
            deployments (List[Dict[str, Any]]): A list of
                :class:`deployments<runway.config.DeploymentDefinition>`
                that are processed in the order they are defined.
            tests (Optional[List[Dict[str, Any]]]): A list of
                :class:`tests<runway.config.TestDefinition>` that are
                processed in the order they are defined.
            ignore_git_branch (bool): Disable git branch lookup when
                using environment folders, non-git VCS, or defining the
                ``DEPLOY_ENVIRONMENT`` environment variable before
                execution. Note that defining ``DEPLOY_ENVIRONMENT``
                will automatically ignore the git branch.

        References:
            - :class:`deployment<runway.config.DeploymentDefinition>`
            - :class:`test<runway.config.TestDefinition>`

        """
        self.deployments = DeploymentDefinition.from_list(deployments)
        self.tests = TestDefinition.from_list(tests)
        self.ignore_git_branch = ignore_git_branch

    @classmethod
    def load_from_file(cls, config_path):
        # type: (str) -> Config
        """Load config file into a Config object."""
        if not os.path.isfile(config_path):
            LOGGER.error("Runway config file was not found (looking for "
                         "%s)",
                         config_path)
            sys.exit(1)
        with open(config_path) as data_file:
            config_file = yaml.safe_load(data_file)
            result = Config(config_file.pop('deployments'),
                            config_file.pop('tests', []),
                            config_file.pop('ignore_git_branch',
                                            config_file.pop(
                                                'ignore-git-branch',
                                                False)))

            if config_file:
                LOGGER.warning(
                    'Invalid keys found in runway file have been ignored: %s',
                    ', '.join(config_file.keys())
                )
            return result

    @classmethod
    def find_config_file(cls, config_dir=None):
        # type: (Optional[str]) -> str
        """Find the Runway config file."""
        if not config_dir:
            config_dir = os.getcwd()

        for name in cls.accepted_names:
            conf_path = os.path.join(config_dir, name)
            if os.path.isfile(conf_path):
                return conf_path

        LOGGER.error('Runway config file was not found. Looking for one '
                     'of %s in %s', str(cls.accepted_names), config_dir)
        sys.exit(1)


from django.shortcuts import render
from .models import Information
import datetime

def information(request):
    now = datetime.datetime.now()
    nowDate = now.strftime('%Y-%m-%d')
    print(nowDate)
    info = Information.objects.all()
    return render(request, 'information/info.html', {
        'info': info,
        'today': nowDate,
    })


import pygame

#  Константи
FPS = 100                                           #   100 кадрів в секунду, або 1 кадр кожні 10 мілісекунд
WINDOW_BACKGROUND_COLOR_WHITE = (255, 255, 255)     #   білий колір, яким ми зафарбовуємо екран (в форматі RGB)
WINDOW_SIZE = (1000, 800)                           #   розміри вікна (розширення)
pygame.init()                                       #   початкові налаштування бібліотеки  pygame
window = pygame.display.set_mode(WINDOW_SIZE)       #   створити вікна програми з заданими розмірами
clock = pygame.time.Clock()                         #   створити годинник

class Panda:
    # створити панду з такими параметрами: картинка панди, координати x та y, ім'я панди
    def __init__(self, image_file, start_x, start_y, panda_name):
        self.image = pygame.image.load(image_file)          #   завантажуємо картинку в пам'ять програми
        self.rectangle = self.image.get_rect()              #   прямокутник навколо картинки
        self.name = panda_name                              #   ім'я панди
        self.rectangle.x = start_x                          #   початкова координата по ширині
        self.rectangle.y = start_y                          #   початкова координата по висоті
        print("Panda " + panda_name + " created")

    # вивести повідомлення, якщо інша панда поряд
    def show_message_if_another_panda_is_near(self, other_panda):
        is_near = self.rectangle.colliderect(other_panda.rectangle) #  colliderect - це вбудована функція бібліотеки pygame
                                                                    #  вона перевіряє, що прямокутники навколо двох панд накладаються один на інший
        if is_near:
            print("I am panda " + self.name + ". Another panda " + other_panda.name + " is near me!")

    # намалювати панду на вікні
    def draw_panda_on_window(self):
        global window
        window.blit(self.image, self.rectangle)     # функція blit - вбудована функція бібліотеки pygame

    # змінити координати панди
    def move(self, x_diff, y_diff, direction_text):
        print("Panda " + self.name + " is moved to: " + direction_text)
        self.rectangle.x = self.rectangle.x + x_diff
        self.rectangle.y = self.rectangle.y + y_diff

# Перевірити, чи треба завершити програму
# Подія pygame.QUIT відбудеться, якщо натиснути на хрестик закриття вікна програми
def need_to_close_proram():
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            return True
    return False

# набори параметрів для функції
# в них показано:  на скільки змінити координати панди по висоті, по ширині і як ця операція буде називатися

MOVE_UP =       (0, -10, "UP")
MOVE_DOWN =     (0, +10, "DOWN")
MOVE_LEFT =     (-10, 0, "LEFT")
MOVE_RIGHT =    (+10, 0, "RIGHT")

# функція яка перевіряє усі натиснуті кнопки і відповідно рухає панду
def move_panda_if_needed(panda):
    all_pressed_keys = pygame.key.get_pressed()

    if all_pressed_keys[pygame.K_LEFT]:
        panda.move(*MOVE_LEFT)

    if all_pressed_keys[pygame.K_RIGHT]:
        panda.move(*MOVE_RIGHT)

    if all_pressed_keys[pygame.K_UP]:
        panda.move(*MOVE_UP)

    if all_pressed_keys[pygame.K_DOWN]:
        panda.move(*MOVE_DOWN)

panda1 = Panda('panda.jpg', 100, 100, "Перша панда")
panda2 = Panda('panda.jpg', 100, 100, "Друга панда")
running = True                                             #   флаг завершення програми

while running:

    move_panda_if_needed(panda1)                            #   перемістити панду 1, якщо натиснуті кнопки вниз вверз вліво вправо

    # перемалювари повністю вікно
    window.fill(WINDOW_BACKGROUND_COLOR_WHITE)              #   намалювати білий екран
    panda1.draw_panda_on_window()                           #   намалювати панду 1
    panda2.draw_panda_on_window()                           #   намалювати панду 2
    pygame.display.update()                                 #   оновити екран
    panda1.show_message_if_another_panda_is_near(panda2)    #   вивести повідомлення, якщо поряд інша панда

    running = not need_to_close_proram()                    #   індикатор завершення програми (використовується в циклі while)

    clock.tick(FPS)                                         #   почекати 10 мілісекунд


import re
import sys
import os
sys.path.insert(1, os.path.join(sys.path[0], '..'))
import utils.utils as ut
from copy import deepcopy
import math
from functools import reduce

def get_active(grid, i, j, k, m):
    num = 0
    for dm in range(-1, 2):
        for di in range(-1, 2):
            for dj in range(-1, 2):
                for dk in range(-1, 2):
                    if grid[i+di][j+dj][k+dk][m+dm] == '#':
                        num += 1

    act  = 1 if grid[i][j][k][m] == '#' else 0
    num -= act
    return [act, num]

def run(grid, num):
    grid2 = [[[['.' for m in range(num)] for k in range(num)] for j in range(num)] for i in range(num)]
    for m in range(1, num-1):
        for i in range(1, num-1):
            for j in range(1, num-1):
                for k in range(1, num-1):
                    act = get_active(grid, i, j, k, m)
                    if (act[0] == 1) and (act[1] == 2 or act[1] == 3):
                        grid2[i][j][k][m] = '#'
                    if (act[0] == 0) and (act[1] == 3):
                        grid2[i][j][k][m] = '#'
    return grid2

def count_active(grid, num):
    count = 0
    for m in range(1, num-1):
        for i in range(1, num-1):
            for j in range(1, num-1):
                for k in range(1, num-1):
                    if grid[i][j][k][m] == '#':
                        count += 1
    return count

def part2(data, cycles):
    num  = (len(data) + (cycles + 3) * 2)
    off  = num // 2 - 1
    print(num, off)

    grid = [[[['.' for m in range(num)] for k in range(num)] for j in range(num)] for i in range(num)]
    for i in range(len(data)):
        for j in range(len(data[i])):
            grid[off][off][i+off][j+off] = data[i][j]

    for c in range(cycles):
        grid = run(grid, num)

    return count_active(grid, num)

def solution(file):
    print("Input: ", file)
    p2 = part2(ut.read_lines(file), 6)
    print("Part2: ", p2)

def main():
    solution("day17-test.txt")
    solution("day17-input.txt")

if __name__ == '__main__':
    main()


import os
import unittest
from deta import Deta, send_email
from timeit import timeit

try:
    from dotenv import load_dotenv

    load_dotenv()
except:
    pass

"""class TestSendEmail(unittest.TestCase):
    def setUp(self):
        self.deta = Deta()

    def test_function(self):
        self.assertIsNone(
            send_email("mustafa@deta.sh", "Hello from test", "this is a test!")
        )

    def test_method(self):
        self.assertIsNone(
            self.deta.send_email(
                "mustafa@deta.sh", "Hello from test", "this is a test!"
            )
        )"""

class TestBaseMethods(unittest.TestCase):
    def setUp(self):
        key = os.getenv("DETA_SDK_TEST_PROJECT_KEY")
        name = os.getenv("DETA_SDK_TEST_BASE_NAME")
        self.assertIsNotNone(key)
        self.assertIsNotNone(name)
        deta = Deta(key)
        self.db = deta.Base(str(name))
        self.item1 = {"key": "existing1", "value": "test"}
        self.item2 = {"key": "existing2", "value": 7}
        self.item3 = {"key": "existing3", "value": 44}
        self.item4 = {"key": "existing4", "value": {"name": "patrick"}}
        self.item5 = {"key": "%@#//#!#)#$_", "value": 0, "list": ["a"]}
        self.db.put_many([self.item1, self.item2, self.item3, self.item4, self.item5])

    def tearDown(self):
        all_items = next(self.db.fetch())
        for item in all_items:
            self.db.delete(item["key"])
        self.db.client.close()

    def test_put(self):
        item = {"msg": "hello"}
        resp = {"key": "one", "msg": "hello"}
        self.assertEqual(self.db.put(item, "one"), resp)
        self.assertEqual(self.db.put(item, "one"), resp)
        self.assertEqual({"msg": "hello"}, item)
        self.assertEqual(set(self.db.put("Hello").keys()), set(["key", "value"]))
        self.assertEqual(set(self.db.put(1).keys()), set(["key", "value"]))
        self.assertEqual(set(self.db.put(True).keys()), set(["key", "value"]))
        self.assertEqual(set(self.db.put(False).keys()), set(["key", "value"]))
        self.assertEqual(set(self.db.put(3.14159265359).keys()), set(["key", "value"]))

    @unittest.expectedFailure
    def test_put_fail(self):
        self.db.put({"msg": "hello"}, 1)
        self.db.put({"msg": "hello", "key": True})

    def test_put_many(self):
        self.assertEqual(len(self.db.put_many([1, 2, 3])["processed"]["items"]), 3)
        ok = self.db.put_many([{"msg": "hello"}, {"msg2": "hi"}])["processed"]["items"]
        self.assertEqual(len(ok), 2)

    @unittest.expectedFailure
    def test_put_many_fail(self):
        self.db.put_many([{"name": "joe", "key": "ok"}, {"name": "mo", "key": 7}])

    @unittest.expectedFailure
    def test_put_many_fail_limit(self):
        self.db.put_many([i for i in range(26)])

    def test_insert(self):
        item = {"msg": "hello"}
        self.assertEqual(set(self.db.insert(item).keys()), set(["key", "msg"]))
        self.assertEqual({"msg": "hello"}, item)

    @unittest.expectedFailure
    def test_insert_fail(self):
        self.db.insert(self.item1)

    def test_get(self):
        self.assertEqual(self.db.get(self.item1["key"]), self.item1)
        self.assertIsNone(self.db.get("key_does_not_exist"))

    def test_delete(self):
        self.assertIsNone(self.db.delete(self.item1["key"]))
        self.assertIsNone(self.db.delete("key_does_not_exist"))

    def test_fetch(self):
        res1 = next(self.db.fetch({"value": "test"}))
        res2 = next(self.db.fetch({"valuexyz": "test_none_existing_value"}))
        res3 = next(self.db.fetch(buffer=3))
        res4 = list(self.db.fetch(buffer=1, pages=4))
        res5 = next(self.db.fetch({"value.name": self.item4["value"]["name"]}))
        res6 = next(self.db.fetch({"value?gte": 7}))
        res7 = next(self.db.fetch([{"value?gt": 6}, {"value?lt": 50}]))
        self.assertTrue(len(res1) > 0)
        self.assertTrue(len(res2) == 0)
        self.assertTrue(len(res3) == 3)
        self.assertTrue(len(res4) == 4)
        self.assertTrue(len(res5) > 0)
        self.assertTrue(len(res6) == 2)
        self.assertTrue(len(res7) == 3)

    def test_update(self):
        self.assertIsNone(self.db.update({"value.name": "spongebob"}, "existing4"))
        expectedItem = {"key": "existing4", "value": {"name": "spongebob"}}
        self.assertEqual(self.db.get("existing4"), expectedItem)

        self.assertIsNone(
            self.db.update({"value.name": self.db.util.trim(), "value.age": 32}, "existing4")
        )
        expectedItem = {"key": "existing4", "value": {"age": 32}}
        self.assertEqual(self.db.get("existing4"), expectedItem)

        self.assertIsNone(
            self.db.update(
                {
                    "list": self.db.util.append(["b", "c"]),
                    "value": self.db.util.increment(),
                },
                "%@#//#!#)#$_",
            )
        )

        self.assertIsNone(
            self.db.update(
                {"list": self.db.util.prepend("x"), "value": self.db.util.increment(2)},
                "%@#//#!#)#$_",
            )
        )
        expectedItem = {"key": "%@#//#!#)#$_", "list": ["x", "a", "b", "c"], "value": 3}
        self.assertEqual(self.db.get("%@#//#!#)#$_"), expectedItem)

        # key does not exist
        self.assertRaises(Exception, self.db.update, {"value": "test"}, "doesNotExist")
        # deleting a key
        self.assertRaises(
            Exception,
            self.db.update,
            {"value": "test", "key": self.db.util.trim()},
            "existing4",
        )
        # updating a key
        self.assertRaises(Exception, self.db.update, {"key": "test"}, "existing4")
        # upper hierarchy does not exist
        self.assertRaises(Exception, self.db.update, {"profile.age": 32}, "existing4")
        # no attributes specified
        self.assertRaises(Exception, self.db.update, {}, "existing4")

        # appending to a key
        self.assertRaises(
            Exception,
            self.db.update,
            {"key": self.db.util.append("test")},
            "%@#//#!#)#$_",
        )

if __name__ == "__main__":
    unittest.main()