from __future__ import print_function
import os,time,cv2, sys, math
import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np
import time, datetime
import argparse
import random
import os, sys
import subprocess
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
from tensorflow.python.client import device_lib

import pywt
import helpers
import utils_ as utils
from utils_ import get_model
from pywt import wavedec2
import matplotlib.pyplot as plt

#from custom_model import build_encoder_decoder_skip
config = tf.ConfigProto(log_device_placement=False)

def str2bool(v):
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')
config.gpu_options.allow_growth = True
parser = argparse.ArgumentParser()
parser.add_argument('--num_epochs', type=int, default=100, help='Number of epochs to train for')
parser.add_argument('--save', type=int, default=4, help='Interval for saving weights')
parser.add_argument('--gpu', type=str, default='0', help='Choose GPU device to be used')
parser.add_argument('--mode', type=str, default="train", help='Select "train", "test", or "predict" mode. \
    Note that for prediction mode you have to specify an image to run the model on.')
parser.add_argument('--checkpoint', type=str, default="checkpoint", help='Checkpoint folder.')
parser.add_argument('--class_balancing', type=str2bool, default=False, help='Whether to use median frequency class weights to balance the classes in the loss')
parser.add_argument('--image', type=str, default=None, help='The image you want to predict on. Only valid in "predict" mode.')
parser.add_argument('--continue_training', type=str2bool, default=False, help='Whether to continue training from a checkpoint')
parser.add_argument('--dataset', type=str, default='lashan', help='Dataset you are using.')
parser.add_argument('--load_data', type=str2bool, default=True, help='Dataset loading type.')
parser.add_argument('--act', type=str2bool, default=True, help='True if sigmoid or false for softmax')
parser.add_argument('--crop_height', type=int, default=224, help='Height of cropped input image to network')
parser.add_argument('--crop_width', type=int, default=224, help='Width of cropped input image to network')
parser.add_argument('--batch_size', type=int, default=8, help='Number of images in each batch')
parser.add_argument('--num_val_images', type=int, default=200, help='The number of images to used for validations')
parser.add_argument('--h_flip', type=str2bool, default=True, help='Whether to randomly flip the image horizontally for data augmentation')
parser.add_argument('--v_flip', type=str2bool, default=True, help='Whether to randomly flip the image vertically for data augmentation')
parser.add_argument('--brightness', type=float, default=None, help='Whether to randomly change the image brightness for data augmentation. Specifies the max bightness change.')
parser.add_argument('--rotation', type=float, default=None, help='Whether to randomly rotate the image for data augmentation. Specifies the max rotation angle.')
parser.add_argument('--model', type=str, default="dunet", help='The model you are using. Currently supports:\
    encoder-decoder, deepUNet,attentionNet, deep, UNet')
args = parser.parse_args()
gpu = str(args.gpu)
os.environ['CUDA_VISIBLE_DEVICES']=  gpu
#os.environ["CUDA_VISIBLE_DEVICES"] = '0'
# Get a list of the training, validation, and testing file paths

def load_divided(dataset_dir):
    train_input_names=[]
    train_output_names=[]
    val_input_names=[]
    val_output_names=[]
    test_input_names=[]
    test_output_names=[]
    cwd = os.getcwd()

    for file in os.listdir(dataset_dir + "/train"):
        train_input_names.append(cwd + "/" + dataset_dir + "/train/" + file)
    for file in os.listdir(dataset_dir + "/train_labels"):
        train_output_names.append(cwd + "/" + dataset_dir + "/train_labels/" + file)
    for file in os.listdir(dataset_dir + "/val"):
        val_input_names.append(cwd + "/" + dataset_dir + "/val/" + file)
    for file in os.listdir(dataset_dir + "/val_labels"):
        val_output_names.append(cwd + "/" + dataset_dir + "/val_labels/" + file)
    if args.mode =='test':
        for file in os.listdir(dataset_dir + "/test"):
            test_input_names.append(cwd + "/" + dataset_dir + "/test/" + file)
        for file in os.listdir(dataset_dir + "/test_labels"):
            test_output_names.append(cwd + "/" + dataset_dir + "/test_labels/" + file)
    return train_input_names, train_output_names, val_input_names, val_output_names, test_input_names, test_output_names

def load_nondivided(dataset_dir):
    train_input_names=[]
    train_output_names=[]
    val_input_names=[]
    val_output_names=[]
    test_input_names=[]
    test_output_names=[]

    all_files = []
    cwd = os.getcwd()
    for file in os.listdir(dataset_dir + "/sat"):
        all_files.append(file)

    selected_val = random.sample(all_files, len(all_files)//18)
    train_input_names= [ cwd + "/" + dataset_dir + "/sat/" + name for name in all_files if name not in selected_val]
    train_output_names= [ cwd + "/" + dataset_dir + "/gt/" + name for name in all_files if name not in selected_val]
    val_input_names= [ cwd + "/" + dataset_dir + "/sat/" + name for name in selected_val]
    val_output_names= [ cwd + "/" + dataset_dir + "/gt/" + name for name in selected_val]
    print('Training data length : {} and validation data length: {}'.format(len(train_input_names), len(val_input_names)))
    return train_input_names,train_output_names, val_input_names, val_output_names, test_input_names, test_output_names

def prepare_data(dataset_dir=args.dataset, type=args.load_data):
    if type:
        train_input_names, train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = load_divided(dataset_dir)
    else:
        train_input_names, train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = load_nondivided(dataset_dir)

    print('Training data length : {} and validation data length: {}'.format(len(train_input_names), len(val_input_names)))
    train_input_names.sort(),train_output_names.sort(), val_input_names.sort(), val_output_names.sort(), test_input_names.sort(), test_output_names.sort()
    return train_input_names,train_output_names, val_input_names, val_output_names, test_input_names, test_output_names

def check_available_gpus():
    local_devices = device_lib.list_local_devices()
    gpu_names = [x.name for x in local_devices if x.device_type == 'GPU']
    gpu_num = len(gpu_names)

    print('{0} GPUs are detected : {1}'.format(gpu_num, gpu_names))

    return gpu_names

def get_names(gpus):
    n_gpus = len(gpus)
    if n_gpus == 1:
        return gpus[0],gpus[0], gpus[0], gpus[0]
    if n_gpus == 2:
        return gpus[0],gpus[0], gpus[1], gpus[1]
    if n_gpus == 3:
        return gpus[0], gpus[1], gpus[2], gpus[0]
    if n_gpus == 4:
        return gpus[0], gpus[1], gpus[2], gpus[3]

def load_image(path):
    #print(path)
    image = cv2.cvtColor(cv2.imread(path,-1), cv2.COLOR_BGR2RGB)
    h, w = args.crop_height, args.crop_width
    image = cv2.resize(image,(h, w))

    return image

def load_image_gray(path):
    img = cv2.imread(path, -1)

    image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    image = cv2.resize(image,(256, 256))
    return image

def data_augmentation(input_image, output_image):
    # Data augmentation
    #print (input_image.shape, output_image.shape)
    dice = random.random()
    crop_width, crop_height = args.crop_width, args.crop_height
    if dice >= 0.15:
        crop_height, crop_width = input_image.shape[1], input_image.shape[0]
    input_image, output_image = utils.random_crop(input_image, output_image, args.crop_height, args.crop_width)

    if args.h_flip and random.randint(0,1):
        input_image = cv2.flip(input_image, 1)
        output_image = cv2.flip(output_image, 1)
    if args.v_flip and random.randint(0,1):
        input_image = cv2.flip(input_image, 0)
        output_image = cv2.flip(output_image, 0)
    if args.brightness:
        factor = random.uniform(-1*args.brightness, args.brightness)
        table = np.array([((i / 255.0) ** factor) * 255 for i in np.arange(0, 256)]).astype(np.uint8)
        input_image = cv2.LUT(input_image, table)
    if args.rotation:
        angle = random.uniform(-1*args.rotation, args.rotation)
    if args.rotation:
        M = cv2.getRotationMatrix2D((input_image.shape[1]//2, input_image.shape[0]//2), angle, 1.0)
        input_image = cv2.warpAffine(input_image, M, (input_image.shape[1], input_image.shape[0]), flags=cv2.INTER_NEAREST)
        output_image = cv2.warpAffine(output_image, M, (output_image.shape[1], output_image.shape[0]), flags=cv2.INTER_NEAREST)

    return input_image, output_image

def download_checkpoints(model_name):
    subprocess.check_output(["python", "get_pretrained_checkpoints.py", "--model=" + model_name])

# Get the names of the classes so we can record the evaluation results
class_names_list, label_values = helpers.get_label_info(os.path.join(args.dataset, "class_dict.csv"))
class_names_string = ""
for class_name in class_names_list:
    if not class_name == class_names_list[-1]:
        class_names_string = class_names_string + class_name + ", "
    else:
        class_names_string = class_names_string + class_name
print('LABEL VALUES: ',label_values)
print(class_names_string)
num_classes = len(label_values)
gpus = check_available_gpus()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess=tf.Session(config=config)
model = args.model

# Get the selected model.
# Some of they require pre-trained ResNet
print("Preparing the model ...", model)

input = tf.placeholder(tf.float32,shape=[None, None, None, 3], name='inputs')
output = tf.placeholder(tf.float32,shape=[None, None, None, num_classes], name='output')

keep_prob = tf.placeholder(tf.float32)
input_A = tf.split(input, int(len(gpus)))
output_A = tf.split(output, int(len(gpus)))

# Load the data
print("Loading the data ...")
train_input_names,train_output_names, val_input_names, val_output_names, test_input_names, test_output_names = prepare_data()

if model == 'sunet':
    aux_output = tf.placeholder(tf.float32, shape=[None, None, None, num_classes], name='aux')

network = None
init_fn = None
if model != 'ssunet':
    network = get_model(model, input, num_classes, keep_prob, args.gpu)
else:
    network, _ = get_model(model, input, num_classes, keep_prob, args.gpu)
act = args.act
# Compute your softmax cross entropy loss
loss = None
loss_l = []
#sigs = ['unet', 'Unet', 'UNet', 'deepunet', 'deepUnet', 'dlink', 'fusion', 'dunet', 'newunet', 'deep', 'dil']
if args.class_balancing:
    print("Computing class weights for", args.dataset, "...")
    class_weights = utils.compute_class_weights(labels_dir=args.dataset + "/train_labels", label_values=label_values)
    if act:
        for gpu_id in range(len(gpus)):
            with tf.device(tf.DeviceSpec(device_type='GPU', device_index=gpu_id)):
                with tf.variable_scope(tf.get_variable_scope(), reuse=(gpu_id > 0)):
                    network = get_model(model, input_A[gpu_id], num_classes, keep_prob, args.gpu)
                    unweighted_loss = (tf.nn.sigmoid_cross_entropy_with_logits(logits=network, labels=output_A[gpu_id]))
                    loss_l.append((unweighted_loss * class_weights))
    else:
        unweighted_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=network, labels=output))
        loss = tf.reduce_mean(unweighted_loss * class_weights)
else:
    if act:
        for gpu_id in range(len(gpus)):
            with tf.device(tf.DeviceSpec(device_type='GPU', device_index=gpu_id)):
                with tf.variable_scope(tf.get_variable_scope(), reuse=(gpu_id > 0)):
                    network = get_model(model, input_A[gpu_id], num_classes, keep_prob, args.gpu)
                    #network = tf.sigmoid(network)
                    #_loss = utils.dice_loss(network, output_A[gpu_id])
                    _loss = (tf.nn.sigmoid_cross_entropy_with_logits(logits=network, labels=output_A[gpu_id]))
                    loss_l.append(_loss)
        #loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=network, labels=output))
    else:
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=network, labels=output))
    #network = tf.nn.sigmoid(network)
    #loss = tf.losses.mean_squared_error(predictions=network, labels=output)
#loss = tf.reduce_mean(loss_l)
loss = tf.reduce_mean(tf.concat(loss_l, axis=0))
opt = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss, var_list=[var for var in tf.trainable_variables()],colocate_gradients_with_ops=True)
saver=tf.train.Saver(max_to_keep=1000)
sess.run(tf.global_variables_initializer())

utils.count_params()

# If a pre-trained ResNet is required, load the weights.
# This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer())
if init_fn is not None:
    init_fn(sess)

# Load a previous checkpoint if desired
check = args.checkpoint
if not os.path.isdir(check):
    os.makedirs(check)
#model_checkpoint_name = "/media/cesar/My Passport/models" + args.model + "_" + args.dataset + ".ckpt"
model_checkpoint_name = check+"/latest_model_" + args.model + "_" + args.dataset + ".ckpt"

if args.continue_training or not args.mode == "train":
    print('Loaded latest model checkpoint')
    saver.restore(sess, model_checkpoint_name)

avg_scores_per_epoch = []

if args.mode == "train":

    print("\n***** Begin training *****")
    print("Dataset -->", args.dataset)
    print("Model -->", args.model)
    print("Crop Height -->", args.crop_height)
    print("Crop Width -->", args.crop_width)
    print("Num Epochs -->", args.num_epochs)
    print("Batch Size -->", args.batch_size)
    print("Num Classes -->", num_classes)

    print("Data Augmentation:")
    print("\tVertical Flip -->", args.v_flip)
    print("\tHorizontal Flip -->", args.h_flip)
    print("\tBrightness Alteration -->", args.brightness)
    print("\tRotation -->", args.rotation)
    print("")

    avg_loss_per_epoch = []

    # Which validation images do we want
    val_indices = []
    num_vals = min(args.num_val_images, len(val_input_names))

    # Set random seed to make sure models are validated on the same validation images.
    # So you can compare the results of different models more intuitively.
    random.seed(16)
    val_indices=random.sample(range(0,len(val_input_names)),num_vals)

    # Do the training here
    for epoch in range(0, args.num_epochs):

        current_losses = []

        cnt=0

        # Equivalent to shuffling
        id_list = np.random.permutation(len(train_input_names))

        num_iters = int(np.floor(len(id_list) / args.batch_size))
        st = time.time()
        epoch_st=time.time()
        for i in range(num_iters):
            # st=time.time()

            input_image_batch = []
            output_image_batch = []
            #wave1 = []
            #wave2 = []
            #wave3 = []
            #wave4 = []

            # Collect a batch of images
            for j in range(args.batch_size):
                index = i*args.batch_size + j
                id = id_list[index]
                input_image = load_image(train_input_names[id])[:args.crop_height, :args.crop_width]
                #input_image[:, :, 0] -= 103.939
                #input_image[:, :, 1] -= 116.779
                #input_image[:, :, 2] -= 123.68

                output_image = load_image(train_output_names[id])[:args.crop_height, :args.crop_width]
                #input_img = np.float32(input_image)
                #input_image_gray = load_image_gray(train_input_names[id])[:args.crop_height, :args.crop_width]

                with tf.device('/cpu:0'):

                    input_image, output_image = data_augmentation(input_image, output_image)
                    # Prep the data. Make sure the labels are in one-hot format
                    input_image = np.float32(input_image) / 255.0
                    #output_image = np.reshape(output_image, (3, 224, 224))
                    output_image = np.float32(helpers.one_hot_it(label=output_image, label_values=label_values))

                input_image_batch.append(np.expand_dims(input_image, axis=0))
                output_image_batch.append(np.expand_dims(output_image, axis=0))

            # ***** THIS CAUSES A MEMORY LEAK AS NEW TENSORS KEEP GETTING CREATED *****
            # input_image = tf.image.crop_to_bounding_box(input_image, offset_height=0, offset_width=0,
            #                                               target_height=args.crop_height, target_width=args.crop_width).eval(session=sess)
            # output_image = tf.image.crop_to_bounding_box(output_image, offset_height=0, offset_width=0,
            #                                               target_height=args.crop_height, target_width=args.crop_width).eval(session=sess)
            # ***** THIS CAUSES A MEMORY LEAK AS NEW TENSORS KEEP GETTING CREATED *****

            # memory()

            if args.batch_size == 1:
                input_image_batch = input_image_batch[0]
                output_image_batch = output_image_batch[0]
            else:
                input_image_batch = np.squeeze(np.stack(input_image_batch, axis=1))
                output_image_batch = np.squeeze(np.stack(output_image_batch, axis=1))

            # Do the training
            _,current=sess.run([opt,loss],feed_dict={input:input_image_batch,output:output_image_batch, keep_prob:0.5})#, wavelet1:wave1, wavelet2:wave2, wavelet3: wave3, wavelet4: wave4, keep_prob:0.25 })
            current_losses.append(current)
            cnt = cnt + args.batch_size
            if cnt % 20 == 0:
                string_print = "Epoch = %d Count = %d Current_Loss = %.4f Time = %.2f"%(epoch,cnt,current,time.time()-st)
                utils.LOG(string_print)
                st = time.time()

        mean_loss = np.mean(current_losses)
        avg_loss_per_epoch.append(mean_loss)

        # Create directories if needed
        if not os.path.isdir("%s/%04d"%(check,epoch)):
            os.makedirs("%s/%04d"%(check,epoch))

        saver.save(sess,model_checkpoint_name)

        if val_indices != 0 and epoch % 50 == 0:
            saver.save(sess,"%s/%04d/model.ckpt"%(check,epoch))

        target=open("%s/%04d/val_scores.csv"%(check,epoch),'w')
        target.write("val_name, avg_accuracy, precision, recall, f1 score, mean iou \n")
        target.write(class_names_string)

        scores_list = []
        class_scores_list = []
        precision_list = []
        recall_list = []
        f1_list = []
        iou_list = []

        # Do the validation on a small set of validation images
        for ind in val_indices:
            input_image = np.float32(load_image(val_input_names[ind])[:args.crop_height, :args.crop_width])
            #input_image[:, :, 0] -= 103.939
            #input_image[:, :, 1] -= 116.779
            #input_image[:, :, 2] -= 123.68

            input_image = np.expand_dims(input_image, axis=0)/255.0
            gt = load_image(val_output_names[ind])[:args.crop_height, :args.crop_width]
            gt = helpers.reverse_one_hot(helpers.one_hot_it(gt, label_values))
            #img_ar = np.array(gt)
            file_name = utils.filepath_to_name(val_input_names[ind])
            input_l = []
        for _ in range(len(gpus)):
            input_l.append(input_image)
            input_l = np.squeeze(np.stack(input_l, axis=1))
        if len(gpus) == 1:
            input_l = np.expand_dims(input_l, axis=0)
        if model != 'ssunet':
            output_image = sess.run(network,feed_dict={input:input_l, keep_prob:1.0})#, wavelet1:wav1, wavelet2:wav2, wavelet3: wav3, wavelet4: wav4, keep_prob:1.0})
        else:
            output_image, aux_out = sess.run(network,feed_dict={input:input_image, keep_prob:1.0})#, wavelet1:wav1, wavelet2:wav2, wavelet3: wav3, wavelet4: wav4, keep_prob:1.0})
            output_image = np.array(aux_out[0,:,:,:])
            output_image = helpers.reverse_one_hot(output_image)
            out_vis_image = helpers.colour_code_segmentation(output_image, label_values)
            cv2.imwrite("%s/%04d/%s_aux_pred.png"%(check,epoch, file_name),cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR))

            output_image = np.array(output_image[0,:,:,:])
            output_image = helpers.reverse_one_hot(output_image)
            out_vis_image = helpers.colour_code_segmentation(output_image, label_values)

            accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation(pred=output_image, label=gt, num_classes=num_classes)

            target.write("\n %s, %f, %f, %f, %f, %f \n"%(file_name, accuracy, prec, rec, f1, iou))
            for item in class_accuracies:
                target.write(", %f"%(item))
            target.write("\n")

            scores_list.append(accuracy)
            class_scores_list.append(class_accuracies)
            precision_list.append(prec)
            recall_list.append(rec)
            f1_list.append(f1)
            iou_list.append(iou)

            gt = helpers.colour_code_segmentation(gt, label_values)

            im = cv2.imread(val_input_names[ind], -1)
            file_name = os.path.basename(val_input_names[ind])
            file_name = os.path.splitext(file_name)[0]
            cv2.imwrite("%s/%04d/%s_pred.png"%(check,epoch, file_name),cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR))
            cv2.imwrite("%s/%04d/%s_gt.png"%(check,epoch, file_name),cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR))
            cv2.imwrite("%s/%04d/%s_img.png"%(check,epoch, file_name),im)

        avg_score = np.mean(scores_list)
        class_avg_scores = np.mean(class_scores_list, axis=0)
        avg_scores_per_epoch.append(avg_score)
        avg_precision = np.mean(precision_list)
        avg_recall = np.mean(recall_list)
        avg_f1 = np.mean(f1_list)
        avg_iou = np.mean(iou_list)

        target.write("\n\n")
        target.write("%s, %s, %s, %s, %s"%('avg_score', 'avg_precision', 'avg_recall', 'avg_f1', 'avg_iou'))
        target.write("%f, %f, %f, %f, %f"%(avg_score, avg_precision, avg_recall, avg_f1, avg_iou))
        target.write("\n\n")
        for index, item in enumerate(class_avg_scores):
            target.write("%s = %f" % (class_names_list[index], item))
            target.write("\n")
        #target.write("\n")
        #target.write(" %04d = %f"% (epoch, avg_score))
        #target.write(" %04d:"% (epoch))
        #target.write("Validation precision = ", avg_precision)
        #target.write("Validation recall = ", avg_recall)
        #target.write("Validation F1 score = ", avg_f1)
        #target.write("Validation IoU score = ", avg_iou)

        target.close()

        print("\nAverage validation accuracy for epoch # %04d = %f"% (epoch, avg_score))
        print("Model name %s"%args.model)
        print("Average per class validation accuracies for epoch # %04d:"% (epoch))
        for index, item in enumerate(class_avg_scores):
            print("%s = %f" % (class_names_list[index], item))
        print("Validation precision = ", avg_precision)
        print("Validation recall = ", avg_recall)
        print("Validation F1 score = ", avg_f1)
        print("Validation IoU score = ", avg_iou)

        epoch_time=time.time()-epoch_st
        remain_time=epoch_time*(args.num_epochs-1-epoch)
        m, s = divmod(remain_time, 60)
        h, m = divmod(m, 60)
        if s!=0:
            train_time="Remaining training time = %d hours %d minutes %d seconds\n"%(h,m,s)
        else:
            train_time="Remaining training time : Training completed.\n"
        utils.LOG(train_time)
        scores_list = []

    fig = plt.figure(figsize=(11,8))
    ax1 = fig.add_subplot(111)

    ax1.plot(range(args.num_epochs), avg_scores_per_epoch)
    ax1.set_title("Average validation accuracy vs epochs")
    ax1.set_xlabel("Epoch")
    ax1.set_ylabel("Avg. val. accuracy")

    plt.savefig('accuracy_vs_epochs.png')

    plt.clf()

    ax1 = fig.add_subplot(111)

    ax1.plot(range(args.num_epochs), avg_loss_per_epoch)
    ax1.set_title("Average loss vs epochs")
    ax1.set_xlabel("Epoch")
    ax1.set_ylabel("Current loss")

    plt.savefig('loss_vs_epochs.png')

elif args.mode == "test":
    print("\n***** Begin testing *****")
    print("Dataset -->", args.dataset)
    print("Model -->", args.model)
    print("Crop Height -->", args.crop_height)
    print("Crop Width -->", args.crop_width)
    print("Num Classes -->", num_classes)
    print("")

    # Create directories if needed
    if not os.path.isdir("%s"%("Val")):
            os.makedirs("%s"%("Val"))

    target=open("%s/val_scores.csv"%("Val"),'w')
    target.write("val_name, avg_accuracy, precision, recall, f1 score, mean iou %s\n" % (class_names_string))
    scores_list = []
    class_scores_list = []
    precision_list = []
    recall_list = []
    f1_list = []
    iou_list = []
    run_times_list = []

    # Run testing on ALL test images
    for ind in range(len(val_input_names)):
        sys.stdout.write("\rRunning test image %d / %d"%(ind+1, len(val_input_names)))
        sys.stdout.flush()

        input_image = np.expand_dims(np.float32(load_image(val_input_names[ind])[:args.crop_height, :args.crop_width]),axis=0)/255.0
        gt = load_image(val_output_names[ind])[:args.crop_height, :args.crop_width]
        gt = helpers.reverse_one_hot(helpers.one_hot_it(gt, label_values))

        st = time.time()
        output_image = sess.run(network,feed_dict={input:input_image})

        run_times_list.append(time.time()-st)

        output_image = np.array(output_image[0,:,:,:])

        output_image = helpers.reverse_one_hot(output_image)
        #print(output_image)
        # exit(1)
        out_vis_image = helpers.colour_code_segmentation(output_image, label_values)

        #accuracy, class_accuracies, prec, rec, f1, iou = utils.evaluate_segmentation(pred=output_image, label=gt, num_classes=num_classes)

        file_name = utils.filepath_to_name(val_input_names[ind])
        #target.write("%s, %f, %f, %f, %f, %f"%(file_name, accuracy, prec, rec, f1, iou))
        #for item in class_accuracies:
            #target.write(", %f"%(item))
        #target.write("\n")

        #scores_list.append(accuracy)
        #class_scores_list.append(class_accuracies)
        #precision_list.append(prec)
        #recall_list.append(rec)
        #f1_list.append(f1)
        #iou_list.append(iou)

        gt = helpers.colour_code_segmentation(gt, label_values)
        im = cv2.imread(val_input_names[ind])
        cv2.imwrite("%s/%s_pred.png"%("Val", file_name),cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR))
        cv2.imwrite("%s/%s_img.png"%("Val", file_name),im)

        cv2.imwrite("%s/%s_gt.png"%("Val", file_name),cv2.cvtColor(np.uint8(gt), cv2.COLOR_RGB2BGR))

    #target.close()

    #avg_score = np.mean(scores_list)
    #class_avg_scores = np.mean(class_scores_list, axis=0)
    #avg_precision = np.mean(precision_list)
    #avg_recall = np.mean(recall_list)
    #avg_f1 = np.mean(f1_list)
    #avg_iou = np.mean(iou_list)
    #avg_time = np.mean(run_times_list)

    #target.write("%s, %f, %f, %f, %f, %f"%(file_name, accuracy, prec, rec, f1, iou))
    #target.close()

    #print("Average test accuracy = ", avg_score)
    #print("Average per class test accuracies = \n")
    #for index, item in enumerate(class_avg_scores):
        #print("%s = %f" % (class_names_list[index], item))
    #print("Average precision = ", avg_precision)
    #print("Average recall = ", avg_recall)
    #print("Average F1 score = ", avg_f1)
    #rint("Average mean IoU score = ", avg_iou)
    #print("Average run time = ", avg_time)

elif args.mode == "predict":

    if args.image is None:
        ValueError("You must pass an image path when using prediction mode.")

    print("\n***** Begin prediction *****")
    print("Dataset -->", args.dataset)
    print("Model -->", args.model)
    print("Crop Height -->", args.crop_height)
    print("Crop Width -->", args.crop_width)
    print("Num Classes -->", num_classes)
    print("Image -->", args.image)
    print("")
    for ind in range(len(test_input_names)):
        sys.stdout.write("\rRunning prediction image %d / %d"%(ind+1, len(test_input_names)))
        sys.stdout.flush()

        input_image = np.expand_dims(np.float32(load_image(test_input_names[ind])[:args.crop_height, :args.crop_width]),axis=0)/255.0

        st = time.time()
        output_image = sess.run(network,feed_dict={input:input_image})

        run_time = time.time()-st

        output_image = np.array(output_image[0,:,:,:])
        output_image = helpers.reverse_one_hot(output_image)
        out_vis_image = helpers.colour_code_segmentation(output_image, class_dict)
        file_name = utils.filepath_to_name(test_input_names[ind])
        cv2.imwrite("%s/%s_pred.png"%("Test", file_name),cv2.cvtColor(np.uint8(out_vis_image), cv2.COLOR_RGB2BGR))

else:
    ValueError("Invalid mode selected.")


# -*- coding: utf-8 -*-
from __future__ import absolute_import
import json
from time import time, sleep
from os import remove, getpid
from os.path import basename
from socket import gethostname
from setproctitle import setproctitle

def process_run(name, index):
    eval("process_%s(%s)" % (name, index))

def _checkActive(mongoMq, procKey):
    hostname = gethostname()
    procData = mongoMq['process_list'].find_one({"hostname":hostname, "title": procKey},{"_id":0})
    if procData:
        mongoMq['process_list'].find_and_modify({"hostname":hostname, "title": procKey},{"$set":{"processid": getpid()}})
    else:
        return False

def process_spider(index):
    '''蜘蛛'''
    mqkey = 'spider'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq
    from business.crawler import crawl

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        row = Mq.consume('spider')
        if not row:
            sleep(10)
            continue

        crawl(row)
        Mq.finish(row['id'], 'spider')

def process_mirror(index):
    '''镜像的进程'''
    mqkey = 'mirror'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)
    from common import mongoMq
    from utils import mq as Mq
    import business.mirror as bMirror
    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        row = Mq.consume('mirror')
        if not row:
            sleep(10)
            continue
        bMirror.generate(row['id'])
        Mq.finish(row['id'], 'mirror')

def process_notify(index):
    '''蜘蛛'''
    mqkey = 'notify'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq
    import business.notify as bNotify

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        row = Mq.consume('notify')
        if not row:
            sleep(10)
            continue

        result = bNotify.send(row['id'])
        Mq.finish(row['id'], 'notify')

def process_piping(index):
    '''数据处理'''
    mqkey = 'piping'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq
    import business.piping as bPiping

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        row = Mq.consume('piping')
        if not row:
            sleep(10)
            continue
        bPiping.piping_all(row['id'])
        Mq.finish(row['id'], 'piping')

def process_snapshot(index):
    '''数据处理'''
    mqkey = 'snapshot'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq
    from utils.logger import logger
    import business.snapshot as bSnapshot

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        row = Mq.consume('snapshot')
        logger.debug("snapshot::::%s" % json.dumps(row))
        if not row:
            sleep(10)
            continue

        bSnapshot.generate(row)
        Mq.finish(row['id'], 'snapshot')

def process_darklink(index):
    '''数据处理'''
    mqkey = 'darklink'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        row = Mq.consume('darklink')
        if not row:
            sleep(10)
            continue

        bSnapshot.generate(row)
        Mq.finish(row['id'], 'darklink')

def process_qiniu(index):
    '''快照图片压缩及上传到七牛
    此程序是守护进程，可start/stop/restart
    程序完成两件事：
    1.查找新生成的快照，压缩，移动到七牛上传图片的文件夹
    2.由七牛的程序将文件上传到七牛的服务器
    '''

    mqkey = 'qiniu'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq
    from utils.logger import logger
    from utils.file import qiniu_upload
    from subprocess import Popen, PIPE, STDOUT
    from config.config import PATH_TMP_SNAPSHOT

    cmdFind = 'find %s -amin +1 -type f' % PATH_TMP_SNAPSHOT
    cmdCompress = 'pngquant -f --ext .png --quality 10 %s'
    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        child = Popen(cmdFind, shell=True, close_fds=True, bufsize=-1, stdout=PIPE, stderr=STDOUT)
        output = child.stdout.read().decode()
        output = output.strip()
        if not output: continue
        files = output.split("\n")
        for localfile in files:
            logger.info(cmdCompress % localfile)
            child = Popen(cmdCompress % localfile, shell=True, close_fds=True, bufsize=-1, stdout=PIPE, stderr=STDOUT)
            output = child.stdout.read().decode()
            key = basename(localfile)
            result = qiniu_upload(key, localfile)
            if result:
                logger.info(result)
                remove(localfile)
        sleep(5)

def process_ckready(index):
    '''
    将数据转入待处理状态
    '''
    mqkey = 'ckready'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        Mq.ready('spider')
        Mq.ready('mirror')
        Mq.ready('piping')
        Mq.ready('notify')
        Mq.ready('snapshot')
        sleep(10)

def process_ckfinish(index):
    '''
    检查数据是否已处理完成
    '''
    mqkey = 'ckfinish'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from utils import mq as Mq
    import business.task as bTask

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        bTask.execute_finish()
        sleep(10)

def process_ckcorrect(index):
    '''
    校正数据
    '''
    mqkey = 'ckcorrect'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from business.devops import mq_correct

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        mq_correct()
        sleep(10)

def process_initexec(index):
    '''校正数据'''
    mqkey = 'initexec'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import db, mongoMq
    import business.task as bTask

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        rows = db.fetchall("select id from task_execute where status=0 order by id asc");
        for row in rows: bTask.execute_init(row['id'])
        sleep(10)

def process_scheduler(index):
    '''校正数据'''
    mqkey = 'scheduler'
    procname = "spider-%s-%s" % (mqkey, index)
    setproctitle(procname)

    from common import mongoMq
    from aps import ApsCheckDb

    start = time()
    while True:
        #如果停止，退出程序
        if time() - start > 10:
            if not _checkActive(mongoMq, procname[7:]): exit()
            start = time()
        ApsCheckDb()
        sleep(10)


#!/usr/local/bin/python

from math import ceil

class Drone:
    """The Drone class is used for searching and walking through the map,
       mining minerals and returning them to the boss"""
    def __init__(self):
        # TicksLeft is used for returning the zerg back using the total ticks
        self.ticksLeft = 1000
        self.canDeploy = True
        # HitPoints is used to keep track of the zergs current HP
        self.hitPoints = 40
        self.currentMap = 0
        self.requestPickup = False
        self.returnHome = False
        self.currentSpace = ""
        # Minerals is used for telling the zerg to return home at a certain
        # Number of minerals
        self.minerals = 0
        # MoveList keeps track of the moves the zerg has made
        self.moveList = []
        # SpiralMoves is used to determine how many moves the spiral method
        # Needs to go before it changes
        self.spiralMoves = 2
        # StripMoves calculates the moves that zerg has taken going south
        self.stripMoves = 0
        # CurrentMoves is used in the spiral searching to check against how
        # Many times the zerg has to move
        self.currentMoves = 0
        # DirectionCheck is used in the spiral search to find what direction
        # To go in next
        self.directionCheck = 0
        self.spiral = True
        self.strip = False
        # StartingHp is used to determine if the zerg has lost a certain
        # Amount of health in the process of searching
        self.startingHp = 40
        self.landingZone = []
        # BackTrack is used for returning back using the moveList
        self.backTrack = {"W": "EAST", "E": "WEST", "N": "SOUTH", "S": "NORTH"}
        self.directionMove = ["EAST", "SOUTH", "WEST", "NORTH"]

    @property
    def zergId(self):
        """The zerg's ID"""
        return self._zergId

    @zergId.setter
    def zergId(self, currId):
        self._zergId = currId

    def spiralSearch(self, context, contexts):
        """The spiralSearch method is used for going through the map in a
           spiral method"""
        # Compare if we have completed all of the moves in that direction
        if self.currentMoves == self.spiralMoves:
            self.directionCheck += 1
            # Check if we're at the pivot point of the spiral search
            if str(self.directionCheck % 4) in "02":
                self.spiralMoves += 2
            self.currentMoves = 0
            # If we hit something while spiraling we break out
            if contexts[self.directionCheck % 4] in "#Z*":
                self.spiral = False
                self.currentMoves = 0
                return 'CENTER'
            self.currentSpace = contexts[self.directionCheck % 4]
            self.currentMoves += 1
            # Append the first letter of the direction so our zerg knows how to
            # Return home when they're done
            self.moveList.append(self.directionMove
                                 [self.directionCheck % 4][0])
            return self.directionMove[self.directionCheck % 4]
        else:
            self.currentMoves += 1
            if contexts[self.directionCheck % 4] in "#Z*":
                self.spiral = False
                self.currentMoves = 0
                return 'CENTER'
            # Append the first letter of the direction so our zerg knows how to
            # Return home when they're done
            self.moveList.append(self.directionMove
                                 [self.directionCheck % 4][0])
            self.currentSpace = contexts[self.directionCheck % 4]
            return self.directionMove[self.directionCheck % 4]

    def stripSearch(self, context, contexts):
        """The stripSearch method is used for going through the map from the
           top left corner of the map and searching line by line of the map"""
        # Check if we hit the bottom left corner of the map so we stop
        # Strip searching
        if contexts[1] == "#" and contexts[2] == "#":
            self.strip = False
        # Search for the best path to take and return that direction
        elif str(self.currentMoves % 4) in "02":
            if contexts[self.currentMoves % 4] in "#Z":
                self.currentMoves += 1
            elif self.currentMoves % 4 == 0:
                if contexts[0] == "*":
                    return 'EAST'
                self.currentSpace = contexts[0]
                self.moveList.append("E")
                return 'EAST'
            else:
                if contexts[2] == "*":
                    return 'WEST'
                self.moveList.append("W")
                self.currentSpace = contexts[2]
                return 'WEST'
        elif str(self.currentMoves % 4) in "13":
            if self.stripMoves == 2 or contexts[self.currentMoves % 4] in "#Z":
                if contexts[0] == "*":
                    return 'EAST'
                self.currentMoves += 1
                self.stripMoves = 0
                self.moveList.append("E")
                self.currentSpace = contexts[0]
                return 'EAST'
            else:
                if contexts[1] == "*":
                    return 'SOUTH'
                self.stripMoves += 1
                self.moveList.append("S")
                self.currentSpace = contexts[1]
                return 'SOUTH'

    def returnZerg(self, contexts):
        """The returnZerg method is used for rerouting the zerg back using the
           path they came from"""
        try:
            # Step through our move list and reassign the current space
            move = self.moveList.pop()
            if move == "W":
                self.currentSpace = contexts[0]
            elif move == "N":
                self.currentSpace = contexts[1]
            elif move == "E":
                self.currentSpace = contexts[2]
            elif move == "S":
                self.currentSpace = contexts[3]
            return self.backTrack.get(move)
        except IndexError:
            return 'CENTER'

    def move(self, context):
        """The move method is used for analyzing the best move of the drone and
           moving them that way"""
        # Set the context dictionary every time with the new map context
        contexts = {0: context.east, 1: context.south, 2: context.west,
                    3: context.north}
        # Set our LZ if we just landed
        if self.currentSpace == "":
            self.landingZone = [context.x, context.y]
            self.currentSpace = "_"
            self.startingHp = self.hitPoints
        # Decrement our internal health counter if we're starting on acid
        if self.currentSpace == "~":
            self.hitPoints -= 3
        # Reset strip maneuver
        if self.returnHome:
            self.strip = False
        # Check if we either have too little health to continue, if we have
        # More than 25 minerals, or if ticks start to dwindle down
        if any((self.hitPoints < self.startingHp/2 + 4,
               self.minerals > 25,
               len(self.moveList) + 5 >= self.ticksLeft)):
            self.returnHome = True

        gps = [context.x, context.y]
        # Below checks if either the zerg is on the LZ and trying to return
        # Home, or if they're at the LZ and have more than a certain amount of
        # Minerals.
        if any(((self.landingZone == gps and self.returnHome),
               (self.landingZone == gps and self.minerals > 15))):
            self.moveList = []
            self.requestPickup = True
            return 'CENTER'

        if self.returnHome:
            result = self.returnZerg(contexts)
            return result

        # Mine if not currently on an acid space
        if self.currentSpace != "~":
            if context.east == "*":
                self.minerals += 1
                return 'EAST'
            elif context.south == "*":
                self.minerals += 1
                return 'SOUTH'
            elif context.west == "*":
                self.minerals += 1
                return 'WEST'
            elif context.north == "*":
                self.minerals += 1
                return 'NORTH'

        # Spiral search if we're not going home
        if self.spiral and not self.returnHome:
            result = self.spiralSearch(context, contexts)
            return result
        else:
            # Check if we're strip searching. If not, get to the point of
            # Beginning the strip search
            if self.strip:
                result = self.stripSearch(context, contexts)
                return result
            else:
                # Check if we're at the upper left most corner
                if contexts[2] in "#" and contexts[3] in "#":
                    self.strip = True
                else:
                    if contexts[2] in " _":
                        self.moveList.append("W")
                        self.currentSpace = contexts[2]
                        return 'WEST'
                    elif contexts[3] in " _~":
                        self.moveList.append("N")
                        self.currentSpace = contexts[3]
                        return 'NORTH'
                    elif contexts[1] in " _":
                        self.moveList.append("S")
                        self.currentSpace = contexts[1]
                        return 'SOUTH'
                    elif contexts[0] in " _" and contexts[2] not in "~":
                        self.moveList.append("E")
                        self.currentSpace = contexts[0]
                        return 'EAST'
                    else:
                        self.strip = True
                        return 'EAST'

        return 'CENTER'

class Overlord:
    """The Overlord class is used to initialize and control drones for mining
       purposes"""
    def __init__(self, ticks):
        # Create the dictionary for maps and drones
        self.maps = {}
        self.zerg = {}
        self.startingTicks = ticks
        self.ticks = ticks
        self.mapCounter = -1
        self.deployed = []
        # Adds the specific drones to their dictionary using ID as the key
        for _ in range(6):
            currZerg = Drone()
            self.zerg[id(currZerg)] = currZerg
            currZerg.zergId = id(currZerg)

    def add_map(self, map_id, summary):
        """The add_map method adds map objects to the Overlord"""
        self.maps[map_id] = summary

    def action(self):
        """The action method is used to decide what they Overlord will do for
           each specific tick of searching"""
        self.ticks -= 1
        # Check if the ticks are less than 20 percent of their starting value.
        # Tell all zergs to return home if this is the case
        if self.ticks < ceil(self.startingTicks * .20):
            for zerg in self.deployed:
                zerg.returnHome = True
        # Update each zergs tick left calculation
        for zerg in self.zerg.values():
            zerg.ticksLeft = self.ticks
        # Check if any zergs are on the LZ and requesting pickup.
        for zerg in self.deployed:
            if zerg.requestPickup:
                self.mapCounter = zerg.currentMap - 1
                zerg.currentMap = 0
                self.deployed.remove(zerg)
                # If the zerg is below a certain health, don't redeploy them
                if zerg.hitPoints < 10:
                    zerg.canDeploy = False
                return 'RETURN {}'.format(zerg.zergId)
        for zerg in self.zerg.values():
            # Don't deploy extra people if there are already enough deployed
            if self.mapCounter >= 2 or len(self.deployed) == 3:
                break
            if zerg in self.deployed:
                if zerg.hitPoints <= 0:
                    zerg.canDeploy = False
                    self.mapCounter = zerg.currentMap - 1
                    self.deployed.remove(zerg)
                continue
            if zerg.canDeploy:
                self.mapCounter += 1
                zerg.currentMap = self.mapCounter
                self.resetZerg(zerg)
                self.deployed.append(zerg)
                return 'DEPLOY {} {}'.format(zerg.zergId, self.mapCounter)
        return 'WAIT'

    def resetZerg(self, zerg):
        """The resetZerg method is used for resetting the data of a zerg so
           they can continue exploring new maps"""
        zerg.requestPickup = False
        zerg.returnHome = False
        zerg.currentSpace = ""
        zerg.minerals = 0
        zerg.moveList = []
        zerg.spiralMoves = 2
        zerg.stripMoves = 0
        zerg.currentMoves = 0
        zerg.directionCheck = 0
        zerg.spiral = True
        zerg.strip = False


import os

from flask_restplus import Api
from flask import Blueprint

from app.main.controller.auth_controller import api as auth_ns
from app.main.controller.task_controller import api as task_ns
from app.main.controller.user_controller import api as user_ns

from app.main import create_app

blueprint = Blueprint('api', __name__)

authorizations = {
    'apikey': {
        'type': 'apiKey',
        'in': 'header',
        'name': 'X-API-KEY'
    }
}

api = Api(
    blueprint,
    title="Task Storage API",
    version='0.1',
    description="A API for storage of tasks",
    authorizations=authorizations
)

api.add_namespace(auth_ns)
api.add_namespace(task_ns)
api.add_namespace(user_ns)

if __name__ == '__main__':
    app = create_app(os.getenv('RUN_ENV') or 'dev')
    app.register_blueprint(blueprint)
    app.app_context().push()
    app.run()


#simple program to do breadth first / depth first search

#   a
#b      c
#   d
#   e
#f      g
#   h

#BFS

#a  b  c  d  e  f  g  h

#DFS

#a  b  d  e  f  h  g  c

def simpelBFS(graph, start, path=[]):

    q = [start]

    while q:
        v = q.pop(0)
        if not v in path:
            path = path + v
            q = q + graph[v]

    return path

def simpleDFS(graph, start, path=[]):

    path = path + [start]

    for node in graph[start]:
        if not node in path:
            path = simpleDFS(graph, node, path)

    return path


# This script uses the result of support_analyzer.py
# This is meant to visualize and report important info about experiments on suppoert sets
# The output will be in $ANALYSES_DIR$

__author__ = "Elaheh"
__date__ = "$Jan 31, 2016 10:37:55 AM$"

import os, sys, shelve
import matplotlib.pyplot as plt
import numpy as np

# should be 405 for the whole experiment
NUM_OF_MODELS = 405
ANALYSES_DIR = 'support_analyses'

if not os.path.exists(ANALYSES_DIR):
    print(ANALYSES_DIR + " doesn't exist. first, run support_analyzer.py")
    sys.exit(-1)

#
# Load data
#

reader = shelve.open(os.path.join(ANALYSES_DIR, 'support_analyses'))

mean = reader ['mean']
min_val = reader ['min']
max_val = reader ['max']
stdev = reader ['pstdev']
mean_points = reader ['all_mean']
min_points = reader ['all_min']
max_points = reader ['all_max']
stdev_points = reader ['all_pstdev']

# we can load more statistical results here if we want
reader.close()

#
# Report results into 'support_analyses.txt'
#

writer = open(os.path.join(ANALYSES_DIR, 'support_analyses.txt'), 'a')
writer.write('-------------------------------------------------------------------------------------------------')
writer.write('\noverall minimum of Jaccard distances among all models is: ' + str(min_val))
writer.write('\noverall maximum of Jaccard distances among all models is: ' + str(max_val))
writer.write('\noverall population standard deviation for Jaccard distances among all models is: ' + str(stdev))
writer.write('\noverall average of Jaccard distances is: ' + str(mean))
writer.write('\n-------------------------------------------------------------------------------------------------')
writer.write('\nfor '+ str(NUM_OF_MODELS) + ' models, the following shows pairwise Jaccard distances min/max/mean/stdev')
writer.write('\neach element in the following lists is related to a model in the benchmarks.\n')
writer.write('\n\nminimum of Jaccard distances for models:\n' + str([round(float(i), 2) for i in min_points]))
writer.write('\n\nmaximum of Jaccard distances for models:\n' + str([round(float(i), 2) for i in max_points]))
writer.write('\n\npopulation standard deviation of Jaccard distances for models:\n' + str([round(float(i), 2) for i in stdev_points]))
writer.write('\n\naverage of Jaccard distances is:\n' + str([round(float(i), 2) for i in mean_points]))
# we can add more statistical results here later if we want
writer.close()

#
# Visualize the results
#

# Build a list for x-axis
x_axis = []
for i in range(NUM_OF_MODELS-10):
    x_axis.append(i)

fig = plt.figure()
ax = plt.subplot(111)

plt.plot(x_axis, min_points, label='min')
plt.plot(x_axis, max_points, label='max')
plt.plot(x_axis, mean_points, label='mean')
plt.plot(x_axis, stdev_points, label='standard deviation')
plt.xlabel('LUS models')
plt.ylabel('Jaccard distance')
plt.title('Pairwise Jaccard distance between Support sets')
box = ax.get_position()
ax.set_position([box.x0, box.y0 + box.height * 0.1, box.width, box.height * 0.9])
ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=5)
plt.show()
fig.savefig(os.path.join(ANALYSES_DIR, 'support_analyses.png'))
plt.close(fig)


import cv2
import time

print('cv2 version:', cv2.__version__)
dispW = 1920
dispH = 1080
flip=2
frameRate = 0

class Rechteck():
    def __init__(self,xCenter0=100,yCenter0=100, radius=50, color = [255,0,255], lineWidth = -1, dt = 1,dispW=1920,dispH=1080):
        self.xCenter0 = xCenter0
        self.yCenter0 = yCenter0
        self.xCenter = xCenter0
        self.yCenter = yCenter0
        self.radius = radius
        self.color = color
        self.lineWidth = lineWidth
        self.dt = dt
        self.time = 0-dt
        self.vx = 20 #random.randint(-1,1)
        self.vy = 20 #random.randint(-1,1)
        self.dispW = dispW
        self.dispH = dispH

    def Move(self):
       # self.time += self.dt
        self.xCenter = int(self.xCenter + self.dt * self.vx)
        self.yCenter = int(self.yCenter + self.dt * self.vy)

        if self.xCenter+self.radius >= int(self.dispW):
            self.vx = -self.vx
            self.xCenter = int(self.dispW) -self.radius

        if self.xCenter <= 0:
            self.vx = -self.vx
            self.xCenter = 0

        if self.yCenter+self.radius >= int(self.dispH):
            self.vy = -self.vy
            self.yCenter = int(self.dispH) - self.radius

        if self.yCenter <=0:
            self.vy = -self.vy
            self.yCenter = 0

'''
#for rasbery pi camera
camSet='nvarguscamerasrc !  video/x-raw(memory:NVMM), \
        width=3264, height=2464, format=NV12, framerate=21/1 ! \
        nvvidconv flip-method='+str(flip)+' ! \
        video/x-raw, width='+str(dispW)+', \
        height='+str(dispH)+', format=BGRx !\
        videoconvert ! video/x-raw, format=BGR ! \
        appsink'
'''
logoBox = Rechteck(dispH=dispH,dispW=dispW)
logo = cv2.imread('pl.jpg')
logo = cv2.resize(logo, (50,50))
cv2.imshow('logo',logo)
cv2.moveWindow('logo',720,0)

logoGray = cv2.cvtColor(logo, cv2.COLOR_BGR2GRAY)
_,logoMask = cv2.threshold(logoGray,225,255,cv2.THRESH_BINARY)

cv2.imshow('logoMask',logoMask)
cv2.moveWindow('logoMask', 780,0)

FGMask = cv2.bitwise_not(logoMask )
cv2.imshow('FG Mask', FGMask)
cv2.moveWindow('FG Mask', 780,120)

FG = cv2.bitwise_and(logo,logo,mask=FGMask)
cv2.imshow('FG ', FG)
cv2.moveWindow('FG ', 780,240)

camNumber = 1
#cam=cv2.VideoCapture(camSet)
cam = cv2.VideoCapture(camNumber)
start = time.time()
while True:
    ret,frame = cam.read()
    print(frame.shape)

    roiBox = frame[logoBox.yCenter:logoBox.yCenter+logoBox.radius,\
            logoBox.xCenter:logoBox.xCenter+logoBox.radius].copy()
    cv2.imshow('roiBox',roiBox)
    cv2.moveWindow('roiBox',840,0)

    BG = cv2.bitwise_and(roiBox,roiBox,mask = logoMask)
    cv2.imshow('BG ',BG )
    cv2.moveWindow('BG ',900,0)

    #roiComb = cv2.addWeighted(FG,0.5,BG,0.5,0)
    roiComb = cv2.add(FG,BG)
    cv2.imshow('roiComb ',roiComb  )
    cv2.moveWindow('roiComb ',960,0)

    frame[logoBox.yCenter:logoBox.yCenter+logoBox.radius,\
            logoBox.xCenter:logoBox.xCenter+logoBox.radius] = roiComb

    cv2.imshow('FLIRcamera',frame)
    cv2.moveWindow('FLIRcamera',0,0)
    frameRate += 1
    logoBox.Move()
    if time.time() - start > 1:
        print('frame per secon: ', frameRate)
        frameRate = 0
        start = time.time()
    if cv2.waitKey(1)==ord('q'):
        break

cam.release()
cv2.destroyAllWindows()


import subprocess
import re
from Multi_Layer_Comparison import multiLayerComparison, indentComparison, varAndOperCount, functionSignatureComp, exe_comp, ksc, ASTmatch, BOWComparison

functions = ["multiLayerComparison","BOWComparison","indentComparison","varAndOperCount","functionSignatureComp","keywordSeqCom","exe_comp", "ASTmatch"]
n_funcs = len(functions)
start = 0
end = start+8

def func_num(x, f1, f2):
    if x==0: return multiLayerComparison(f1,f2,1)
    elif x==1: return BOWComparison(f1,f2)
    elif x==2: return indentComparison(f1,f2)
    elif x==3: return varAndOperCount(f1,f2)
    elif x==4: return functionSignatureComp(f1,f2)
    elif x==5: return ksc(f1,f2)
    elif x==6: return exe_comp(f1,f2)
    elif x==7: return ASTmatch(f1,f2)

with open('truth2.txt', 'r') as f:
	plag_files_str = f.read()
#print("%r"%plag_files_str)
plag_files = plag_files_str.split('- ')
plag_files.pop(0)
for i in range(len(plag_files)):
    plag_files[i] = plag_files[i].split('\n')
    plag_files[i].pop()
    try:
        t = [plag_files[i][0]]
    except:
        print(plag_files)
        assert Error
    for x in plag_files[i][1:]:
        t.append(x.split(','))
    plag_files[i] = t

c_pos,c_neg, f_pos, f_neg, error, avg_plag, avg_nplag = [0 for i in range(n_funcs)], [0 for i in range(n_funcs)], [0 for i in range(n_funcs)], [0 for i in range(n_funcs)], [0 for i in range(n_funcs)], [0 for i in range(n_funcs)], [0 for i in range(n_funcs)]

thresholds = [None, 65, 65, 65, 75, 85, 85, 45]
data_folder = 'dataset3'

for assignment in plag_files:

    folder = assignment[0]
    #print(folder)

    process_list_files = subprocess.run(f"ls {data_folder}/{folder}", shell=True, executable='/bin/bash', stdout = subprocess.PIPE, stderr = subprocess.PIPE)
    print(process_list_files.stderr)

    test_files_str = process_list_files.stdout.decode('ascii')
    #print(test_files_str)
    test_files = test_files_str.split('\n')
    test_files.pop()
    n_files = len(test_files)

    for i in range(n_files):
        for j in range(i+1, n_files):
            #print(i,j)
            x = test_files[i]
            y = test_files[j]
            #print(x,y)
            # Be VERY careful about format of file to be sent, extensions, address, and all.
            plagiarised = [None for i in range(n_funcs)]
            for k in assignment[1:]:
            #print(x,y,k,sep = '\n')
                if (x[:-4] in k) and (y[:-4] in k):
                    actually_plagiarised = True
                    break
                else:
                    actually_plagiarised = False

            for v in range(start, end):
                try:
                    if v==0 or v==6:
                        value = func_num(v, data_folder+'/'+folder+'/'+x ,data_folder+'/'+folder+'/'+y)
                    else:
                        with open(data_folder+'/'+folder+'/'+x, 'r') as f1, open(data_folder+'/'+folder+'/'+y, 'r') as f2:
                            value = func_num(v, f1, f2)
                    #print('RESULT:',value,x,y)

                    if v>0:
                        if value>thresholds[v]:
                            plagiarised[v] = True
                        else: plagiarised[v] = False
                    else:
                        plagiarised[v] = value[0]
                    #print(plagiarised[v])

                    if actually_plagiarised:
                        if v!=0: avg_plag[v] += value
                        if plagiarised[v] == actually_plagiarised:
                            c_pos[v] += 1
                        else:
                            f_neg[v] += 1
                    else:
                        if v!=0: avg_nplag[v] += value
                        if plagiarised[v] == actually_plagiarised:
                            c_neg[v] += 1
                        else:
                            f_pos[v] += 1

                except ZeroDivisionError:
                    print(functions[v],folder, x, y, '/0')
                    error[v] += 1
                '''
                except:
                    print(functions[v],folder, x, y, 'ERROR\n\n\n')
                    error[v]+=1'''
print('\n')
perc, perc_plag, perc_nplag = [0 for i in range(n_funcs)], [0 for i in range(n_funcs)], [0 for i in range(n_funcs)]

for i in range(start, end):
    try:
        perc[i] = 100*(c_pos[i] + c_neg[i])/(c_pos[i] + c_neg[i]+f_pos[i]+f_neg[i])
        perc_plag[i] = 100*c_pos[i]/(c_pos[i]+f_neg[i])
        perc_nplag[i] = 100*c_neg[i]/(c_neg[i]+f_pos[i])
        print(f"{functions[i]}: {perc_plag[i]}% of plag pairs and {perc_nplag[i]}% of non-plag pairs correctly identified. Tested {c_pos[i] + c_neg[i]+f_pos[i]+f_neg[i]} pairs. {error[i]} pairs gave an error.")
        if i!=0:
            print(f"On plag pairs, avg perc match was {avg_plag[i]/(c_pos[i] + f_neg[i])}%, and on non-plag pairs, it was {avg_nplag[i]/(c_neg[i]+f_pos[i])}%.")
        print("\n")
    except ZeroDivisionError:
        print(c_pos[i],c_neg[i],f_pos[i],f_neg[i],error[i])
    #print(c_pos[i],c_neg[i],f_pos[i],f_neg[i],error[i])


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""r2pipe

This module provides an API to interact with the radare2
commandline interface from Python using a pipe.

The pipe can be connected to the parent process to run
Python scripts from the radare2 shell itself, or it can
spawn a new process, connect via HTTP to a remote r2 http
server, etc.

Some r2 commands display the information in JSON, that's
why r2pipe provides `-j` methods to directly parse it
and return a native Python object.

Example:
  $ python
  > import r2pipe
  > r = r2pipe.open("/bin/ls")
  > print(r.cmd("pd 10"))
  > print(r.cmdj("aoj")[0]['size'])
  > r.quit()
"""

import os
import re
import sys
import time
import json
import fcntl
import socket
import urllib
from io import TextIOWrapper
from subprocess import Popen, PIPE
try:
        import r2lang
except:
        r2lang = None

try:
        from .native import RCore
        has_native = True
except:
        has_native = False

VERSION = "0.9.3"

if sys.version_info >= (3, 0):
        import urllib.request
        urlopen = urllib.request.urlopen
        import urllib.error
        URLError = urllib.error.URLError
else:
        import urllib2
        urlopen = urllib2.urlopen
        URLError = urllib2.URLError
if os.name == "nt":
        from ctypes import byref, c_ulong, create_string_buffer, windll
        GENERIC_READ = 0x80000000
        GENERIC_WRITE = 0x40000000
        OPEN_EXISTING = 0x3
        INVALID_HANDLE_VALUE = -1
        PIPE_READMODE_MESSAGE = 0x2
        ERROR_PIPE_BUSY = 231
        ERROR_MORE_DATA = 234
        BUFSIZE = 4096
        szPipename = "\\\\.\\pipe\\"
        chBuf = create_string_buffer(BUFSIZE)
        cbRead = c_ulong(0)
        cbWritten = c_ulong(0)

def version():
        """Return string with the version of the r2pipe library
        """
        return VERSION

def in_rlang():
        return r2lang is not None and r2lang.cmd is not None

class open:
        """Class representing an r2pipe connection with a running radare2 instance
        """
        def __init__(self, filename='', flags=[]):
                """Open a new r2 pipe
                The 'filename' can be one of the following:

                * absolute or relative path to file
                * http://<host>:<port>/cmd to connect to an r2 webserver
                * tcp://<host>:<port> to connect to an r2 tcp server
                * #!pipe when launching it from r2 via RLang.pipe

                Args:
                        filename (str): path to filename or uri
                        flags (list of str): arguments, either in comapct form
                                ("-wdn") or sepparated by commas ("-w","-d","-n")
                Returns:
                        Returns an object with methods to interact with r2 via commands
                """
                if not filename and in_rlang():
                        self._cmd = self._cmd_rlang
                        return
                try:
                        if os.name == "nt":
                                mypipename = os.environ['r2pipe_path']
                                while 1:
                                        hPipe = windll.kernel32.CreateFileA(szPipename + mypipename, GENERIC_READ | GENERIC_WRITE, 0, None, OPEN_EXISTING, 0, None)
                                        if (hPipe != INVALID_HANDLE_VALUE):
                                                break
                                        else:
                                                print("Invalid Handle Value")
                                        if (windll.kernel32.GetLastError() != ERROR_PIPE_BUSY):
                                                print("Could not open pipe")
                                                return
                                        elif ((windll.kernel32.WaitNamedPipeA(szPipename, 20000)) == 0):
                                                print("Could not open pipe\n")
                                                return
                                windll.kernel32.WriteFile(hPipe, "e scr.color=false\n", 18, byref(cbWritten), None)
                                windll.kernel32.ReadFile(hPipe, chBuf, BUFSIZE, byref(cbRead), None)
                                self.pipe = [hPipe, hPipe]
                                self._cmd = self._cmd_pipe
                        else:
                                self.pipe = [int(os.environ['R2PIPE_IN']), int(os.environ['R2PIPE_OUT'])]
                                self._cmd = self._cmd_pipe
                        self.url = "#!pipe"
                        return
                except:
                        pass
                if filename.startswith("#!pipe"):
                        raise Exception("ERROR: Cannot use #!pipe without R2PIPE_{IN|OUT} env")
                if filename.startswith("http"):
                        self._cmd = self._cmd_http
                        self.uri = filename + "/cmd"
                elif filename.startswith("ccall://"):
                        self._cmd = self._cmd_native
                        self.uri = filename[7:]
                elif filename.startswith("tcp"):
                        r = re.match(r'tcp://(\d+\.\d+.\d+.\d+):(\d+)/?', filename)
                        if not r:
                                raise Exception("String doesn't match tcp format")
                        self._cmd = self._cmd_tcp
                        self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                        self.conn.connect((r.group(1), int(r.group(2))))
                else:
                        self._cmd = self._cmd_process
                        cmd = ["radare2", "-q0", filename]
                        cmd = cmd[:1] + flags + cmd[1:]
                        try:
                                self.process = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE)
                        except:
                                raise Exception("ERROR: Cannot find radare2 in PATH")
                        self.process.stdout.read(1)  # Reads initial \x00
                        # make it non-blocking to speedup reading
                        self.nonblocking = True
                        if self.nonblocking:
                                fd = self.process.stdout.fileno()
                                fl = fcntl.fcntl(fd, fcntl.F_GETFL)
                                fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

        def _cmd_process(self, cmd):
                cmd = cmd.strip().replace("\n", ";")
                if sys.version_info >= (3, 0):
                        self.process.stdin.write(bytes(cmd + '\n', 'utf-8'))
                        # XXX: Use the TextIOWrapper or we can get stuck in an endless loop!
                        r = TextIOWrapper(self.process.stdout, encoding='utf8')
                else:
                        self.process.stdin.write(cmd + '\n')
                        r = self.process.stdout
                self.process.stdin.flush()
                out = ''
                while True:
                        if self.nonblocking:
                                try:
                                        foo = r.read(4096)
                                except:
                                        continue
                        else:
                                foo = r.read(1)
                        if len(foo) > 0 and foo[-1] == '\x00':
                                out += foo[0:-1]
                                break
                        out += foo
                return out

        def _cmd_tcp(self, cmd):
                res = b''
                self.conn.sendall(str.encode(cmd, 'utf-8'))
                data = self.conn.recv(512)
                while data:
                        res += data
                        data = self.conn.recv(512)
                return res.decode('utf-8')

        def _cmd_pipe(self, cmd):
                out = ''
                cmd = cmd.strip().replace("\n", ";")
                if os.name == "nt":
                        windll.kernel32.WriteFile(self.pipe[1], cmd, len(cmd), byref(cbWritten), None)
                        while True:
                                windll.kernel32.ReadFile(self.pipe[1], chBuf, BUFSIZE, byref(cbRead), None)
                                out += chBuf.value
                                if ord(chBuf[cbRead.value - 1]) == 0:
                                        out = out[0:-1]
                                        break
                else:
                        os.write(self.pipe[1], cmd)
                        while True:
                                res = os.read(self.pipe[0], 4096)
                                if res[-1] == b'\x00':
                                        res = res[0:-1]
                                if (len(res) < 1):
                                        break
                                out += res
                                if (len(res) < 4096):
                                        break
                return out.decode('utf-8')

        def _cmd_native(self, cmd):
                cmd = cmd.strip().replace("\n", ";")
                if not has_native:
                        raise Exception('No native ctypes connector available')
                if not hasattr(self, 'native'):
                        self.native = RCore()
                        self.native.cmd_str("o " + self.uri)
                return self.native.cmd_str(cmd)

        def _cmd_rlang(self, cmd):
                return r2lang.cmd(cmd)

        def _cmd_http(self, cmd):
                try:
                        try:
                                quocmd = urllib.parse.quote(cmd)
                        except:
                                quocmd = urllib.quote(cmd)
                        response = urlopen('{uri}/{cmd}'.format(uri=self.uri, cmd=quocmd))
                        return response.read().decode('utf-8')
                except URLError:
                        pass
                return None

        def quit(self):
                """Quit current r2pipe session and kill
                """
                self.cmd("q")
                if hasattr(self, 'process'):
                        self.process.stdin.flush()
                        self.process.terminate()
                        self.process.wait()

        # r2 commands
        def cmd(self, cmd):
                """Run an r2 command return string with result
                Args:
                        cmd (str): r2 command
                Returns:
                        Returns an string with the results of the command
                """
                res = self._cmd(cmd)
                if res is not None:
                        return res.strip()
                return None

        def cmdj(self, cmd):
                """Same as cmd() but evaluates JSONs and returns an object
                Args:
                        cmd (str): r2 command
                Returns:
                        Returns a Python object respresenting the parsed JSON
                """
                try:
                        data = json.loads(self.cmd(cmd).replace('\n', ''))
                except (ValueError, KeyError, TypeError) as e:
                        sys.stderr.write("r2pipe.cmdj.Error: %s\n" % (e))
                        data = None
                return data

        def syscmd(self, cmd):
                """Executes a program and returns the output (stdout only)
                Args:
                        cmd (str): commandline shell command
                Returns:
                        Returns a string with the output
                """
                p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE)
                out, err = p.communicate()
                return out

        def syscmdj(self, cmd):
                """Executes a program and returns an object representing the parsed JSON of the output
                Args:
                        cmd (str): commandline shell command
                Returns:
                        Returns an object constructed by parsing the JSON returned by the command
                """
                try:
                        data = json.loads(self.syscmd(cmd))
                except (ValueError, KeyError, TypeError) as e:
                        sys.stderr.write("r2pipe.syscmdj.Error %s\n" % (e))
                        data = None
                return data

# Hello World
if __name__ == "__main__":
        print("[+] Spawning r2 tcp and http servers")
        os.system("pkill r2")
        os.system("r2 -qc.:9080 /bin/ls &")
        os.system("r2 -qc=h /bin/ls &")
        time.sleep(1)
        # Test r2pipe with local process
        print("[+] Testing python r2pipe local")
        rlocal = open("/bin/ls")
        print(rlocal.cmd("pi 5"))
        # print rlocal.cmd("pn")
        info = rlocal.cmdj("ij")
        print("Architecture: " + info['bin']['machine'])

        # Test r2pipe with remote tcp process (launch it with "r2 -qc.:9080 myfile")
        print("[+] Testing python r2pipe tcp://")
        rremote = open("tcp://127.0.0.1:9080")
        disas = rremote.cmd("pi 5")
        if not disas:
                print("Error with remote tcp conection")
        else:
                print(disas)

        # Test r2pipe with remote http process (launch it with "r2 -qc=H myfile")
        print("[+] Testing python r2pipe http://")
        rremote = open("http://127.0.0.1:9090")
        disas = rremote.cmd("pi 5")
        if not disas:
                print("Error with remote http conection")
        else:
                print(disas)
        os.system("pkill -INT r2")


import csv
from itertools import islice
from collections import namedtuple

f_names = 'cars.csv', 'personal_info.csv'

def get_dialect(f_name):
    with open(f_name) as f:
        return csv.Sniffer().sniff(f.read(1000))

class FileParser:
    def __init__(self, f_name):
        self.f_name = f_name

    def __enter__(self):
        self._f = open(self.f_name, 'r')
        self._reader = csv.reader(self._f, get_dialect(self.f_name))
        headers = map(lambda x: x.lower(), next(self._reader))
        self._nt = namedtuple('Data', headers)
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self._f.close()
        return False

    def __iter__(self):
        return self

    def __next__(self):
        if self._f.closed:
            # file has been closed - so we're can't iterate anymore!
            raise StopIteration
        else:
            return self._nt(*next(self._reader))

with FileParser('cars.csv') as data:
    for row in islice(data, 3):
        print(row)

with FileParser('personal_info.csv') as data:
    for row in islice(data, 3 ):
        print(row)


#!/usr/bin/env python

import sys
from Bio import SeqIO
from Bio.Seq import Seq
import re

def format_read(read):
    z = re.split('[|={,]+', read.description)
    return read.seq, z[3]

def load_meta_reads(filename, type='fasta'):
    try:
        seqs = list(SeqIO.parse(filename, type))
        #reads = []
        #labels = []
        is_paired_end = False
        if len(seqs) > 2 and seqs[0].id[-1:] != seqs[1].id[-1:]:
            is_paired_end = True

        label_list = dict()
        label_index = 0

        for i in range(0, len(seqs), 2 if is_paired_end else 1):
            read, label = format_read(seqs[i])
            if is_paired_end:
                read2, label2 = format_read(seqs[i + 1])
                read += read2
            print(str(i) + " " + str(read))

            if label not in label_list:
                label_list[label] = label_index
                label_index += 1
            print(str(i) + " " + str(label_list[label]))

        del seqs

    except:
        print('Error when loading file {} '.format(filename))

load_meta_reads(sys.stdin, type='fasta')


import os
import pathlib

pathloc = pathlib.Path.cwd() / "nem.txt"

def loc():

    if os.path.isfile(pathloc) == True :
        return True
    else :
        return False

def createfile():
    f = open(pathloc, mode="w+")

    f.close()

print (loc())


import sys
import numpy as np
import statsmodels.stats.contingency_tables as sm

in_file = sys.argv[1]
out_file = sys.argv[2]

df = np.loadtxt(in_file, dtype=float, delimiter=',')

p_values = []

for i in range(0, df.shape[1]):
	print(i)
	a1 = np.array([[df[0,i], df[1,i]],[df[2,i], df[3,i]]])
	a2 = np.array([[df[4,i], df[5,i]],[df[6,i], df[7,i]]])
	test_array = np.stack((a1, a2), axis=0)
	test_array = test_array + 0.5
	results = sm.StratifiedTable(test_array)
	odds_test = results.test_equal_odds()
	p_values.append(odds_test.pvalue)

listed_windows = open(out_file, 'w')

listed_windows.write("window1"+","+"window2"+","+"p-value"+ '\n')

wi1 = 1
wi2 = 1
window_row2 = 0
cutoff = 2578
for x in p_values:
	window_row2 +=1
	if window_row2 == cutoff:
		listed_windows.write(str(wi1) + "," + str(wi2) + "," + str(x) + '\n')
		wi2 = 1
		wi1 +=1
		window_row2 = 0
	else:
		listed_windows.write(str(wi1) + "," + str(wi2) + "," + str(x) + '\n')
		wi2 += 1

listed_windows.close()


# APPROACH 1: BRUTE FORCE SOLUTION
# Time Complexity : O(n^2), n: length of heights
# Space Complexity : O(1)
# Did this code successfully run on Leetcode : No (TIME LIMIT EXCEEDED)
# Any problem you faced while coding this : None
#
#
# Your code here along with comments explaining your approach
# 1. Calculate area for every pair of height such that second height is after the first height.
# 2. Keep track of max area.

class Solution:
    def maxArea(self, height: List[int]) -> int:

        if not height or len(height) < 2:
            return None

        max_area = 0
        for height_1 in range(len(height) - 1):
            for height_2 in range(height_1 + 1, len(height)):

                curr_area = (height_2 - height_1) * min(height[height_1], height[height_2])
                max_area = max(max_area, curr_area)

        return max_area

# APPROACH 2: OPTIMAL SOLUTION
# Time Complexity : O(n), n: length of heights
# Space Complexity : O(1)
# Did this code successfully run on Leetcode : Yes
# Any problem you faced while coding this : None
#
#
# Your code here along with comments explaining your approach
# 1. start from both end of heights.
# 2. Calculate area at each step -> distance between 2 heights * min of 2 heights
# 3. Then we shrink the distance between the 2 heights by moving either of the pointers. Since we are shrinking the distance, in order to get greater area need to find greater
#    height. So we shift the min of two heights, in hopes of finding a larger height

class Solution:
    def maxArea(self, height: List[int]) -> int:

        if not height or len(height) < 2:
            return None

        column_start, column_end = 0, len(height) - 1
        max_area = 0

        while column_start < column_end:
            curr_area = (column_end - column_start) * min(height[column_start], height[column_end])
            max_area = max(max_area, curr_area)

            if height[column_start] < height[column_end]:
                column_start += 1
            else:
                column_end -= 1

        return max_area


# Exercício 9
# Considerando o trecho de código que conecta em uma url na internet, imprima o dataframe conforme abaixo.

import json
import requests
import pandas as pd

print('\n\n\n\nExercício 09')

url = 'https://api.github.com/repos/pandas-dev/pandas/issues'

resp = requests.get(url)

data = resp.text

jsonData = json.loads(data)

values = pd.DataFrame(jsonData)

print(values)


# Define Global Conditions
CND_INIT = 1
CND_FINISH = 2
CND_BASIC_SEG_PRE_STOP = 3
CND_NEW_PRODUCT = 4
CND_PRODUCT_DONE = 5
CND_GRIPPER_RDY_1 = 6
CND_GRIPPER_RDY_2 = 7
CND_GRIPPER_RDY_3 = 8
CND_PRODUCT_ACTION = 9
CND_PROGRAM_DONE = 17

CND_CAMERA_TAKE_PIC = 20
CND_GRIPPER_ON = 21
CND_GRIPPER_OPEN = 22
CND_CLOUD_CONNECTED = 23
CND_START_CLOUD_PGM = 24

# define basic segment points index
PT_CURRENT = 0
PT_WAIT = 1
PT_GRIPPER_1 = 2
PT_GRIPPER_1_VO_IN = 3
PT_GRIPPER_1_VO_OUT = 4
PT_GRIPPER_2 = 5
PT_GRIPPER_2_VO_IN = 6
PT_GRIPPER_2_VO_OUT = 7
PT_GRIPPER_3 = 8
PT_GRIPPER_3_VO_IN = 9
PT_GRIPPER_3_VO_OUT = 10
PT_GRIPPER_4 = 11
PT_GRIPPER_4_VO_IN = 12
PT_GRIPPER_4_VO_OUT = 13
PT_PRODUCT_1 = 14
PT_PRODUCT_2 = 15
PT_PRODUCT_3 = 16
PT_PRODUCT_4 = 17
PT_GRIPPER_REF = 18
PT_PRODUCT_REF = 19
PT_PRODUCT_CAMERA = 20

PT_PALLETE_0_START = 32
PT_PALLETE_0_CNT_X = 4
PT_PALLETE_0_CNT_Y = 4

PT_PALLETE_1_START = 48
PT_PALLETE_1_CNT_X = 4
PT_PALLETE_1_CNT_Y = 4

PT_MAGAZIN_0_CAMERA_POS = 80
PT_MAGAZIN_1_CAMERA_POS = 81
PT_MAGAZIN_2_CAMERA_POS = 82
PT_MAGAZIN_3_CAMERA_POS = 83
PT_MAGAZIN_4_CAMERA_POS = 84

PT_MAGAZIN_ARRAY = [
    PT_MAGAZIN_0_CAMERA_POS,
    PT_MAGAZIN_1_CAMERA_POS,
    PT_MAGAZIN_2_CAMERA_POS,
    PT_MAGAZIN_3_CAMERA_POS,
    PT_MAGAZIN_4_CAMERA_POS
]

# Define program indexes
INDEX_START = 1
INDEX_MOVE_WAIT = 30
INDEX_MONTAGE = 100
INDEX_DEMONTAGE = 250
INDEX_ACTION_DONE = 400
INDEX_CLOUD_CMDS = 450
CLOUD_CMD_LENGTH = 500

# define Magazin
MAGAZINS_NAMES = ["MAGAZIN_0","MAGAZIN_1","MAGAZIN_2","MAGAZIN_3","MAGAZIN_4"]

MAGAZIN_POSITION_CAMERA = \
    [
        [610, 270, 150],
        [1000, 270, 150],
        [1440, 260, 150],
        [1850, 250, 150],
        [2360, 250, 150]
    ]

# Define product position camera
PRODUCT_POSITION_CAMERA = [1442, -94, 170]

PRODUCT_SINGLE_PLACE_POSITION = [1442, -94, 0]

PRODUCT_PLACE_POSITION_IN_BOX = [
    [1360, -78, 10.0],
    [1360, -230, 10.0],
    [1500, -78, 10.0],
    [1500, -230, 10.0]
]


from SignalModel import Signal
from LayerSelect import LayerSelect
import matplotlib.pyplot as plt
import numpy as np
import math as mt
from BlockThMethod import BlockThMethod
from ThresholdSelect import ThSelect
from BestThEstimation import BestThEstimate
from ThSearchAlgorithm import ThSearchAlgorithm
from WtProcess import DWTP
from DenoiseResult import DenoiseRsult

if __name__ == '__main__':
    s = Signal(1000)
    od1 = s.createSin(50, 60, 4)
    od2 = s.createPulse(50, 60, 4)
    od3 = s.createSpikes(4)
    ods = [od1, od2, od3]
    dnames = ['Sine', 'Pulse', 'Spike']
    gthss = []
    bsthss = []
    gthpds = []
    bsthpds =[]
    dl = 4
    for od in ods:
        dwd = DWTP(od)
        coeffs = dwd.dwtdec(wtname='db4', delevel=dl)
        gths = []
        bsths = []
        for i in range(1, len(coeffs)):
            bthe = BestThEstimate(coeffs[i], thfunction='thf3', thp=2+2*i)
            thsa = ThSearchAlgorithm()
            gth = ThSelect(coeffs[i]).DonohoThEx()
            bsth = thsa.FibonacciSearch(bthe.msesurefuction, [0, 1.5 * gth], 0.001)
            gths.append(gth)
            bsths.append(bsth)
        gthss.append(gths)
        bsthss.append(bsths)

    for i in range(len(ods)):
        dwd = DWTP(ods[i])
        coeffs = dwd.dwtdec(wtname='db4', delevel=dl)
        gths = gthss[i]
        bsths = gthss[i]
        ncoeffs1 = dwd.thprocess2(coeffs, ths=gths, thf='thf3', thps=[2+2*k for k in range(dl)])
        ncoeffs2 = dwd.thprocess2(coeffs, ths=bsths, thf='thf3', thps=[2+2*k for k in range(dl)])
        gthpd = dwd.dwtrec(ncoeffs1, wtname='db4')
        bsthpd = dwd.dwtrec(ncoeffs2, wtname='db4')
        gthpds.append(gthpd)
        bsthpds.append(bsthpd)

    gthsnrs = []
    bsthsnrs = []
    gthmses = []
    bsthmses = []
    tds = [s.createSin(50, 60, 0), s.createPulse(50, 60, 0), s.createSpikes(0)]
    for i in range(len(tds)):
        gthdr = DenoiseRsult(tds[i], gthpds[i])
        bsthdr = DenoiseRsult(tds[i], bsthpds[i])
        gthsnrs.append(gthdr.snr())
        bsthsnrs.append(bsthdr.snr())
        gthmses.append(gthdr.mse())
        bsthmses.append(bsthdr.mse())
    print('gthsnrs = {0}'.format(gthsnrs))
    print('bsthsnrs = {0}'.format(bsthsnrs))
    print('gthmses = {0}'.format(gthmses))
    print('bsthmses = {0}'.format(bsthmses))

    x = [x for x in range(1000)]
    plt.figure()
    for i in range(len(ods)):
        plt.subplot(3, 2, 2 * i + 1)
        plt.title('{0} signal processed by gth'.format(dnames[i]))
        plt.plot(x, gthpds[i])
        plt.subplot(3, 2, 2 * i + 2)
        plt.title('{0} signal processed by bsth'.format(dnames[i]))
        plt.plot(x, bsthpds[i])
    plt.tight_layout()
    plt.show()

Experiment 1 with self-definitive signal/t1.7  processing by new threshold function with different parameter.py


# coding: utf-8

"""
    Yapily API

    To access endpoints that require authentication, use your application key and secret created in the Dashboard (https://dashboard.yapily.com)  # noqa: E501

    OpenAPI spec version: 0.0.155

    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

import pprint
import re  # noqa: F401

import six

from yapily.models.account_identification import AccountIdentification  # noqa: F401,E501
from yapily.models.amount import Amount  # noqa: F401,E501
from yapily.models.payee import Payee  # noqa: F401,E501

class PaymentRequest(object):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """

    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'payment_idempotency_id': 'str',
        'payer_account_identifications': 'list[AccountIdentification]',
        'amount': 'Amount',
        'reference': 'str',
        'context_type': 'str',
        'type': 'str',
        'payment_date_time': 'datetime',
        'payee': 'Payee'
    }

    attribute_map = {
        'payment_idempotency_id': 'paymentIdempotencyId',
        'payer_account_identifications': 'payerAccountIdentifications',
        'amount': 'amount',
        'reference': 'reference',
        'context_type': 'contextType',
        'type': 'type',
        'payment_date_time': 'paymentDateTime',
        'payee': 'payee'
    }

    def __init__(self, payment_idempotency_id=None, payer_account_identifications=None, amount=None, reference=None, context_type=None, type=None, payment_date_time=None, payee=None):  # noqa: E501
        """PaymentRequest - a model defined in Swagger"""  # noqa: E501

        self._payment_idempotency_id = None
        self._payer_account_identifications = None
        self._amount = None
        self._reference = None
        self._context_type = None
        self._type = None
        self._payment_date_time = None
        self._payee = None
        self.discriminator = None

        self.payment_idempotency_id = payment_idempotency_id
        if payer_account_identifications is not None:
            self.payer_account_identifications = payer_account_identifications
        if amount is not None:
            self.amount = amount
        if reference is not None:
            self.reference = reference
        if context_type is not None:
            self.context_type = context_type
        if type is not None:
            self.type = type
        if payment_date_time is not None:
            self.payment_date_time = payment_date_time
        self.payee = payee

    @property
    def payment_idempotency_id(self):
        """Gets the payment_idempotency_id of this PaymentRequest.  # noqa: E501

        :return: The payment_idempotency_id of this PaymentRequest.  # noqa: E501
        :rtype: str
        """
        return self._payment_idempotency_id

    @payment_idempotency_id.setter
    def payment_idempotency_id(self, payment_idempotency_id):
        """Sets the payment_idempotency_id of this PaymentRequest.

        :param payment_idempotency_id: The payment_idempotency_id of this PaymentRequest.  # noqa: E501
        :type: str
        """
        if payment_idempotency_id is None:
            raise ValueError("Invalid value for `payment_idempotency_id`, must not be `None`")  # noqa: E501

        self._payment_idempotency_id = payment_idempotency_id

    @property
    def payer_account_identifications(self):
        """Gets the payer_account_identifications of this PaymentRequest.  # noqa: E501

        :return: The payer_account_identifications of this PaymentRequest.  # noqa: E501
        :rtype: list[AccountIdentification]
        """
        return self._payer_account_identifications

    @payer_account_identifications.setter
    def payer_account_identifications(self, payer_account_identifications):
        """Sets the payer_account_identifications of this PaymentRequest.

        :param payer_account_identifications: The payer_account_identifications of this PaymentRequest.  # noqa: E501
        :type: list[AccountIdentification]
        """

        self._payer_account_identifications = payer_account_identifications

    @property
    def amount(self):
        """Gets the amount of this PaymentRequest.  # noqa: E501

        :return: The amount of this PaymentRequest.  # noqa: E501
        :rtype: Amount
        """
        return self._amount

    @amount.setter
    def amount(self, amount):
        """Sets the amount of this PaymentRequest.

        :param amount: The amount of this PaymentRequest.  # noqa: E501
        :type: Amount
        """

        self._amount = amount

    @property
    def reference(self):
        """Gets the reference of this PaymentRequest.  # noqa: E501

        :return: The reference of this PaymentRequest.  # noqa: E501
        :rtype: str
        """
        return self._reference

    @reference.setter
    def reference(self, reference):
        """Sets the reference of this PaymentRequest.

        :param reference: The reference of this PaymentRequest.  # noqa: E501
        :type: str
        """

        self._reference = reference

    @property
    def context_type(self):
        """Gets the context_type of this PaymentRequest.  # noqa: E501

        :return: The context_type of this PaymentRequest.  # noqa: E501
        :rtype: str
        """
        return self._context_type

    @context_type.setter
    def context_type(self, context_type):
        """Sets the context_type of this PaymentRequest.

        :param context_type: The context_type of this PaymentRequest.  # noqa: E501
        :type: str
        """
        allowed_values = ["BILL", "GOODS", "SERVICES", "OTHER", "PERSON_TO_PERSON"]  # noqa: E501
        if context_type not in allowed_values:
            raise ValueError(
                "Invalid value for `context_type` ({0}), must be one of {1}"  # noqa: E501
                .format(context_type, allowed_values)
            )

        self._context_type = context_type

    @property
    def type(self):
        """Gets the type of this PaymentRequest.  # noqa: E501

        :return: The type of this PaymentRequest.  # noqa: E501
        :rtype: str
        """
        return self._type

    @type.setter
    def type(self, type):
        """Sets the type of this PaymentRequest.

        :param type: The type of this PaymentRequest.  # noqa: E501
        :type: str
        """
        allowed_values = ["DOMESTIC_PAYMENT", "DOMESTIC_VARIABLE_RECURRING_PAYMENT", "DOMESTIC_SCHEDULED_PAYMENT"]  # noqa: E501
        if type not in allowed_values:
            raise ValueError(
                "Invalid value for `type` ({0}), must be one of {1}"  # noqa: E501
                .format(type, allowed_values)
            )

        self._type = type

    @property
    def payment_date_time(self):
        """Gets the payment_date_time of this PaymentRequest.  # noqa: E501

        :return: The payment_date_time of this PaymentRequest.  # noqa: E501
        :rtype: datetime
        """
        return self._payment_date_time

    @payment_date_time.setter
    def payment_date_time(self, payment_date_time):
        """Sets the payment_date_time of this PaymentRequest.

        :param payment_date_time: The payment_date_time of this PaymentRequest.  # noqa: E501
        :type: datetime
        """

        self._payment_date_time = payment_date_time

    @property
    def payee(self):
        """Gets the payee of this PaymentRequest.  # noqa: E501

        :return: The payee of this PaymentRequest.  # noqa: E501
        :rtype: Payee
        """
        return self._payee

    @payee.setter
    def payee(self, payee):
        """Sets the payee of this PaymentRequest.

        :param payee: The payee of this PaymentRequest.  # noqa: E501
        :type: Payee
        """
        if payee is None:
            raise ValueError("Invalid value for `payee`, must not be `None`")  # noqa: E501

        self._payee = payee

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, PaymentRequest):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        return not self == other


# 用户可以通过本程序实现键盘输入SDK程序控制Tello无人机

# 调用相关的模块
import socket
import threading
import time
import sys

# Tello的IP和接口
tello_address = ('192.168.31.143', 8889)

# 本地电脑的IP和接口
local_address = ('192.168.31.172', 8890)

# 创建一个接收用户指令的UDP连接
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(local_address)

# 定义send命令，发送send里面的指令给Telo无人机
def send(message):
  try:
    sock.sendto(message.encode(), tello_address)
    print("Sending message: " + message)
  except Exception as e:
    print("Error sending: " + str(e))

# 定义receive命令，循环接收来自Tello的信息
def receive():
  while True:
    try:
      response, ip_address = sock.recvfrom(128)
      print("Received message: " + response.decode(encoding='utf-8'))
    except Exception as e:
      sock.close()
      print("Error receiving: " + str(e))
      break

# 开始一个监听线程，利用receive命令持续监控无人机发回的信号
receiveThread = threading.Thread(target=receive)
receiveThread.daemon = True
receiveThread.start()

# 告诉用户做什么
print('输入一个Tello SDK命令并按回车键，初始命令command，输入“quit”结束程序')

# 无限循环等待指令输入直到输入“quit”或ctrl-c
while True:

  try:
    # 读取用户从键盘输入的指令
    if (sys.version_info > (3, 0)):
      message = input('')
    else:
      message = raw_input('')

    # 如果用户输入quit就结束循环并关闭socket
    if 'quit' in message:
      print("Program exited sucessfully")
      sock.close()
      break

    # 把指令发送给Tello
    send(message)

  # ctrl-c退出并关闭socket
  except KeyboardInterrupt as e:
    sock.close()
    break