from __future__ import print_function

import os

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO
from tqdm import trange
from models import *
from tensorflow.contrib.slim.python.slim.nets import resnet_v1
from tensorflow.contrib.slim.python.slim.nets.mobilenet import mobilenet_v2

class Trainer(object):
    def __init__(self, config, get_loader):

        self.config = config
        self.get_loader = get_loader
        self.dataset = config.dataset

        self.beta1 = config.beta1
        self.beta2 = config.beta2
        self.batch_size = config.batch_size

        self.step = tf.Variable(0, name='step', trainable=False)

        self.j = config.j
        self.lr = tf.Variable(config.lr, name='lr')
        self.lr_mtplr = tf.Variable(config.lr_mtplr, name='lr_mtplr')

        self.model_dir = config.model_dir
        self.load_path = config.load_path
        self.summary_path = os.path.join(config.ckpt_dir)
        self.pre_dir = config.data_dir[0] + '/' + config.pre_dir

        self.use_gpu = config.use_gpu
        self.data_format = config.data_format

        self.start_step = 0
        self.log_step = config.log_step
        self.save_step = config.save_step
        self.k_t = tf.Variable(0., trainable=False)

        data = self.get_loader
        self.iterator = data.make_initializable_iterator()
        self.data = self.iterator.get_next()

        self.is_train = config.is_train
        self.build_model()

        self.saver = tf.compat.v1.train.Saver()
        self.summary_writer = tf.compat.v1.summary.FileWriter(self.summary_path)

        sv = tf.compat.v1.train.Supervisor(logdir=self.model_dir,
                                 is_chief=True,
                                 saver=self.saver,
                                 summary_op=None,
                                 summary_writer=self.summary_writer,
                                 save_model_secs=1200,
                                 global_step=self.step,
                                 ready_for_local_init_op=None)

        gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)
        sess_config = tf.compat.v1.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options)

        self.sess = sv.prepare_or_wait_for_session(config=sess_config)

        if not self.is_train:
            g = tf.get_default_graph()
            g._finalized = False

    def train(self):
        self.initial_iterator()
        self.init_fn(self.sess)
        self.init_fn_1(self.sess)
        # teacher first, student last
        # 2d train first, 3d last
        for step in trange(self.start_step, self.max_step):
            if step < self.two_d_pose:
                fetch_dict = {
                    '2d_t': self.two_d_teacher_optim,
                }
            elif step < self.two_d_pose + self.three_d_pose:
                if step is self.two_d_pose:
                    self.data_change()
                fetch_dict = {
                    '3d_t': self.three_d_teachter_optim,
                }
            elif step < self.two_d_pose + self.three_d_pose + self.two_d_pose:
                fetch_dict = {
                    '2d_s': self.two_d_teacher_optim,
                }
            else:
                fetch_dict = {
                    '3d_s': self.three_d_student_optim,
                }
            if step % self.log_step == 0:
                fetch_dict.update({
                    "summary": self.summary_op,
                    "teacher_heat_map_loss": self.t_hm_loss,
                    "teacher_local_map_loss": self.t_lm_loss,
                    "student_heat_map_loss": self.s_hm_loss,
                    "student_local_map_loss": self.s_lm_loss,
                })
            result = self.sess.run(fetch_dict)

            if step % self.log_step == 0:
                self.summary_writer.add_summary(result['summary'], step)
                self.summary_writer.flush()
                t_hm_loss = result['teacher_heat_map_loss']
                t_lm_loss = result['teacher_local_map_loss']
                s_hm_loss = result['student_heat_map_loss']
                s_lm_loss = result['student_local_map_loss']
                print(
                    "[{}/{}] Loss_FD: [{:.6f}] Loss_SD: [{:.6f}] || Loss_G: [{:.6f}]" \
                        .format(step, self.max_step, t_hm_loss, t_lm_loss, s_hm_loss, s_lm_loss))

    def build_model(self):
        with tf.compat.v1.variable_scope('preprocessing'):
            two_d_x, two_d_gt = self.preprocessing()
            x, gt = two_d_x, two_d_gt

        batch_input = utils.upscale_to(x, [224, 224], 'NCHW')
        batch_input, batch_mean_rgb = self.mean_rgb(batch_input)
        t, self.t_var = self.teacher(batch_input, self.j)
        s, self.s_var = self.student(batch_input, self.j)
        # print(t, s)
        with tf.compat.v1.variable_scope("loss_design"):
            with tf.compat.v1.variable_scope("optimizer"):
                two_d_optimizer = tf.compat.v1.train.RMSPropOptimizer(self.lr)
                three_d_optimizer = tf.compat.v1.train.AdamOptimizer(self.lr * self.lr_mtplr)

            with tf.compat.v1.variable_scope("loss_function"):
                t = utils.upscale_to_bil(t, [256, 256], 'NCHW')
                s = utils.upscale_to_bil(s, [256, 256], 'NCHW')
                t_h, t_l = tf.split(t, [self.j, -1], axis=1)  # heat for joint, x/y/z for joint
                s_h, s_l = tf.split(s, [self.j, -1], axis=1)  # heat for joint, x/y/z for joint
                gt_h, gt_l = two_d_gt, tf.concat([two_d_gt, two_d_gt, two_d_gt], 1)  # but data has only where is heat and x/y/z for joint?
                with tf.compat.v1.variable_scope("t_loss"):
                    self.t_hm_loss = tf.reduce_mean(tf.pow(tf.abs(t_h - gt_h), 2.))
                    self.t_lm_loss = tf.reduce_mean(tf.pow(tf.abs(t_l - gt_l), 2.))
                with tf.compat.v1.variable_scope("s_loss"):
                    alpha = 0.5
                    self.s_hm_loss = tf.reduce_mean(alpha * tf.pow(tf.abs(s_h - gt_h), 2.)
                                            + (1.-alpha) * tf.pow(tf.abs(s_h - t_h), 2.)
                    )
                    self.s_lm_loss = tf.reduce_mean(alpha * tf.pow(tf.concat([gt_h, gt_h , gt_h], 1) * tf.abs(s_l - gt_l), 2.)
                                             + (1. - alpha) * tf.pow(tf.concat([gt_h, gt_h , gt_h], 1) * tf.abs(s_l - t_l), 2.)
                                             )
            with tf.compat.v1.variable_scope("minimizer"):
                self.two_d_teacher_optim = two_d_optimizer.minimize(self.t_hm_loss + self.t_lm_loss, var_list=self.t_var)
                self.two_d_studnet_optim = two_d_optimizer.minimize(self.s_hm_loss + self.s_lm_loss, var_list=self.s_var)
                self.three_d_teachter_optim = three_d_optimizer.minimize(self.t_hm_loss + self.t_lm_loss, var_list=self.t_var)
                self.three_d_student_optim = three_d_optimizer.minimize(self.s_hm_loss + self.s_lm_loss, var_list=self.s_var)

        with tf.compat.v1.variable_scope("de_normal"):
            self.t_h = tf.clip_by_value((t_h + 1.) * 127.5, 0., 255.)
            self.t_l = tf.clip_by_value((t_l + 1.) * 127.5, 0., 255.)
            self.s_h = tf.clip_by_value((s_h + 1.) * 127.5, 0., 255.)
            self.s_l = tf.clip_by_value((s_l + 1.) * 127.5, 0., 255.)
            self.gt_h = tf.clip_by_value((gt_h + 1.) * 127.5, 0., 255.)
            self.gt_l = tf.clip_by_value((gt_l + 1.) * 127.5, 0., 255.)

        self.summary_op = tf.compat.v1.summary.merge([
            tf.compat.v1.summary.scalar("loss/t/hm_loss", self.t_hm_loss),
            tf.compat.v1.summary.scalar("loss/t/lm_loss", self.t_lm_loss),
            tf.compat.v1.summary.scalar("loss/s/hm_loss", self.s_hm_loss),
            tf.compat.v1.summary.scalar("loss/s/lm_loss", self.s_lm_loss),
            tf.compat.v1.summary.image("t/h", self.t_h),
            tf.compat.v1.summary.image("t/l", self.t_l),
            tf.compat.v1.summary.image("s/h", self.s_h),
            tf.compat.v1.summary.image("s/l", self.s_l),
            tf.compat.v1.summary.image("gt/h", self.gt_h),
            tf.compat.v1.summary.image("gt/l", self.gt_l),
        ])

    def build_test_model(self):
        print("test_model")

    def test(self):
        self.build_test_model()

    def initial_iterator(self):
        self.sess.run(self.iterator.initializer)

    def preprocessing(self):
        with tf.compat.v1.variable_scope("Pre"):
            two_d_x, two_d_joint = self.data  # [B, 256, 256, 3]  [B, 256, 256, 15]
            print(two_d_x, two_d_joint)
            two_d_x= self.norm(two_d_x)
            two_d_x = utils.nhwc_to_nchw(two_d_x)
            two_d_joint = utils.nhwc_to_nchw(two_d_joint)
        return two_d_x, two_d_joint

    def norm(self, x):
        return tf.clip_by_value((x/127.5)-1., -1., 1.)

    def load_data(self):
        self.initial_iterator()
        x = self.data
        return x

    def generate(self, image, fixed, root_path=None, path=None, idx=None, save=True):
        x = self.sess.run(self.Gz, {self.z: image})
        if path is None and save:
            path = os.path.join(root_path, '{}_G.mp4'.format(idx))
            utils.save_video(x, fixed, path)
            print("[*] Samples saved: {}".format(path))
        return x

    def mean_rgb(self, x):
        r, g, b = tf.split(x, 3, axis=1)
        mean_r = tf.reduce_mean(r, axis=[2, 3], keep_dims=True)
        mean_g = tf.reduce_mean(g, axis=[2, 3], keep_dims=True)
        mean_b = tf.reduce_mean(b, axis=[2, 3], keep_dims=True)
        r = r - mean_r
        g = g - mean_g
        b = b - mean_b
        return tf.concat([r, g, b], 1), tf.concat([mean_r, mean_g, mean_b], 0)

    def teacher(self, x, j):
        with slim.arg_scope(resnet_v1.resnet_arg_scope()):
            x = utils.nchw_to_nhwc(x)
            batch_out, batch_list = resnet_v1.resnet_v1_50(x, 1000, is_training=True)
            feature = batch_list['resnet_v1_50/block2/unit_4/bottleneck_v1/conv1']

        self.init_fn_1 = slim.assign_from_checkpoint_fn(
            self.pre_dir + '/resnet_v1_50.ckpt', slim.get_model_variables('resnet_v1_50'))
        '''
        del which has no gradient
        '''
        # print(batch_list)
        x = utils.nhwc_to_nchw(feature)
        x, var = vnect(x, j)
        return x, var

    def student(self, x, j):
        with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope()):
            x = utils.nchw_to_nhwc(x)
            batch_out, batch_list = mobilenet_v2.mobilenet_v2_035(x, 1000, is_training=True)
            # print(batch_list)
            mobilenet_feature = batch_list['layer_12/output']

        self.init_fn = slim.assign_from_checkpoint_fn(
            self.pre_dir + '/mobilenet_v2_035.ckpt', slim.get_model_variables('MobilenetV2'))

        x = utils.nhwc_to_nchw(mobilenet_feature)
        x, var = movnect(x, j)
        return x, var


from PyQt4.QtCore import *
from PyQt4.QtGui import *
import resources

class Ui_MainWindow(object):
    def setupUi(self, window):
        window.setWindowTitle("Forest Trails")

        self.centralWidget = QWidget(window)
        self.centralWidget.setMinimumSize(800, 400)
        window.setCentralWidget(self.centralWidget)

        self.menubar = window.menuBar()
        self.fileMenu = self.menubar.addMenu("File")
        self.mapMenu = self.menubar.addMenu("Map")
        self.editMenu = self.menubar.addMenu("Edit")
        self.toolsMenu = self.menubar.addMenu("Tools")

        self.toolbar = QToolBar(window)
        window.addToolBar(Qt.TopToolBarArea, self.toolbar)

        self.actionQuit = QAction("Quit", window)
        self.actionQuit.setShortcut(QKeySequence.Quit)

        icon = QIcon(":/resources/mActionZoomIn.png")
        self.actionZoomIn = QAction(icon, "Zoom In", window)
        self.actionZoomIn.setShortcut(QKeySequence.ZoomIn)

        icon = QIcon(":/resources/mActionZoomOut.png")
        self.actionZoomOut = QAction(icon, "Zoom Out", window)
        self.actionZoomOut.setShortcut(QKeySequence.ZoomOut)

        icon = QIcon(":/resources/mActionPan.png")
        self.actionPan = QAction(icon, "Pan", window)
        self.actionPan.setShortcut("Ctrl+1")
        self.actionPan.setCheckable(True)

        icon = QIcon(":/resources/mActionEdit.png")
        self.actionEdit = QAction(icon, "Edit", window)
        self.actionEdit.setShortcut("Ctrl+2")
        self.actionEdit.setCheckable(True)

        icon = QIcon(":/resources/mActionAddTrack.png")
        self.actionAddTrack = QAction(icon, "Add Track", window)
        self.actionAddTrack.setShortcut("Ctrl+A")
        self.actionAddTrack.setCheckable(True)

        icon = QIcon(":/resources/mActionEditTrack.png")
        self.actionEditTrack = QAction(icon, "Edit Track", window)
        self.actionEditTrack.setShortcut("Ctrl+E")
        self.actionEditTrack.setCheckable(True)

        icon = QIcon(":/resources/mActionDeleteTrack.png")
        self.actionDeleteTrack = QAction(icon, "Delete Track", window)
        self.actionDeleteTrack.setShortcut("Ctrl+D")
        self.actionDeleteTrack.setCheckable(True)

        icon = QIcon(":/resources/mActionGetInfo.png")
        self.actionGetInfo = QAction(icon, "Get Info", window)
        self.actionGetInfo.setShortcut("Ctrl+I")
        self.actionGetInfo.setCheckable(True)

        icon = QIcon(":/resources/mActionSetStartPoint.png")
        self.actionSetStartPoint = QAction(icon, "Set Start Point", window)
        self.actionSetStartPoint.setCheckable(True)

        icon = QIcon(":/resources/mActionSetEndPoint.png")
        self.actionSetEndPoint = QAction(icon, "Set End Point", window)
        self.actionSetEndPoint.setCheckable(True)

        icon = QIcon(":/resources/mActionFindShortestPath.png")
        self.actionFindShortestPath = QAction(icon, "Find Shortest Path",
                                              window)
        self.actionFindShortestPath.setCheckable(True)

        self.fileMenu.addAction(self.actionQuit)

        self.mapMenu.addAction(self.actionZoomIn)
        self.mapMenu.addAction(self.actionZoomOut)
        self.mapMenu.addAction(self.actionPan)
        self.mapMenu.addAction(self.actionEdit)

        self.editMenu.addAction(self.actionAddTrack)
        self.editMenu.addAction(self.actionEditTrack)
        self.editMenu.addAction(self.actionDeleteTrack)
        self.editMenu.addAction(self.actionGetInfo)

        self.toolsMenu.addAction(self.actionSetStartPoint)
        self.toolsMenu.addAction(self.actionSetEndPoint)
        self.toolsMenu.addAction(self.actionFindShortestPath)

        self.toolbar.addAction(self.actionZoomIn)
        self.toolbar.addAction(self.actionZoomOut)
        self.toolbar.addAction(self.actionPan)
        self.toolbar.addAction(self.actionEdit)
        self.toolbar.addSeparator()
        self.toolbar.addAction(self.actionAddTrack)
        self.toolbar.addAction(self.actionEditTrack)
        self.toolbar.addAction(self.actionDeleteTrack)
        self.toolbar.addAction(self.actionGetInfo)
        self.toolbar.addSeparator()
        self.toolbar.addAction(self.actionSetStartPoint)
        self.toolbar.addAction(self.actionSetEndPoint)
        self.toolbar.addAction(self.actionFindShortestPath)

        window.resize(window.sizeHint())


import time
import graphics
from base_graphics import GraphicObjectContainer

from interface_control import MainScreen

class MainEngine(object):

    exit_now = False
    shape_1_name = 'battleship'

    def __init__(self):
        self.shape_dct = {}
        user_boat = graphics.BattleShip(MainScreen.screen_width, MainScreen.screen_height)
        self.user_obj = GraphicObjectContainer(user_boat)

        self.main_screen = MainScreen(left_fun=self.user_obj.left,
                                      right_fun=self.user_obj.right,
                                      speed_up_fun=self.user_obj.speed_up,
                                      slow_down_fun=self.user_obj.slow_down)
        self.register_shapes_to_screen()
        self.user_obj.primary_object.register_shape(self.shape_1_name)

        self.run_engine()

    def run_engine(self):
        while True:
            self.user_obj.move()
            time.sleep(.02)
            self.main_screen.get_input()
            if self.main_screen.exit_now:
                self.main_screen.screen.bye()
                break

    def register_shapes_to_screen(self):
        # list all shapes used
        self.shape_dct[self.shape_1_name] = graphics.BattleShip(MainScreen.screen_width, MainScreen.screen_height)

        # register, then hide and delete these objects, we are only using these for shape registration.
        for shape_name, shape_obj in self.shape_dct.iteritems():
            self.main_screen.screen.register_shape(shape_name, shape_obj.return_shape())
            shape_obj.boat.clear()
            shape_obj.boat.ht()
            del shape_obj
            self.shape_dct[self.shape_1_name] = None

eng = MainEngine()


# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/gaogaotiantian/viztracer/blob/master/NOTICE.txt

import os
import multiprocessing
import builtins
import signal
import sys
from .tracer import _VizTracer
from .flamegraph import FlameGraph
from .report_builder import ReportBuilder
from .vizplugin import VizPluginManager
from .vizevent import VizEvent

# This is the interface of the package. Almost all user should use this
# class for the functions
class VizTracer(_VizTracer):
    def __init__(self,
                 tracer_entries=1000000,
                 verbose=1,
                 max_stack_depth=-1,
                 include_files=None,
                 exclude_files=None,
                 ignore_c_function=False,
                 ignore_frozen=False,
                 log_func_retval=False,
                 log_func_args=False,
                 log_print=False,
                 log_gc=False,
                 log_sparse=False,
                 log_async=False,
                 vdb=False,
                 pid_suffix=False,
                 file_info=True,
                 register_global=True,
                 output_file="result.json",
                 plugins=[]):
        super().__init__(
            tracer_entries=tracer_entries,
            max_stack_depth=max_stack_depth,
            include_files=include_files,
            exclude_files=exclude_files,
            ignore_c_function=ignore_c_function,
            ignore_frozen=ignore_frozen,
            log_func_retval=log_func_retval,
            log_print=log_print,
            log_gc=log_gc,
            vdb=vdb,
            log_func_args=log_func_args,
            log_async=log_async
        )
        self.verbose = verbose
        self.pid_suffix = pid_suffix
        self.file_info = file_info
        self.output_file = output_file
        self.system_print = None
        self.log_sparse = log_sparse
        if register_global:
            self.register_global()

        self._afterfork_cb = None
        self._afterfork_args = None
        self._afterfork_kwargs = None

        # load in plugins
        self._plugin_manager = VizPluginManager(self, plugins)

    @property
    def verbose(self):
        return self.__verbose

    @verbose.setter
    def verbose(self, verbose):
        if type(verbose) is str:
            try:
                self.__verbose = int(verbose)
            except ValueError:
                raise ValueError("Verbose needs to be an integer, not {}".format(verbose))
        elif type(verbose) is int:
            self.__verbose = verbose
        else:
            raise ValueError("Verbose needs to be an integer, not {}".format(verbose))

    @property
    def pid_suffix(self):
        return self.__pid_suffix

    @pid_suffix.setter
    def pid_suffix(self, pid_suffix):
        if type(pid_suffix) is bool:
            self.__pid_suffix = pid_suffix
        else:
            raise ValueError("pid_suffix needs to be a boolean, not {}".format(pid_suffix))

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, type, value, trace):
        self.stop()
        if type is None:
            self.save()
        self.terminate()

    def register_global(self):
        builtins.__dict__["__viz_tracer__"] = self

    def install(self):
        if sys.platform == "win32":
            print("remote install is not supported on Windows!")
            exit(1)

        def signal_start(signum, frame):
            self.start()

        def signal_stop(signum, frame):
            self.stop()
            self.save()

        signal.signal(signal.SIGUSR1, signal_start)
        signal.signal(signal.SIGUSR2, signal_stop)

    def log_event(self, event_name):
        call_frame = sys._getframe(1)
        return VizEvent(self, event_name, call_frame.f_code.co_filename, call_frame.f_lineno)

    def set_afterfork(self, callback, *args, **kwargs):
        self._afterfork_cb = callback
        self._afterfork_args = args
        self._afterfork_kwargs = kwargs

    def start(self):
        if not self.enable:
            self._plugin_manager.event("pre-start")
            _VizTracer.start(self)

    def stop(self):
        if self.enable:
            _VizTracer.stop(self)
            self._plugin_manager.event("post-stop")

    def run(self, command, output_file=None):
        self.start()
        exec(command)
        self.stop()
        self.save(output_file)

    def save(self, output_file=None, save_flamegraph=False, file_info=None):
        if file_info is None:
            file_info = self.file_info
        enabled = False
        if self.enable:
            enabled = True
            self.stop()
        if not self.parsed:
            self.parse()
        if output_file is None:
            output_file = self.output_file
        if self.pid_suffix:
            output_file_parts = output_file.split(".")
            output_file_parts[-2] = output_file_parts[-2] + "_" + str(os.getpid())
            output_file = ".".join(output_file_parts)

        self._plugin_manager.event("pre-save")

        output_file = os.path.abspath(output_file)
        if not os.path.isdir(os.path.dirname(output_file)):
            os.makedirs(os.path.dirname(output_file), exist_ok=True)

        rb = ReportBuilder(self.data, self.verbose)
        rb.save(output_file=output_file, file_info=file_info)

        if save_flamegraph:
            self.save_flamegraph(".".join(output_file.split(".")[:-1]) + "_flamegraph.html")

        if enabled:
            self.start()

    def generate_json(self, allow_binary=False):
        rb = ReportBuilder(self.data, self.verbose)
        return rb.generate_json(allow_binary=allow_binary)

    def generate_report(self):
        rb = ReportBuilder(self.data, self.verbose)
        return rb.generate_report()

    def fork_save(self, output_file=None, save_flamegraph=False):
        if multiprocessing.get_start_method() != "fork":
            # You have to parse first if you are not forking, address space is not copied
            # Since it's not forking, we can't pickle tracer, just set it to None when
            # we spawn
            if not self.parsed:
                self.parse()
            tracer = self._tracer
            self._tracer = None
        else:
            # Fix the current pid so it won't give new pid when parsing
            self._tracer.setpid()

        p = multiprocessing.Process(target=self.save, daemon=False,
                                    kwargs={"output_file": os.path.abspath(output_file), "save_flamegraph": save_flamegraph})
        p.start()

        if multiprocessing.get_start_method() != "fork":
            self._tracer = tracer
        else:
            # Revert to the normal pid mode
            self._tracer.setpid(0)

    def save_flamegraph(self, output_file=None):
        flamegraph = FlameGraph(self.data)
        if output_file is None:
            name_list = self.output_file.split(".")
            output_file = ".".join(name_list[:-1]) + "_flamegraph.html"
        flamegraph.save(output_file)

    def terminate(self):
        self._plugin_manager.terminate()


from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestNeighbors
from csv import reader
import sys
import numpy as np
from pyspark import SparkContext
from csv import reader
from operator import add
import pandas as pd
import heapq
from pyspark.ml.feature import OneHotEncoder, StringIndexer
from pyspark.sql.types import BooleanType
from pyspark.sql.functions import udf,col,when
from pyspark.sql.types import IntegerType,StringType
from pyspark.sql.functions import isnan

data = sc.textFile("/FileStore/tables/h9gi_nx95-fba74.tsv")
data = data.mapPartitions(lambda x: reader(x, delimiter='\t'))
header = data.first()
data = data.filter(lambda row : row != header)

def isfloat(value):
  try:
    float(value)
    return True
  except ValueError:
    return False
def dofloat(entry):
  k = []
  for i in range(len(entry)):
    if isfloat(entry[i]) != True:
      k.append(i)
  return k
df = spark.createDataFrame(data, header)

m = df.count()
for i in range(len(header)):
  k = df.filter((df[header[i]] == "") | df[header[i]].isNull() | isnan(df[header[i]])).count()
  null_percent = float(k)/m
  if null_percent > 0.1:
    df = df.drop(header[i])
    continue

  categories = df.select(header[i]).sample(withReplacement=False, fraction=0.1).distinct().replace('', '0').rdd.flatMap(lambda x : x).collect()
  tem = dofloat(categories)
  if len(tem) == 0:
    continue
  if len(tem) >10:
    df = df.drop(header[i])
    continue
  categories = df.select(header[i]).distinct().rdd.flatMap(lambda x : x).collect()
  for category in categories:
      function = udf(lambda item: 1 if item == category else 0, StringType())
      new_column_name = header[i]+'_'+category
      df = df.withColumn(new_column_name, function(col(header[i])))
  df = df.drop(header[i])

df = df.replace('', '0')
data = df.rdd.map(list)
data_collect = df.rdd.map(list).collect()

k_neighbors = 25
knnobj = NearestNeighbors(n_neighbors = k_neighbors).fit(data_collect)
bc_knnobj = sc.broadcast(knnobj)
results = data.map(lambda x: bc_knnobj.value.kneighbors(x))

def cleanresult(entry,k):
  k_neighbor = entry[0][0][k]
  neighbors = entry[1][0]
  neighboures_distance = entry[0][0]
  return [k_neighbor, neighbors,neighboures_distance]

kdistanceres= results.map(lambda x:cleanresult(x,k_neighbors-1)).zipWithIndex().collect()

def add_kdiatanceofO(kdistanceres):
  for i in range(len(kdistanceres)):
    kdiatanceofO = []
    for j in kdistanceres[i][0][1]:
      kdiatanceofO.append(kdistanceres[j][0][0])
    kdistanceres[i][0].append(kdiatanceofO)

add_kdiatanceofO(kdistanceres)

kdistanceres_rdd = sc.parallelize(kdistanceres)
def reach_distance(d1,d2):
  return max(d1,d2)

def lrd(entry,k):
  sum_reach_distance = 0
  for i in range(1,k):
    sum_reach_distance += reach_distance(entry[0][2][i], entry[0][3][i])
  return ([k/sum_reach_distance,entry[0][1] ], entry[1])

lrd_rdd = kdistanceres_rdd.map(lambda x:lrd(x,k_neighbors))
lrd = lrd_rdd.collect()

def add_lrdOfO(lrd):
  for i in range(len(lrd)):
    sum_lrdOfO = 0
    for j in lrd[i][0][1]:
      sum_lrdOfO += lrd[j][0][0]
    lrd[i][0].append(sum_lrdOfO)
add_lrdOfO(lrd)

lrd_rdd = sc.parallelize(lrd)
def lof(entry,k):
  lof = entry[0][2]/k/entry[0][0]
  return [lof,entry[1]]
lrd = lrd_rdd.map(lambda x:lof(x,k_neighbors)).collect()

def n_outlier(entry,n):
  entry = sorted(entry, key=lambda x: x[0] ,reverse = True)
  return entry[0:n]
outlier = n_outlier(lrd,100)


import datetime
from sqlalchemy import (
    MetaData,
    Table,
    Column,
    Integer,
    Date,
    Boolean,
    DateTime,
    BigInteger,
)

meta = MetaData()

def upgrade(migrate_engine):
    meta.bind = migrate_engine

    t = Table("user", meta, autoload=True)

    project_id = Column("project_id", Integer, index=True)
    project_id.create(t, index_name='idx__user__project_id')
    current_portal_user_yn = Column("current_portal_user_yn", Boolean)
    current_portal_user_yn.create(t)
    gv_end_del_log = Column("gv_end_del_log", Date)
    gv_end_del_log.create(t)
    last_update_timestamp = Column("last_update_timestamp", BigInteger, nullable=True, index=True)
    last_update_timestamp.create(t, index_name="idx__user__last_update_timestamp")
    is_imported = Column("is_imported", Boolean)
    is_imported.create(t)

def downgrade(migrate_engine):
    meta.bind = migrate_engine
    t = Table("user", meta, autoload=True)
    t.c.project_id.drop()
    t.c.current_portal_user_yn.drop()
    t.c.gv_end_del_log.drop()
    t.c.last_update_timestamp.drop()
    t.c.is_imported.drop()


'''
This python script wraps some R packages such as annotables and qqman, or some R codes in python environment using rpy2
Please see more documentation at http://rpy2.readthedocs.org/en/version_2.7.x/
'''

'''
################################
#### Software prerequistes #####
################################
'''
# python packages:
# rpy2:       pip install rpy2

# R packages:
# annotables:     install.packages("devtools")
#                 devtools::install_github("stephenturner/annotables")
# dplyr:          install.packages('dplyr')
# ggplot2:        install.packages("tidyverse")
# qqman:          install.packages("qqman")

'''
###################
#### Modules  #####
###################
'''
from rpy2.robjects import r
from rpy2.robjects.packages import importr
from rpy2.robjects import pandas2ri
from rpy2 import robjects
from helpers import *
from datetime import datetime
import os, pandas, glob, sqlite3, shutil, subprocess
import uuid as myuuid

pandas2ri.activate()

projectID = str(myuuid.uuid4())

'''
######################
#### Annotations #####
######################
'''
def annotate_metaxcan_result(projectName):

    global projectID

    # load R packages
    importr('annotables')
    importr('dplyr',  on_conflict="warn")

    # current path
    currentPath = os.getcwd()

    # root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # input path
    input_path = root_path + '/input/'

    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    # annoated_files output file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/'
    if not os.path.exists(annoated_files_path):
        os.makedirs(annoated_files_path)

    # get input file lists (raw metaxcan output files *.csv)
    inputFileList = glob.glob("*.csv")

    # loop through file lists
    for inputFilename in inputFileList:
        msg = "ANNOTATING GENES: " + inputFilename
        add_log(msg)

        # read data
        r("data <- read.csv('%s')" %inputFilename)
        data = r('na.omit(data)')
        robjects.globalenv['dataframe'] = data

        # annotation library
        grch37 = r('grch37')
        robjects.globalenv['dataframe'] = grch37

        # annotating
        # if inputFilename == 'xxxxxDGN-WB-unscaled.csv':
        #     annotatedData = r("inner_join(data[, c('gene', 'gene_name', 'zscore', 'effect_size', 'pvalue', 'n_snps_in_model', 'pred_perf_p', 'pred_perf_R2')], grch37[, c('symbol', 'chr', 'start', 'end')], by=c('gene'='symbol'))")
        # else:
        annotatedData = r("inner_join(data[, c('gene', 'gene_name', 'zscore', 'effect_size', 'pvalue', 'n_snps_in_model', 'pred_perf_p', 'pred_perf_R2')], grch37[, c('ensgene', 'chr', 'start', 'end')], by =c('gene'='ensgene'))")
        annotatedData = annotatedData.drop_duplicates()

        # ouput annotated data
        os.chdir(annoated_files_path)
        annotatedData.to_csv(annoated_files_path + inputFilename[:-4] + '_annotated.csv', index=None)

        os.chdir(input_path)

    # merge all annotated metaxcan files
    # get annotated file lists
    os.chdir(annoated_files_path)
    outputFileListWithoutSNPs = glob.glob("*.csv")

    # merged annotated file path
    merged_annotated_file_path = annoated_files_path + 'merged/'
    if not os.path.exists(merged_annotated_file_path):
        os.makedirs(merged_annotated_file_path)

    dfListWithoutSNPs = []
    for outputFileName in outputFileListWithoutSNPs:
        # read data
        msg = "MERGING ALL ANNOTATED FILES ..."
        add_log(msg)
        msg = "CONCATING FILE: " + outputFileName
        add_log(msg)

        r("data <- read.csv('%s')" %outputFileName)
        data = r('data <- na.omit(data)')
        robjects.globalenv['dataframe'] = data

        if 'CrossTissue_elasticNet' in outputFileName:
            outputFileName = outputFileName[:-25]
        elif 'DGN-WB-unscaled' in outputFileName:
            outputFileName = outputFileName [:-23]
        else:
            outputFileName = outputFileName[:-25]
            outputFileName = outputFileName[3:]

        data.insert(2, 'tissue', outputFileName)
        dfListWithoutSNPs.append(data)
    concatDfWithoutSNPs = pandas.concat(dfListWithoutSNPs, axis = 0)

    #output data
    os.chdir(merged_annotated_file_path)
    concatDfWithoutSNPs.to_csv(merged_annotated_file_path + 'merged_annotated_metaxcan_output.csv', index = None)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
####################
####  QQ-Plot  #####
####################
'''
def qqplot(projectName):

    global projectID

    # load R package
    qqman = importr('qqman')

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # annotated metaxcan file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/'
    os.chdir(annoated_files_path)

    qqplot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/QQ-plot/'
    if not os.path.exists(qqplot_output_path):
         os.makedirs(qqplot_output_path)

    outputFileListWithoutSNPs = glob.glob("*.csv")

    # loop through file lists
    for outputFileName in outputFileListWithoutSNPs:
        msg = "QQ-PLOT(no snps): " + outputFileName
        add_log(msg)

        # get GWAS output data
        r("data <- read.csv('%s')" %outputFileName)
        data = r('data <- na.omit(data)')
        robjects.globalenv['dataframe'] = data
        os.chdir(qqplot_output_path)

        # draw qq-plot and save them to files
        r.png('%s%s%s'%('QQ-Plot_', outputFileName[:-4],'.png'))
        qqman.qq(data['pvalue'],  main = "MetaXcan genes")
        r['dev.off']()
        os.chdir(annoated_files_path)

    msg = "QQ-PLOT(no snps): " + "merged_annotated_metaxcan_output"
    add_log(msg)

    # set up path and read merged annotated metaxcan output files
    os.chdir(annoated_files_path + 'merged/')
    r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    data = r('data <- na.omit(data)')
    robjects.globalenv['dataframe'] = data
    os.chdir(qqplot_output_path)

    # draw qq-plot and save them to files
    r.png('%s%s%s'%('QQ-Plot_', 'metaxcan_output', '.png'))
    qqman.qq(data['pvalue'],  main = "QQ plot")
    r['dev.off']()

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
###########################
####  Manhattan-Plot  #####
###########################
'''
def manhattan(projectName):

    global projectID

    # load module
    qqman = importr('qqman')

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # annotated metaxcan file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/annotated_metaxcan_output_files/'
    os.chdir(annoated_files_path)

    manhattanplot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/Manhattan-Plot/'
    if not os.path.exists(manhattanplot_output_path):
         os.makedirs(manhattanplot_output_path)

    # get file lists
    outputFileListWithoutSNPs = glob.glob("*.csv")

    # loop through file lists
    for outputFileName in outputFileListWithoutSNPs:
        msg = "MANHATTAN-PLOT(no snps): " + outputFileName
        add_log(msg)

        # read data
        r("data <- read.csv('%s')" %outputFileName)
        r('data$chr <- as.numeric(as.character(data$chr))')
        data = r('data <- na.omit(data)')
        robjects.globalenv['dataframe'] = data

        os.chdir(manhattanplot_output_path)

        # draw manhattan and save them to files
        r.png('%s%s%s'%('Manhattan-Plot_', outputFileName[:-4], '.png'))
        qqman.manhattan(data, chr = 'chr', bp='start', p='pvalue', snp='gene_name', main = 'MetaXcan genes')
        r['dev.off']()
        os.chdir(annoated_files_path)

    msg = "MANHATTAN-PLOT(no snps): " + "merged_annotated_metaxcan_output"
    add_log(msg)

    # read data
    os.chdir(annoated_files_path + 'merged/')
    r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    r('data$chr <- as.numeric(as.character(data$chr))')
    data = r('data <- na.omit(data)')
    robjects.globalenv['dataframe'] = data

    os.chdir(manhattanplot_output_path)

    # draw manhattan and save them to files
    r.png('%s%s%s'%('Manhattan-Plot_', 'merged_annotated_metaxcan_output.csv', '.png'))
    qqman.manhattan(data, chr = 'chr', bp='start', p='pvalue', snp='gene_name', main = 'Manhattan plot')
    r['dev.off']()

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
######################################
####   Top Gene List without SNPs ####
######################################
'''
def sortTopGeneList(projectName):

    global projectID

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # annotated metaxcan file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/annotated_metaxcan_output_files/'
    os.chdir(annoated_files_path)

    top_genes_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/top_genes/'
    if not os.path.exists(top_genes_output_path):
         os.makedirs(top_genes_output_path)

    # get file lists
    fileLists = glob.glob("*.csv")

    # dfList = []
    # total_rows_in_total = 0
    for filename in fileLists:
        msg = "TOP GENE LIST: " + filename
        add_log(msg)

        df = pandas.read_csv(filename)

        if 'CrossTissue_elasticNet' in filename:
            tissue_name = filename[:-25]
        elif 'DGN-WB-unscaled' in filename:
            tissue_name = filename [:-23]
        else:
            tissue_name = filename[:-25]
            tissue_name = tissue_name[3:]
        df.insert(2, 'tissue', tissue_name)

        # sort data by defined column
        df.sort_values(['pvalue', 'n_snps_in_model'], ascending=[1, 0], inplace=True)
        total_rows = df.shape[0] # number of row count shape[1] is number of col count
        msg = 'total_row within tissue: ' + str(total_rows)
        add_log(msg)
        # total_rows_in_total += total_rows
        cut_off = 0.05/total_rows

        # cut_off_list.append(cut_off)
        top_gene_list = df[df['pvalue'] < cut_off]
        # top_gene_list = top_gene_list[top_gene_list['pred_perf_R2'] > 0.01]
        # top_gene_list.drop('gene', axis=1, inplace=True)
        msg = 'CALCULATING TISSUE-WIDE P-VALUE: ' + str(cut_off)
        add_log(msg)
        add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

        #output data
        top_gene_list.to_csv(top_genes_output_path + "sorted_top_genes_%s"%filename, index = None)

        # dfList.append(top_gene_list)

    os.chdir(annoated_files_path + 'merged/')

    msg = "TOP GENES: merged_annotated_metaxcan_output.csv"
    add_log(msg)

    df = pandas.read_csv('merged_annotated_metaxcan_output.csv')

    total_rows_in_total = df.shape[0]
    msg = 'total_row_in_total: ' + str(total_rows_in_total)
    add_log(msg)

    total_cut_off = 0.05/total_rows_in_total

    msg = 'CALCULATING GENOME-WIDE P-VALUE: ' + str(total_cut_off)
    add_log(msg)

    df = df[df['pvalue'] < total_cut_off]

    # sort data by defined column
    df.sort_values(['pvalue', 'n_snps_in_model'], ascending=[1, 0], inplace=True)

    #output data
    df.to_csv(top_genes_output_path+ 'sorted_top_genes.csv', index = None)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
####################################
####   Top Gene List with SNPs #####
####################################
'''
def sortTopGeneListWithSNPs(projectName):

    global projectID

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # set up path
    top_genes_file_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    os.chdir(top_genes_file_path)

    # read data
    top_genes = pandas.read_csv('sorted_top_genes.csv')

    # add into files
    gene_lists = top_genes['gene_name']
    tissue_lists = top_genes['tissue']
    pvalue_lists = top_genes['pvalue']
    zscore_lists = top_genes['zscore']
    model_n_lists = top_genes['n_snps_in_model']
    pred_perf_R2_lists = top_genes['pred_perf_R2']
    chr_lists = top_genes['chr']
    start_lists = top_genes['start']
    end_lists = top_genes['end']

    # get db lists
    input_path = root_path + '/input/'

    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    dbFileList = glob.glob("*.db")

    database_names = []
    for dbFilename in dbFileList:
       database_names.append(dbFilename)

    # Loop through databases
    query_output_list = []
    for i in range(len(database_names)):
        for k in range(len(tissue_lists)):
            if tissue_lists[k] in database_names[i]:
                # Connect databases
                conn = sqlite3.connect(database_names[i])

                full_query_name = None
                if database_names[i] == 'DGN-WB-unscaled_0.5.db':
                    full_query_name = SQL_QUERY_PREFIX_DNG + gene_lists[k] + "'"
                else:
                    full_query_name = SQL_QUERY_PREFIX + gene_lists[k] + "'"

                query_output = pandas.read_sql(full_query_name, conn, index_col=None)

                if database_names[i] == 'DGN-WB-unscaled_0.5.db':
                    query_output.rename(columns={'gene':'genename'}, inplace=True)

                # # Add correspinding parameters to the new output file
                query_output['tissue'] = tissue_lists[k]
                query_output['pvalue'] =  pvalue_lists[k]
                query_output['zscore'] =  zscore_lists[k]
                query_output['model_n'] = model_n_lists[k]
                query_output['pred_perf_R2'] = pred_perf_R2_lists[k]
                query_output['chr'] =  chr_lists[k]
                query_output['start'] = start_lists[k]
                query_output['end'] = end_lists[k]
                query_output_list.append(query_output)

                msg = 'FETCH SNPs for GENE %s ' % gene_lists[k] +  'FROM DATABASE: %s' % database_names[i]
                add_log(msg)

                # Close database
                conn.close()

    # Merge output data
    query_output_of = pandas.concat(query_output_list, axis = 0)
    # gwas_snp = pandas.read_csv('gwas_snp.txt', sep="\s+")
    # gwas_snp.rename(columns={'MarkerName':'rsid', 'P-value':'gwas_pvalue'}, inplace=True)

    # query_output_of.merge(gwas_snp, on='rsid', how='inner')

    # get tissue abbr name
    # load module
    ggplot2 = importr('ggplot2')
    dplyr = importr('dplyr',  on_conflict="warn")

    r("""

    tissue_abbr <- read.delim('gtex_tissue_abbr.txt', sep='\t') %>%
    select(tissue_site_detail_abbr,tissue_site_detail_id)
    colnames(tissue_abbr)[colnames(tissue_abbr) == 'tissue_site_detail_id'] = "tissue"

    """)

    top_genes_snps_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes_snps/'

    if not os.path.exists(top_genes_snps_output_path):
         os.makedirs(top_genes_snps_output_path)
    os.chdir(top_genes_snps_output_path)

    # Output merged data
    query_output_of.to_csv("top_genes_snps.csv", index=None)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
######################
###  Bubble Plot #####
######################
'''
def bubbleplot(projectName):

    global projectID

    # load module
    ggplot2 = importr('ggplot2')
    dplyr = importr('dplyr',  on_conflict="warn")

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # get gwas_lead_snp.txt file
    input_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    gwas_lead_snp = r("""
        gwas_lead_snp <- read.csv('sorted_top_genes.csv') %>%
          select(gene_name, chr, start) %>%
          distinct()
        colnames(gwas_lead_snp) = c('snpsNames', 'chrosome', 'startSites')

        """)
    robjects.globalenv['dataframe'] = gwas_lead_snp

    # annotated metaxcan file path
    merged_annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/merged/'
    os.chdir(merged_annoated_files_path)

    data = r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    robjects.globalenv['dataframe'] = data

    bubble_plot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/bubble_plot/'
    if not os.path.exists(bubble_plot_output_path):
         os.makedirs(bubble_plot_output_path)
    os.chdir(bubble_plot_output_path)

    # for loop through each snps site +/- 1000000 bp
    r("""
        # read dataframe
        startSites = as.numeric(as.character(gwas_lead_snp$startSites))
        snpsNames = as.character(gwas_lead_snp$snpsNames)
        chrosome = as.numeric(as.character(gwas_lead_snp$chrosome))

        # for loop through each snps site +/- 1000000 bp
        for (i in 1:length(startSites))
        {
            print(paste("BUBBLE PLOTING", ": ", snpsNames[i]))

            # subset data for each snps
            subData <- subset(data, data$start > startSites[i] - 1000000  &
                data$start < startSites[i] + 1000000 & data$chr==chrosome[i])
            subData <- subData[order(subData$start),]
            subData <- mutate(subData, z_score=ifelse(subData$zscore > 0, '   +   ', '   -   '))
            # Labels = subData$gene_name
            subData$gene_name <- factor(subData$gene_name, levels=subData$gene_name)
            # subData$tissue <- factor(subData$tissue, levels=subData$tissue)
            subData <- subData %>%
            inner_join(tissue_abbr, by = "tissue")

            # draw plot
            p <- ggplot(subData, aes(x=subData$gene_name, y=subData$tissue_site_detail_abbr, size=abs(subData$zscore)))
            p +
            geom_point(aes(colour=z_score)) +
            scale_color_manual(values=c('blue', 'brown')) +
            scale_size_continuous(guide=FALSE, range=c(0,max(abs(subData$zscore))))+
            # ggtitle(paste('locus: ', snpsNames[i], '(chromosome', chrosome[i], ')')) +
            labs(x='Gene', y='Tissue') +
            ggtitle(snpsNames[i])+
            # scale_x_discrete(breaks = subData$gene_name, labels=Labels) +
            theme(axis.text.x = element_text(size=16, face='bold', angle = 90, hjust = 1)) +
            theme(axis.text.y = element_text(size=16, face='bold')) +
            theme(plot.title = element_text(size=18, face='bold')) +
            # theme(axis.title= element_text(size=18, face='bold')) +
            theme(axis.title.x = element_blank(), axis.title.y = element_blank()) +
            theme(legend.position = "none") +
            theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank())

            # save plot
            ggsave(paste(snpsNames[i], '_bubble_plot', '.png', sep=''), width=12, height=12)
        }
    """)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
######################
###  Region Plot #####
######################
'''
def regionplot(projectName):

    global projectID

    # load module
    ggplot2 = importr('ggplot2')
    dplyr = importr('dplyr',  on_conflict="warn")

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # get gwas_lead_snp.txt file
    input_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    gwas_lead_snp = r("""
        gwas_lead_snp <- read.csv('sorted_top_genes.csv') %>%
          select(gene_name, chr, start) %>%
          distinct()
        colnames(gwas_lead_snp) = c('snpsNames', 'chrosome', 'startSites')

        """)
    robjects.globalenv['dataframe'] = gwas_lead_snp

    # annotated metaxcan file path
    merged_annoated_files_path = root_path + '/output/' + projectName +  "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/merged/'
    os.chdir(merged_annoated_files_path)

    data = r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    robjects.globalenv['dataframe'] = data

    bubble_plot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/region_plot/'
    if not os.path.exists(bubble_plot_output_path):
         os.makedirs(bubble_plot_output_path)
    os.chdir(bubble_plot_output_path)

    r("""
        # read dataframe
        startSites = as.numeric(as.character(gwas_lead_snp$startSites))
        snpsNames = as.character(gwas_lead_snp$snpsNames)
        chrosome = as.numeric(as.character(gwas_lead_snp$chrosome))

        # loop through snps site +/- 1000,000 bps
        for (i in 1:length(startSites))
        {
        print(paste("REGION PLOTING", ": ", snpsNames[i]))

        # subset data
        subData <- subset(data, data$start > startSites[i] - 1000000  & data$start < startSites[i]
            + 1000000 & data$chr==chrosome[i])
        subData$logp <- -log10(subData$pvalue)
        subData <- subData[order(subData$start),]

        subData <- mutate(subData, sig=ifelse(subData$logp > -log10(0.05/nrow(data)), 'Most Sig',
        ifelse(subData$logp > 5.30103 & subData$logp <= -log10(0.05/nrow(data)), 'Sig',
        ifelse(subData$logp > -log10(0.05/nrow(subData))
            & subData$logp <= 5.30103, 'Less Sig','Not Sig'))))
        subData$gene_name <- factor(subData$gene_name, levels=subData$gene_name)

        p <- ggplot(subData, aes(x=gene_name, y=logp))
        p + geom_point(aes(colour = sig)) +
        scale_color_manual(guide=FALSE, values=c('black', 'black', 'black', 'black')) +
            # ggtitle(paste("locus: ", snpsNames[i], '(chromosome', chrosome[i], ')')) +
        labs(x='Gene', y='-log10(p-value)') +
        ggtitle(snpsNames[i])+
        geom_hline(yintercept = 5.30103, linetype='dashed', color='black') +
        geom_hline(yintercept = -log10(0.05/nrow(data)), color='black') +
        geom_hline(yintercept = -log10(0.05/nrow(subData)), linetype='dashed', color='black') +
        theme(axis.text.x = element_text(size=16, face='bold', angle = 90, hjust = 1)) +
        theme(axis.text.y = element_text(size=16, face='bold')) +
        theme(plot.title = element_text(size=18, face='bold')) +
        theme(axis.title.x = element_blank(), axis.title.y=element_text(size=18, face='bold')) +
        theme(legend.position = "none")+
        theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank())

        # output plot
        ggsave(paste(snpsNames[i], '_region_plot', '.png',sep=''), width=12, height=12)
        }
    """)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
#########################
###  locuszoom Plot #####
#########################
'''
def locuszoom_plot(projectName):

    dplyr = importr('dplyr',  on_conflict="warn")

    global projectID

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # make batch_locuszoom.txt file
    input_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    gwas_lead_snp = r("""
        gwas_lead_snp <- read.csv('sorted_top_genes.csv') %>%
          select(gene_name, chr, start, end) %>%
          distinct() %>%
          mutate(flank='1.0MB', run = 'yes', m2zargs= "showAnnot=F")
          colnames(gwas_lead_snp) = c('snp', 'chr', 'start', 'stop', 'flank', 'run', 'm2zargs')
          write.table(gwas_lead_snp, file="batch_locuszoom.txt", sep=" ", quote=FALSE, row.names=F)

        """)

    # write gwas_lead_snp into txt file
    # gwas_lead_snp.to_csv("batch_locuszoom.txt", sep='\t')

    # create a folder to hold locuszoom results
    locuszoom_plot_path = root_path + '/locuszoom/locuszoom_plots/'
    if not os.path.exists(locuszoom_plot_path):
        os.makedirs(locuszoom_plot_path)

    # source files from input path
    # copy the files including plink, run_locuszoom.py and two other .txt from input path into locuszoom program
    destination = locuszoom_plot_path     # locuszoom plot destination path
    plink_destination = root_path + '/locuszoom'

    source = os.listdir(input_path)

    for file in source:
        if file.endswith(".txt"):
            shutil.copy(file, destination)
            msg = "COPYING: the file '%s' into the folder '%s'" % (file, destination)
            add_log(msg)
    os.system('rm batch_locuszoom.txt')

    # input path
    input_path = root_path + '/input/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' % input_path)
    os.chdir(input_path)

    source = os.listdir(input_path)

    plink_file = 'plink'
    shutil.copy(plink_file, plink_destination)

    msg = "COPYING: the file '%s' into the folder '%s'" % (plink_file, plink_destination)
    add_log(msg)

    for file in source:
        if file.endswith(".txt"):
            shutil.copy(file, destination)
            msg = "COPYING: the file '%s' into the folder '%s'" % (file, destination)
            add_log(msg)

    locuszoom_path = root_path + '/MetaXcan-Postprocess/scripts/Python_scripts'
    os.chdir(locuszoom_path)

    locuszoom_scrip_cmd = 'run_locuszoom.py'
    shutil.copy(locuszoom_scrip_cmd, destination)
    msg = "COPYING: the file '%s' into the folder '%s'" % (locuszoom_scrip_cmd, destination)
    add_log(msg)

    # run locuszoom program
    os.chdir(destination)
    os.system('./run_locuszoom.py')
    add_log('RUNNING: run_locuszoom.py...')

    # setup locuszoom plot output file path and move all files to this new output folder
    locuszoom_plot_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/looszoom_plot/'
    if not os.path.exists(locuszoom_plot_files_path):
        os.makedirs(locuszoom_plot_files_path)
    os.chdir(destination)

    src = destination
    dst = locuszoom_plot_files_path
    os.system('mv %s %s' % (src, dst))

    msg = "MOVING: all files from the folder '%s' into the folder '%s'" % (src, locuszoom_plot_files_path)
    add_log(msg)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")


#######################################################################
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
# Permission given to modify the code as long as you keep this        #
# declaration at the top                                              #
#######################################################################

from baby_rl import *

# TD3
def td3_continuous(**kwargs):
    generate_tag(kwargs)
    kwargs.setdefault('log_level', 0)
    config = Config()
    config.merge(kwargs)

    config.task_fn = lambda: Task(config.game)
    config.eval_env = config.task_fn()
    config.max_steps = int(1e5)
    config.eval_interval = int(1e4)
    config.eval_episodes = 20

    config.network_fn = lambda: TD3Net(
        config.action_dim,
        actor_body_fn=lambda: FCBody(config.state_dim, (400, 300), gate=F.relu),
        critic_body_fn=lambda: FCBody(
            config.state_dim+config.action_dim, (400, 300), gate=F.relu),
        actor_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-3),
        critic_opt_fn=lambda params: torch.optim.Adam(params, lr=1e-3))

    config.replay_fn = lambda: Replay(memory_size=int(1e6), batch_size=100)
    config.discount = 0.99
    config.random_process_fn = lambda: GaussianProcess(
        size=(config.action_dim,), std=LinearSchedule(0.1))
    config.td3_noise = 0.2
    config.td3_noise_clip = 0.5
    config.td3_delay = 2
    config.warm_up = int(1e4)
    config.target_network_mix = 5e-3
    run_steps(TD3Agent(config))

if __name__ == '__main__':
    mkdir('log')
    mkdir('tf_log')
    mkdir('data')
    set_one_thread()
    random_seed()
    # select_device(-1)
    select_device(0)

    game = 'CartPole-v0'
    # dqn_feature(game=game)
    # quantile_regression_dqn_feature(game=game)
    # categorical_dqn_feature(game=game)
    # a2c_feature(game=game)
    # n_step_dqn_feature(game=game)
    # option_critic_feature(game=game)

    # game = 'MountainCarContinuous-v0'
    game = 'Pendulum-v0'
    # game = 'HalfCheetah-v2'
    # game = 'Hopper-v2'
    # a2c_continuous(game=game)
    # ppo_continuous(game=game)
    # ddpg_continuous(game=game)
    td3_continuous(game=game)

    game = 'BreakoutNoFrameskip-v4'
    # dqn_pixel(game=game)
    # quantile_regression_dqn_pixel(game=game)
    # categorical_dqn_pixel(game=game)
    # a2c_pixel(game=game)
    # n_step_dqn_pixel(game=game)
    # option_critic_pixel(game=game)


# -*- coding:utf-8 -*-
# edit by fuzongfei

from django.urls import path, re_path
from django.contrib.auth.decorators import login_required

from .views import BeautifySQLView, GetInceptionHostConfigView, \
    GetDatabaseListView, \
    GetRemarkInfo, GetGroupView, GetDbaLeaderView, GetContactsView, \
    IncepOnlineSqlRecordsView, IncepOnlineClickVerifyView, IncepOnlineClickFinishView, IncepOnlineClickCloseView, \
    OnlineAuditDetailView, \
    OnlineSqlReplyView, IncepSqlCheckView, IncepOnlineSqlCheckView, IncepTasksRecordsView, \
    IncepTasksDetailView, IncepExecTaskView, IncepTasksResultView

urlpatterns = [
    path(r'get_inception_hostconfig/', login_required(GetInceptionHostConfigView.as_view()),
         name='p_inception_hostconfig'),
    path(r'get_database/', login_required(GetDatabaseListView.as_view()), name='p_get_database'),

    # 线上工单
    path(r'incep_online_sql_check/', login_required(IncepOnlineSqlCheckView.as_view()),
         name='p_incep_online_sql_check'),
    path(r'incep_online_sql_records/', login_required(IncepOnlineSqlRecordsView.as_view()),
         name='p_incep_online_sql_records'),
    re_path(r'online_sql_detail/(?P<id>\d+)/(?P<group_id>\d+)/', login_required(OnlineAuditDetailView.as_view())),

    # 获取信息
    path(r'get_remark_info/', login_required(GetRemarkInfo.as_view()), name='p_remark_info'),
    path(r'get_group_info/', login_required(GetGroupView.as_view()), name='p_get_group'),
    path(r'get_dba_leader/', login_required(GetDbaLeaderView.as_view()), name='p_get_dba_leader'),
    path(r'get_contacts/', login_required(GetContactsView.as_view()), name='p_get_contacts'),

    # 线上工单审批
    path(r'incep_online_click_verify/', login_required(IncepOnlineClickVerifyView.as_view()),
         name='p_incept_online_click_verify'),
    path(r'incep_online_click_finish/', login_required(IncepOnlineClickFinishView.as_view()),
         name='p_incep_online_click_finish'),
    path(r'incep_online_click_close/', login_required(IncepOnlineClickCloseView.as_view()),
         name='p_incep_online_click_close'),
    path(r'online_sql_reply/', login_required(OnlineSqlReplyView.as_view()), name='p_online_reply'),

    # 线下工单
    path(r'incep_sql_check/', login_required(IncepSqlCheckView.as_view()), name='p_incep_sql_check'),
    path(r'incep_exec_task/', login_required(IncepExecTaskView.as_view()), name='p_incep_exec_task'),
    path(r'incep_tasks_record/', login_required(IncepTasksRecordsView.as_view()), name='p_incep_tasks_records'),
    re_path(r'incep_tasks_detail/(?P<taskid>.*)/', login_required(IncepTasksDetailView.as_view())),
    re_path(r'incep_tasks_result/', login_required(IncepTasksResultView.as_view()), name='p_incep_tasks_result'),

    path(r'beautify_sql/', login_required(BeautifySQLView.as_view()), name='p_beautify_sql'),
]


from django.contrib import admin
from django.urls import path

from  store.views import Home,SignUp,Login,Cart

urlpatterns = [
    path('',Home.Index.as_view(),name='homepage'),
    path('signup',SignUp.SignUp.as_view(),name='SignUp'),
    path('login',Login.Login.as_view(),name='Login'),
    path('logout',Login.logout,name='logout'),
    path('cart',Cart.Cart.as_view(),name='cart')
]


import uuid
from flask import Flask, request, jsonify
from marshmallow import Schema, fields, ValidationError
from celery import Celery
from celery.result import AsyncResult

class PayloadSchema(Schema):
    fab = fields.Str()
    area = fields.Str()
    tool_name = fields.Str()
    start_date = fields.Str()
    end_date = fields.Str()

app = Flask(__name__)
app.config['CELERY_BROKER_URL'] = 'redis://localhost:5000'
app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:5555'
celery = Celery(
    app.name,
    broker=app.config['CELERY_BROKER_URL'],
    backend=app.config['CELERY_RESULT_BACKEND'])

@celery.task(bind=True, name="task.tool_health_spc_query", max_retries=3)
def tool_health_spc_query(self, final_payload):
    # SPC query code...
    data = 'results from SPC query'
    print('hello world')

    return {
        'status': 'task completed',
        'data': data
    }

@celery.task(bind=True, name="task.tool_health_fd_query", max_retries=3)
def tool_health_fd_query(self, final_payload):
    # FD query code...
    data = 'results from FD query'
    print('hello world')

    return {
        'status': 'task completed',
        'data': data
    }

class ToolHealthFactory:
    def __init__(self, payload):
        self.payload = payload.copy()   # payload is the request in dictionary

    def queue_task(self):
        pass

class SPCQuery(ToolHealthFactory):
    def __init__(self, payload, spc_db, spc_query):
        self.spc_db = spc_db
        self.spc_query = spc_query
        super().__init(payload)

    def queue_task(self):
        # add additional key value pairs to payload dictionary

        self.payload['spc_db'] = self.spc_db
        self.payload['spc_query'] = self.spc_query

        # add task to queue
        task = tool_health_spc_query.delay(self.payload)
        return {'id': task.id}

class FDQuery(ToolHealthFactory):
    def __init__(self, payload, endpoint, param):
        self.endpoint = endpoint
        self.param = param
        super().__init(payload)

    def queue_task(self):
        # add additional key value pairs to payload dictionary

        self.payload['endpoint'] = self.endpoint
        self.payload['param'] = self.param

        # add task to queue
        task = tool_health_fd_query.delay(self.payload)
        return {'id': task.id}

@app.route('/toolhealth', methods=['POST', 'GET'])
def tool_health():
    payload_json = request.body()
    try:
        payload = PayloadSchema().load(payload_json)

    except ValidationError as e:
        return jsonify({'error': e}), 400

    # create SPC and FD objects
    spc_query = SPCQuery(payload, 'spc db', 'spc query')
    fd_query = FDQuery(payload, 'endpoint', 'param')

    # pushing tasks to queue
    spc_task = spc_query.queue_task()
    fd_task = fd_query.queue_task()

    ids_dict = {
        'tool_health_id': uuid.uuid1(),
        'task_ids': {
            'spc_query_id': spc_task['id'],
            'fd_task_id': fd_task['id']
        }
    }

    return jsonify({
        'message': 'queue tasks successful',
        'ids': ids_dict
        }), 202

@app.route('/tool_health/<tool_health_id>', methods=['POST', 'GET'])
def tool_health_results(tool_health_id):
    payload = request.get_json()    # request body is ids_dict

    if payload['tool_health_id'] == tool_health_id:
        spc_result = AsyncResult(payload['task_ids']['spc_query_id'])
        fd_result = AsyncResult(payload['task_ids']['fd_query_id'])
        return jsonify({
            'spc status': spc_result.status,
            'spc results': spc_result.get(),
            'fd status': fd_result.status,
            'fd results': fd_result.get(),
            })

    return jsonify({
        'message': 'tool_health_id not found. Please try again.'
    })


import random
import subprocess, os
import pickle
import itertools
import math
import time
import copy
import collections

#load the dictionary for the cost function
with open('costF.pickle', 'rb') as restoreData:
    costF = pickle.load(restoreData)

######################################################################
#parameters
######################################################################
grid = 4   # we are going to create a grid by grid square
gridsize = 10**2
n_route = 4   # number of different routes for each of the four start-to-end trips, each route can have multiple cars
n_car = 10
randSeed = 1
crushTime = 100
badRoad1 = "11to12"
badRoad2 = "12to11"

speed = 50         #speed of the cars
acc = 3            #acceleration of the car
decel = 3          #decceleration of the car
max_Car = 4        #max cars per 100 meters

f_start = "<nodes>"
f_end = "</nodes>"
g_start = "<edges>"
g_end = "</edges>"
h_start = "<routes>"
h_end = "</routes>"

######################################################################
#Functions
######################################################################
### writing the node file
def writeNodeFile(filename, xgrid, ygrid):
    f = open(filename, 'w')
    f.write(f_start)
    f.write("\n")
    # the lower left point is the origin
    thisline = '    <node id="00" x="0.0" y="0.0" />'
    f.write(thisline)
    f.write('\n')
    #create the grid
    for i in range(1,grid):
        thisline = '    <node id="' + str(i) + str(0) + '"' + ' type="traffic_light" x="' + str(xgrid[i-1]) + '"'
        thisline += ' y="0.0"' + '/>'
        f.write(thisline)
        f.write('\n')
    thisline = '    <node id="' + str(grid) + str(0) + '"' + ' x="' + str(xgrid[grid-1]) + '"'
    thisline += ' y="0.0"' + '/>'
    f.write(thisline)
    f.write('\n')
    for j in range(1,grid):
        thisline = '    <node id="' + str(0) + str(j) + '"' + ' type="traffic_light" x="0.0"'
        thisline += ' y="' + str(ygrid[j-1]) + '"' + '/>'
        f.write(thisline)
        f.write('\n')
    thisline = '    <node id="' + str(0) + str(grid) + '"' + ' x="0.0"'
    thisline += ' y="' + str(ygrid[grid-1]) + '"' + '/>'
    f.write(thisline)
    f.write('\n')
    for i in range(1,grid+1):
        for j in range(1,grid+1):
            if(i==grid and j==grid):
                thisline = '    <node id="' + str(grid) + str(grid) + '"' + ' x="' + str(xgrid[grid-1]) + '"'
                thisline += ' y="' + str(ygrid[grid-1]) + '"' '/>'
                f.write(thisline)
                f.write('\n')
            else:
                thisline = '    <node id="' + str(i) + str(j) + '"' + ' type="traffic_light"  x="' + str(xgrid[i-1]) + '"'
                thisline += ' y="' + str(ygrid[j-1]) + '"' '/>'
                f.write(thisline)
                f.write('\n')
    f.write(f_end)
    f.close()

###writing the edge file
def writeEdgeFile(filename, num_of_grids):
    g = open(filename, 'w')
    g.write(g_start)
    g.write('\n')
    #connecting adjacent nods
    for i in range(0, num_of_grids+1):
        for j in range(0, num_of_grids+1):
            if(i<grid):
                rightline = '    <edge from="' + str(i)+str(j) + '"'
                rightline += ' id="' + str(i) + str(j) + 'to' + str(i+1) + str(j) + '"'
                rightline += ' to="' + str(i+1) + str(j) + '"' + '/>'
                g.write(rightline)
                g.write('\n')
                rightlineback = '    <edge from="' + str(i+1)+str(j) + '"'
                rightlineback += ' id="' + str(i+1) + str(j) + 'to' + str(i) + str(j) + '"'
                rightlineback += ' to="' + str(i) + str(j) + '"' + '/>'
                g.write(rightlineback)
                g.write('\n')
            if(j<grid):
                upline = '    <edge from="' + str(i)+str(j) + '"'
                upline += ' id="' + str(i) + str(j) + 'to' + str(i) + str(j+1) + '"'
                upline += ' to="' + str(i) + str(j+1) + '"' + '/>'
                g.write(upline)
                g.write('\n')
                uplineback = '    <edge from="' + str(i)+str(j+1) + '"'
                uplineback += ' id="' + str(i) + str(j+1) + 'to' + str(i) + str(j) + '"'
                uplineback += ' to="' + str(i) + str(j) + '"' + '/>'
                g.write(uplineback)
                g.write('\n')
    g.write(g_end)
    g.close()

###writing the additional file regarding the traffic light
def writeAddFile(filename, num_of_grids):
    j = open(filename, 'w')
    j.write('<additional>')
    j.write('\n')
    for ipos in range(0, num_of_grids+1):
        for jpos in range(0, num_of_grids+1):
            if([ipos,jpos] != [0,0] and [ipos,jpos] != [0, num_of_grids] and [ipos,jpos] != [num_of_grids, 0] and [ipos,jpos] != [num_of_grids,num_of_grids]):
                offsetTime = 0
                thisId = str(ipos) + str(jpos)
                thisline ='    <tlLogic id="' + thisId + '" programID="my_program" offset="' + str(offsetTime) + '" type="static">'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="31" state="GGggrrrrGGggrrrr"/>'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="5" state="yyggrrrryyggrrrr"/>'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="6" state="rrGGrrrrrrGGrrrr"/>'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="5" state="rryyrrrrrryyrrrr"/>'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="31" state="rrrrGGggrrrrGGgg"/>'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="5" state="rrrryyggrrrryygg"/>'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="6" state="rrrrrrGGrrrrrrGG"/>'
                j.write(thisline)
                j.write('\n')
                thisline = '        <phase duration="5" state="rrrrrryyrrrrrryy"/>'
                j.write(thisline)
                j.write('\n')
                j.write('    </tlLogic>')
                j.write('\n')
    j.write('</additional>')
    j.close()

###writing the route file
def writeRouFile(filename):
    vehRous = {}
    h = open(filename, 'w')
    h.write(h_start)
    h.write('\n')
    #car type
    thisline = '    <vType accel="' + str(acc) + '" decel="' + str(decel) + '" id="CarA" length="5.0" minGap="2.5" maxSpeed="'
    thisline += str(speed) + '"  sigma="0.5" />'
    h.write(thisline)
    h.write('\n')
    for i in range(0,len(trips)):
        if(i == 0):    #lowerleft_to_upperright
            x_movement = 1
            y_movement = 1
            x_start = 0
            y_start = 0
        if(i == 2):    #upperleft_to_lowerright
            x_movement = 1
            y_movement = -1
            x_start = 0
            y_start = grid
        if(i == 1):    #lowerright_to_upperleft
            x_movement = -1
            y_movement = 1
            x_start = grid
            y_start = 0
        if(i == 3):    #upperright_to_lowerleft
            x_movement = -1
            y_movement = -1
            x_start = grid
            y_start = grid
        #for each trip, generate n_route random routes
        for j in range(0, n_route):
            pos = list(range(0, grid*2))   #position to be shuffled
            random.seed(j)
            random.shuffle(pos)
            finalpos = sorted(pos[0:grid]) #position of the x-movement
            thisline = '    <route id="' + trip_names[i] + str(j) + '" edges="'
            thisRou = ''
            x_pos_1 = x_start
            y_pos_1 = y_start
            #the exact route
            for k in range(0, grid*2):
                if(k in finalpos):
                    x_pos_2 = x_pos_1 + x_movement
                    y_pos_2 = y_pos_1
                else:
                    x_pos_2 = x_pos_1
                    y_pos_2 = y_pos_1 + y_movement
                thisRou += str(x_pos_1) + str(y_pos_1) + 'to' + str(x_pos_2) + str(y_pos_2) + ' '
                x_pos_1 = x_pos_2
                y_pos_1 = y_pos_2
            thisRou = thisRou[0:-1]   #remove the last space
            thisline += thisRou
            thisline += '"/>'
            h.write(thisline)
            h.write('\n')
            for m in range(0, n_car):
                thisline = '    <vehicle depart="0" id="'
                vehId = trip_names[i] + str(j) + 'veh' + str(m)
                vehRous[vehId] = thisRou
                thisline += vehId + '" '
                thisline += 'route="' + trip_names[i] + str(j) + '" type="CarA" />'
                h.write(thisline)
                h.write('\n')
    h.write(h_end)
    h.close()
    return vehRous

###generate the starting and ending location for the new car
def newPosition(grid):
    xpos = list(range(0, grid+1))
    random.seed(randSeed)
    random.shuffle(xpos)
    ypos = list(range(0, grid+1))
    random.seed(randSeed+2)
    random.shuffle(ypos)
    return (xpos[0], ypos[0]), (xpos[1], ypos[1]), xpos[1] - xpos[0], ypos[1] - ypos[0]

###generate all the possible routes for the new car. The idea is to generate all possible binary sequences
###for given fixed number of zeros and ones. Then zeros correspond to the x-axis moves, ones correspond to the y-axis moves
def allBinarySquence(zeros, ones):
    total = zeros + ones  #total number of digits
    S = list(range(0,total))
    tmp = list(itertools.combinations(S, zeros))
    res = []   #return a list of strings
    for elem in tmp:
        tmp = ""
        for pos in range(0,total):
            if(pos in elem):
                tmp = tmp + "0"
            else:
                tmp = tmp + "1"
        res.append(tmp)
    return res

###read data from the output file
def readFile(filename, unaffectedVeh):
    f = open(filename, 'r')
    finishTime = {}   #records the finish time for all vehicles, key is vehicle name, value is finish time
    vehNums = {}      #number of vehicles on a specific edge at a specific time
                  #key is a tuple with time and edge name, value is a number
    startTime = {}    #records the time for every car to be on the road
    crushEdge = {}  #records the affected car edges at the time of crush
    newTime = {}    #records the time when the cars arrive at a new node right after the crush
                    #this is the last time the car in the road where the crush happens
    for line in f:
        linelist = line.split()
        if(len(linelist) > 1 and linelist[0] == '<timestep'):
            endPos = 6
            while(True):
                if(linelist[1][endPos] == '.'):
                    break
                else:
                    endPos += 1
            curTime = int(linelist[1][6:endPos])
        elif(len(linelist) > 1 and linelist[0] == '<edge'):
            endPos = 4
            while(True):
                if(linelist[1][endPos] == '"'):
                    break
                else:
                    endPos += 1
            curEdge = linelist[1][4:endPos]
        elif(len(linelist) > 1 and linelist[0] == '<vehicle'):
            endPos = 4
            while(True):
                if(linelist[1][endPos] == '"'):
                    break
                else:
                    endPos += 1
            curVeh = linelist[1][4:endPos]
            #if the car is right at a cross, do something
            if("_" in curEdge):
                curNode = curEdge.split('_')[0][1:]
                edgeList = vehRous[curVeh].split()
                for e in edgeList:
                    if(curNode == e[0:2]):
                        curEdge = e
                        break
            #after the crush, only record the unaffected cars
            if(curTime >= crushTime and curVeh in unaffectedVeh):
                curKey = (curTime, curEdge)
                if(curKey in vehNums):
                    vehNums[curKey] += 1
                else:
                    vehNums[curKey] = 1
            #remember the cars locations at the time of the crush
            if(curTime == crushTime):
                crushEdge[curVeh] = curEdge
                newTime[curVeh] = curTime
            #update
            if(curVeh in crushEdge and crushEdge[curVeh] == curEdge):
                newTime[curVeh] = curTime
            finishTime[curVeh] = curTime + 1
            if(curVeh not in startTime):
                startTime[curVeh] = curTime
    f.close()
    return vehNums, finishTime, startTime, crushEdge, newTime

###generate the route file with the new car
def writeNewRouFile(filename, newRoutes):
    newRoutes_Ordered = collections.OrderedDict(sorted(newRoutes.items(), key = lambda t:t[0]))
    h = open(filename, 'w')
    h.write(h_start)
    h.write('\n')
    #car type
    thisline = '    <vType accel="' + str(acc) + '" decel="' + str(decel) + '" id="CarA" length="5.0" minGap="2.5" maxSpeed="'
    thisline += str(speed) + '"  sigma="0.5" />'
    h.write(thisline)
    h.write('\n')
    addedRoute = []
    for veh in newRoutes_Ordered:
        thisRoute = newRoutes_Ordered[veh]
        if(thisRoute not in addedRoute):
            thisline = '    <route id="' + thisRoute + '" edges="' + thisRoute + '"/>'
            addedRoute.append(thisRoute)
            h.write(thisline)
            h.write('\n')
        thisline = '    <vehicle depart="0" id="' + veh + '" '
        thisline += 'route="' + thisRoute + '" type="CarA" />'
        h.write(thisline)
        h.write('\n')
    h.write(h_end)
    h.close()

###for any binary sequence, generate the corresponding route, update the vehNums along the route
###return the new route, a copy of the updated vehNums
def route_and_vehNums(route_binary, vehNums, cur_x, cur_y, cur_Time, edgeLen):
    curNewRoute = ""
    vehNums_copy = copy.deepcopy(vehNums)
    for elem in route_binary:
        if(elem == '0'):
            next_x = cur_x + x_step
            next_y = cur_y
        else:
            next_x = cur_x
            next_y = cur_y + y_step
        curEdge = str(cur_x)+str(cur_y) + 'to' + str(next_x) + str(next_y)
        curNewRoute += curEdge + ' '
        curDist = edgeLen[curEdge]
        curKey = (cur_Time, curEdge)
        curOffset = cur_Time%94
        if(curKey in vehNums):
            curNum = vehNums[curKey]
        else:
            curNum = 0
        if(curNum > curDist * max_Car/100):
            curNum = curDist * max_Car/100
        timeSpent = costF[str((curDist, curOffset, curNum))]
        cur_x = next_x
        cur_y = next_y
        for timeSlot in range(0, timeSpent):
            modifyTime = cur_Time + timeSlot
            modifyKey = (modifyTime, curEdge)
            if(modifyKey in vehNums_copy):
                vehNums_copy[modifyKey] += 1
            else:
                vehNums_copy[modifyKey] = 1
        cur_Time += timeSpent
    return curNewRoute[:-1], vehNums_copy, cur_Time

###generate the new route file with the new car and run the simulation with the new car
def run_new_simu(curNewRoute, new_car_start_time):
    ##writing the new route file
    writeNewRouFile('newmaze.rou.xml', 'maze.rou.xml', curNewRoute[0:-1], str(new_car_start_time))
    ## call the command line to run sumo with newcar
    subprocess.call(['sumo64', '-a', 'maze.add.xml', '-c', 'newmaze.sumo.cfg'], shell=True)

###given the new vehNums_copy, estimate all unaffected cars total time and the sum
###the total time is calculated from the time of the crush
def estOtherTotal(theseVeh, edgeLen, vehNums_copy, newTime, newRoutes):
    timeEst = {}
    new_total_est = 0
    for everyV in theseVeh:
        nowTime = newTime[everyV]
        nowRoute = newRoutes[everyV]
        routeList = nowRoute.split()
        for edge in routeList:
            nowOffset = nowTime%94
            nowDist = edgeLen[edge]
            locations = edge.split("to")
            trafficLight = locations[1]
            nowKey = (nowTime, edge)
            if(nowKey in vehNums_copy):
                nowNum = vehNums_copy[nowKey]
            else:
                nowNum = 0
            nowNum = min(nowNum, int(nowDist/100*max_Car))
            timeCost = costF[str((nowDist, nowOffset, nowNum))]
            nowTime += timeCost
        timeEst[everyV] = nowTime
        new_total_est += timeEst[everyV]
    return new_total_est

###read data from the output file
def totalTime(filename):
    f = open(filename, 'r')
    finishTime = {}   #records the finish time for all vehicles, key is vehicle name, value is finish time
    for line in f:
        linelist = line.split()
        if(len(linelist) > 1 and linelist[0] == '<timestep'):
            endPos = 6
            while(True):
                if(linelist[1][endPos] == '.'):
                    break
                else:
                    endPos += 1
            curTime = int(linelist[1][6:endPos])
        elif(len(linelist) > 1 and linelist[0] == '<edge'):
            endPos = 4
            while(True):
                if(linelist[1][endPos] == '"'):
                    break
                else:
                    endPos += 1
            curEdge = linelist[1][4:endPos]
        elif(len(linelist) > 1 and linelist[0] == '<vehicle'):
            endPos = 4
            while(True):
                if(linelist[1][endPos] == '"'):
                    break
                else:
                    endPos += 1
            curVeh = linelist[1][4:endPos]
            finishTime[curVeh] = curTime + 1
    f.close()
    timeSpent = 0
    for veh in finishTime:
        timeSpent += finishTime[veh]
    return timeSpent

######################################################################
#Generate the grid points
######################################################################
x_grid = []
y_grid = []

#randomly generate the grid
random.seed(randSeed)
x_grid.append(random.randint(5,10)*gridsize)
random.seed(randSeed)
y_grid.append(random.randint(5,10)*gridsize)

for i in range(1, grid):
    random.seed(randSeed+i)
    x_grid.append(x_grid[-1] + random.randint(5,10)*gridsize)
    random.seed(randSeed-i)
    y_grid.append(y_grid[-1] + random.randint(5,10)*gridsize)

##generate the dictionary to store all the lenghts of the edges
edgeLen = {}
for i in range(0, grid+1):
    for j in range(0, grid+1):
        if(i<grid):
            if(i==0):
                dist_tmp = x_grid[0]
            else:
                dist_tmp = x_grid[i] - x_grid[i-1]
            edgeLen[str(i)+str(j) + 'to' + str(i+1) + str(j)] = dist_tmp
            edgeLen[str(i+1)+str(j) + 'to' + str(i) + str(j)] = dist_tmp
        if(j<grid):
            if(j==0):
                dist_tmp = y_grid[0]
            else:
                dist_tmp = y_grid[j] - y_grid[j-1]
            edgeLen[str(i)+str(j) + 'to' + str(i) + str(j+1)] = dist_tmp
            edgeLen[str(i)+str(j+1) + 'to' + str(i) + str(j)] = dist_tmp

######################################################################
#trips, tripnames
######################################################################
trips = []
trip_names = []
veh_names = []
#four start-to-end trips, each has n_route many routes
trips.append('00TO'+ str(grid) + str(grid))             #lowerleft_to_upperright
trip_names.append("lowerleft_to_upperright")
trips.append(str(grid) + '0'+ 'TO' + '0' + str(grid))   #lowerright_to_upperleft
trip_names.append("lowerright_to_upperleft")
trips.append('0' + str(grid) + 'TO' + str(grid) + '0')  #upperleft_to_lowerright
trip_names.append("upperleft_to_lowerright")
trips.append(str(grid) + str(grid) + 'TO00')            #upperright_to_lowerleft
trip_names.append("upperright_to_lowerleft")

######################################################################
#Generate all the necessary files
######################################################################
# writing the nod file
writeNodeFile('maze.nod.xml', x_grid, y_grid)

#writing the edge file
writeEdgeFile('maze.edg.xml', grid)

#writing the additional file regarding the traffic lights
writeAddFile('maze.add.xml', grid)

#writing the route file, vehRous returns the routes for all the cars
vehRous_raw = writeRouFile('maze.rou.xml')

vehRous = collections.OrderedDict(sorted(vehRous_raw.items(), key = lambda t:t[0]))

#find the affected vehicles and unaffected vehicles
affectedVeh = []
unaffectedVeh = []

for veh in vehRous:
    if(badRoad1 in vehRous[veh] or badRoad2 in vehRous[veh]):
        affectedVeh.append(veh)
    else:
        unaffectedVeh.append(veh)

######################################################################
#Run the simulation without the new car
######################################################################
# call the command line to generate the maze.net.xml file
subprocess.call(['netconvert64', '-c', 'maze.netc.cfg'], shell=True)

# call the command line to run sumo
subprocess.call(['sumo64', '-a', 'maze.add.xml', '-c', 'maze.sumo.cfg'], shell=True)

#read data from the output file
vehNums, finishTime , startTime, crushEdge, newTime = readFile('maze.output.xml', unaffectedVeh)

#record the new routes for the unaffected vehicles for later calculation
newRoutes = {}
for allVeh in unaffectedVeh:
    thisRoute = vehRous[allVeh]
    newRoutes[allVeh] = thisRoute.split(crushEdge[allVeh])[1][1:]

#compute the total time without the new car
old_total_simu = 0
for ve in finishTime:
    old_total_simu += finishTime[ve]

print("If no accident: ", old_total_simu, '#########')

######################################################################
#here we are going to recompute the routes for the affected cars one by one
#compare two ways,
##A: after computing the new route for an affected car, we do not update the traffic info
##B: to update the traffic info according to this new car's route
######################################################################
vehNums_A = copy.deepcopy(vehNums)
newRoutes_A = copy.deepcopy(newRoutes)

vehNums_B = copy.deepcopy(vehNums)
newRoutes_B = copy.deepcopy(newRoutes)
unaffectedVeh_B = copy.deepcopy(unaffectedVeh)

for anyVeh in affectedVeh:
    curEdge = crushEdge[anyVeh]
    newStart = curEdge.split("to")[-1]
    new_car_start_pos = [int(newStart[0]), int(newStart[1])]
    newEnd = vehRous[anyVeh].split()[-1].split("to")[-1]
    new_car_end_pos = [int(newEnd[0]), int(newEnd[1])]
    new_car_start_time = newTime[anyVeh]
    x_move = int(newEnd[0]) - int(newStart[0])
    y_move = int(newEnd[1]) - int(newStart[1])
    x_step = int(abs(x_move)/x_move)
    y_step = int(abs(y_move)/y_move)
    #print("New car starts at time: ", new_car_start_time)
    #print("New car start location: ", new_car_start_pos)
    #print("New car end location: ", new_car_end_pos)
    #generate all binary sequence, each corresponds to route for the new car
    route_sequence = allBinarySquence(abs(x_move), abs(y_move))
    #find the optimal route for this particular vehicle
    optimal_grandTotal_est_B = 10**6
    optimal_route_est_B = ""
    optimal_grandTotal_est_A = 10**6
    optimal_route_est_A = ""
    #in method B we will update the vehNums according to the optimal routes, now keep a copy
    vehNums_B_tmp = {}
    #for this vehicle, choose the best route
    for everyRoute in route_sequence:
        cur_x = new_car_start_pos[0]
        cur_y = new_car_start_pos[1]
        #method B: we will update the traffic info after we choose the optimal route for the first affected car
        curNewRoute_B, vehNums_copy, cur_Time_B = route_and_vehNums(everyRoute, vehNums_B, cur_x, cur_y, new_car_start_time, edgeLen)
        if(badRoad1 in curNewRoute_B or badRoad2 in curNewRoute_B):
            continue
        new_car_travel_time_est_B = cur_Time_B - new_car_start_time
        ##given the new vehNums, estimate all other cars total time
        new_total_est_B = estOtherTotal(unaffectedVeh_B, edgeLen, vehNums_copy, newTime, newRoutes_B)
        new_grandTotal_est_B = new_car_travel_time_est_B + new_total_est_B
        #update the optimal time if necessary (estimation)
        if(optimal_grandTotal_est_B > new_grandTotal_est_B):
            optimal_grandTotal_est_B = new_grandTotal_est_B
            optimal_route_est_B = curNewRoute_B
            vehNums_B_tmp = copy.deepcopy(vehNums_copy)

        #method A: we will not update the traffic info after we choose the optimal route for the first affected car
        curNewRoute_A, vehNums_copy_A, cur_Time_A = route_and_vehNums(everyRoute,
                  vehNums_A, cur_x, cur_y, new_car_start_time, edgeLen)
        new_car_travel_time_est_A = cur_Time_A - new_car_start_time
        ##given the new vehNums, estimate all other cars total time
        new_total_est_A = estOtherTotal(unaffectedVeh, edgeLen, vehNums_copy_A, newTime, newRoutes)
        new_grandTotal_est_A = new_car_travel_time_est_A + new_total_est_A
        #update the optimal time if necessary (estimation)
        if(optimal_grandTotal_est_A > new_grandTotal_est_A):
            optimal_grandTotal_est_A = new_grandTotal_est_A
            optimal_route_est_A = curNewRoute_A
    #update the vehNums according to this optimal routes
    vehNums_B = copy.deepcopy(vehNums_B_tmp)
    #add this route to the new route, which will be used for other affected cars
    newRoutes_B[anyVeh] = optimal_route_est_B
    newRoutes_A[anyVeh] = optimal_route_est_A
    unaffectedVeh_B.append(anyVeh)

#update the final routes
for veh in affectedVeh:
    oldRoute = vehRous[veh]
    firstPart = oldRoute.split(crushEdge[veh])[0]
    newRoutes_B[veh] = firstPart + crushEdge[veh] + ' ' + newRoutes_B[veh]
    newRoutes_A[veh] = firstPart + crushEdge[veh] + ' ' + newRoutes_A[veh]

for veh in unaffectedVeh:
    newRoutes_B[veh] = vehRous[veh]
    newRoutes_A[veh] = vehRous[veh]

#from final routes to write the files smartly coordinated
writeNewRouFile('newmazeB.rou.xml', newRoutes_B)
## call the command line to run sumo with newcar
subprocess.call(['sumo64', '-a', 'maze.add.xml', '-c', 'newmazeB.sumo.cfg'], shell=True)

timeSpent_B = totalTime('newmazeB.output.xml')

print("If accident and smartly coordinated, total time spent: ", timeSpent_B, '#########')

#from final routes to write the files, not smartly coordinated
writeNewRouFile('newmazeA.rou.xml', newRoutes_A)
## call the command line to run sumo with newcar
subprocess.call(['sumo64', '-a', 'maze.add.xml', '-c', 'newmazeA.sumo.cfg'], shell=True)

timeSpent_A = totalTime('newmazeA.output.xml')

print("If accident and naively coordinated, total time spent: ", timeSpent_A, '#########')


# Import flask which will handle our communication with the frontend
# Also import few other libraries
import warnings
warnings.filterwarnings("ignore")
from flask import Flask, render_template, request
from scipy.misc import imread, imresize, imsave
import numpy as np
import re
import keras.models
import tensorflow as tf
from keras.models import model_from_json
import sys
import base64
import os

# Path to our saved model
sys.path.append(os.path.abspath("./model"))

from load import *
# Initialize flask app
app = Flask(__name__)

def init():
	json_file = open('C:/Users/visha/flask_deploy/model/model.json','r')
	loaded_model_json = json_file.read()
	json_file.close()
	loaded_model = model_from_json(loaded_model_json)
	#load weights into new model
	loaded_model.load_weights('C:/Users/visha/flask_deploy/model/model.h5')
	print("Loaded Model from disk")

	#compile and evaluate loaded model
	loaded_model.compile(loss = 'categorical_crossentropy', optimizer = 'adam',metrics = ['accuracy'])
	#loss,accuracy = model.evaluate(X_test,y_test)
	#print('loss:', loss)
	#print('accuracy:', accuracy)
	graph = tf.get_default_graph()

	return loaded_model,graph

#Initialize some global variables
global model, graph
model, graph = init()

def convertImage(imgData1):
  imgstr = re.search(r'base64,(.*)', str(imgData1)).group(1)
  with open('output.png', 'wb') as output:
    output.write(base64.b64decode(imgstr))

@app.route('/')
def index():
  return render_template("index.html")

@app.route('/predict/', methods=['GET', 'POST'])
def predict():
  # Predict method is called when we push the 'Predict' button
  # on the webpage. We will feed the user drawn image to the model
  # perform inference, and return the classification
  imgData = request.get_data()
  convertImage(imgData)
  # read the image into memory
  x = imread('output.png', mode = 'L')
  # make it the right size
  x = imresize(x, (28, 28))/255
  #You can save the image (optional) to view later
  imsave('final_image.jpg', x)
  x = x.reshape(1, 28, 28, 1)

  with graph.as_default():
    out = model.predict(x)
    response = np.argmax(out, axis = 1)
    return str(response[0])

if __name__ == "__main__":
	# run the app locally on the given port
	app.run(host = '0.0.0.0', port = 5000)


# first thing we're going to import is the patch
# function from the unit tests.mock module.
# This is going to allow us to mock the
# behavior of the Django get database function.

from unittest.mock import patch

from django.core.management import call_command
from django.db.utils import OperationalError
# when db is not available
from django.test import TestCase

class CommandsTestCase(TestCase):

    def test_wait_for_db_ready(self):
        """Test waiting for db when db is available"""

        # we're gonna mock the behavior
        with patch('django.db.utils.ConnectionHandler.__getitem__') as gi:
            gi.return_value = True
            # return this value that we specify here and the second thing
            # is it allows us to monitor how many times it was called and
            # the different calls that were made to it
            call_command('wait_for_db')
            # wait_for_db is the name of management command
            self.assertEqual(gi.call_count, 1)

    @patch('time.sleep', return_value=True)
    # mock the time.sleep, speed up the test
    def test_wait_for_db(self, ts):
        """Test waiting for db"""

        with patch('django.db.utils.ConnectionHandler.__getitem__') as gi:
            gi.side_effect = [OperationalError] * 5 + [True]
            #  the first five times you call this get item it's going to
            #  raise the operational error.
            # then on the sixth time it won't raise the error it will
            # just return.
            call_command('wait_for_db')
            self.assertEqual(gi.call_count, 6)
            # call this function for 6 times


#!/usr/bin/env python3

n = int(input())
xs = [int(x) - 1 for x in input().strip().split()]

cnt = 0
spot = -1

while True:
    corr = True
    for i in range(n):
        if xs[i] != i:
            corr = False
            break
    if corr:
        break
    cnt += 1
    if spot != -1 and xs[spot] == -1:
        xs[spot] = spot
        spot = -1
        continue
    prio = -1
    for i in range(n):
        if xs[i] != -1 and xs[xs[i]] == -1:
            prio = i
    if prio != -1:
        xs[xs[prio]] = xs[prio]
        xs[prio] = -1
        continue
    assert(spot == -1)
    for i in range(n):
        if xs[i] == i:
            continue
        spot = xs[i]
        xs[i] = -1
        break

print(cnt)


import os
import subprocess
from .global_config import *

class YoloException(Exception):
    pass

class YoloDetector():
    BOUNDS = []
    x0 = y0 = x1 = y1 = 0
    CONTAIN_ICON = False
    CONTAIN_ICON_TYPE = []
    CONTAIN_ICONS_INFO = []  #[['info', 0, 0 , 1, 1], ['close', 0, 0, 1, 1]]
    # Because one screenshot may have several icons with the same type

    def __init__(self, pic, bounds):
        self.SCREENSHOT_PATH = pic
        self.filename = os.path.split(pic)[-1]
        self.Prediction_dir = Prediction_dir
        if not os.path.exists(self.Prediction_dir):
            os.mkdir(self.Prediction_dir)
        self.tmp_txt_path = self.SCREENSHOT_PATH + ".txt"
        self.txt_path = os.path.join(self.Prediction_dir, self.filename + ".txt")
        self.after_pre = os.path.join(self.Prediction_dir, self.filename)

        self.BOUNDS = bounds
        self.x0 = bounds[0][0]
        self.y0 = bounds[0][1]
        self.x1 = bounds[1][0]
        self.y1 = bounds[1][1]

    def analyze_result(self, txt_path):
        with open(txt_path, "r") as f:
            lines = f.readlines()
            for line in lines:
                if line.strip():
                    self.CONTAIN_ICON = True
                    row = line.split(" ")
                    class_name = row[-1]
                    if class_name not in self.CONTAIN_ICON_TYPE:
                        self.CONTAIN_ICON_TYPE.append(class_name)
                    self.CONTAIN_ICONS_INFO.append(row)

    def start(self):
        retval = os.getcwd()
        os.chdir(DARKNET_DIR)
        if not self.filename.endswith(".jpg"):
            return

        """
        ./darknet detect yolov3-voc-test.cfg yolov3-voc_1000.weights $file -out $savename
        """
        try:
            cmd = "./darknet detect yolov3-voc-test.cfg yolov3-voc_1000.weights "\
                + self.SCREENSHOT_PATH + " -out " + self.after_pre
            out_bytes = subprocess.check_output(cmd, shell=True)
        except subprocess.CalledProcessError as e:
            out_bytes = e.output  # Output generated before error
            code = e.returncode  # Return code
        try:
            cmd2 = "mv " + self.tmp_txt_path + " " + self.txt_path
            out_bytes2 = subprocess.check_output(cmd2, shell=True)
        except subprocess.CalledProcessError as e:
            out_bytes2 = e.output  # Output generated before error
            code = e.returncode  # Return code

        self.analyze_result(self.txt_path)
        os.chdir(retval)
        return self.CONTAIN_ICON