import logging
import colorlog
import argparse
import os
import sys
import dns.resolver
from multiprocessing import Process,Queue
import multiprocessing
import gevent

'''
set log this code is Useless
log.debug  is white ,info is green ,warn is yellow ,error is red ,critical  red!
'''
handler = colorlog.StreamHandler()
formatter = colorlog.ColoredFormatter(
    '%(log_color)s%(asctime)s [%(name)s] [%(levelname)s] %(message)s%(reset)s',
    datefmt=None,
    reset=True,
    log_colors={
        'DEBUG': 'cyan',
        'INFO': 'green',
        'WARNING': 'yellow',
        'ERROR': 'red',
        'CRITICAL': 'red,bg_white',
    },
    secondary_log_colors={},
    style='%'
)
handler.setFormatter(formatter)
log = colorlog.getLogger('subdns')
log.addHandler(handler)
log.setLevel(logging.INFO)

subname_list = []   #子域名字典   #   统计域名解析ip次数  设置超过十次视为泛解析ip  遇到此ip   pass
dns_server = []   #  办公网不能用其他dns    dns测试  自定义dns后期加上

class Subdns:
    def __init__(self,count_sub,domain,ip_con,name_list,scan_total,domain_result,allip_dict):     #   ,count_sub
        self.domain = domain
        self.ip_con = ip_con
        self.resolver = dns.resolver.Resolver(configure=True)   #  改为False
        self.name_list = name_list
        self.count_sub = count_sub
        self.scan_total = scan_total
        self.resolver.lifetime = self.resolver.timeout = 1.0
        self.domain_result = domain_result
        self.allip_dict = allip_dict

    def search_domain(self,sub):  # sub 子域名前缀
        res = str(sub)+"."+self.domain
        self.count_sub.value = self.count_sub.value -1
        print_msg("remain "+str(self.count_sub.value)+"  | Found"+str(self.scan_total.value)+'\r')
        try:
            result = self.resolver.query(res)
            if result:
                for i in result:
                    i = str(i)
                    if i in self.allip_dict.keys():
                        if self.allip_dict[i] <self.ip_con:    #  改为config配置设置
                            self.allip_dict[i] += 1
                        else:
                            return False
                    else:
                        self.allip_dict[i] = 1
                    res = res+"\t"+i
            self.scan_total.value = self.scan_total.value + 1
            log.info(res)
            self.domain_result.append(res+'\n')
        except Exception as e:
            # log.warning(e)
            return False

    def run(self):
        tasks = []
        for sub in self.name_list:
            tasks.append(gevent.spawn(self.search_domain,sub ))
            gevent.joinall(tasks)

def print_msg(msg=None, left_align=True):
    if left_align:
        sys.stdout.write('\r' + msg )
    sys.stdout.flush()

def run_process(allip_dict,domain_result,count_sub,scan_total,domain,ip_con,name_list):
    s = Subdns(allip_dict=allip_dict,domain_result=domain_result,domain=domain,scan_total=scan_total,ip_con=ip_con,name_list=name_list,count_sub=count_sub)
    s.run()

def main():
    parser = argparse.ArgumentParser(description='to use get subnames of dns')
    parser.add_argument("-v", "--version", action='version', version=' 1.0')
    parser.add_argument("-d", "--dict",type=str,help='Specify a dictionary',default='test.txt')  #   更改为mini_names.txt
    parser.add_argument("-u", "--domain" ,type=str, help='Designated domain name')
    parser.add_argument("-s","--deep",type=int , help='Domain depth',default=2)
    parser.add_argument("-t","--thread", type=int , help='Number of processes',default=4)
    args = parser.parse_args()    #  域名  args.domain    字典  args.dict  深度 args.deep   进程数  args.thread

    if args.domain == None:
        log.error("Please input domain  such as python subdns.py -u baidu.com")
        sys.exit()
    subname = args.domain          #   scan  domain
    subname_dict = args.dict
    if os.name == 'nt':    #subname_dict   字典物理地址
        subname_dict = os.getcwd()+'\\dict\\'+subname_dict
        save_name = os.getcwd()+'\\output\\'+subname+'.txt'
    else:
        subname_dict = os.getcwd()+'/dict/'+subname_dict
        save_name = os.getcwd()+'/output/'+subname+'.txt'
    log.info("check  dict is "+subname_dict)
    with open(subname_dict,'r') as f:
        for  sub in f.readlines():
            sub = sub.replace("\n","")
            subname_list.append(sub)
    count_sub = len(subname_list)
    log.info("total "+str(count_sub)+"  will scan")
    log.info("Start the injection process....")
    flag = count_sub//args.thread - 1

    manager = multiprocessing.Manager()
    domain_result = manager.list()
    allip_dict = manager.dict()
    count_sub = manager.Value("i",count_sub)
    scan_total = manager.Value("i",0)

    s = Subdns(allip_dict=allip_dict,domain_result=domain_result,domain=subname,scan_total=scan_total,ip_con=5,name_list=subname_list,count_sub=count_sub)
    s.run()

    log.warning("Total  scan "+str(len(domain_result))+" subname")

    with open(save_name,'w') as sa:
        for z in domain_result:
            sa.write(z)

if __name__ == '__main__':
    main()


#import time
from multiprocessing import Process, Pipe, Event
from . import rq
from . import ur
from .sim.simenv import SimEnv

def create(use_sim, config):
    '''
    Creates the appropriate server for either sim or real (hardware) backing.
    '''
    server_type = SimRobotServer if use_sim else RealRobotServer
    return RemoteRobotProxy(server_type, config)

class InProcRobotServer():
    def __init__(self, config):
        self.activate_hand = config.get("hand.activate", True)

    def connect(self):
        self._arm_con.connect()
        self._hand_con.connect(self.activate_hand)
        self.arm = ur.ArmController(self._arm_con)
        self.hand = rq.HandController(self._hand_con)

    def disconnect(self):
        self._arm_con.disconnect()
        self._hand_con.disconnect()

    def update(self):
        pass

class RealRobotServer(InProcRobotServer):
    def __init__(self, config):
        super().__init__(config)
        self._arm_con = ur.Connection()
        self._hand_con = rq.Connection()

class SimRobotServer(InProcRobotServer):
    def __init__(self, config):
        super().__init__(config)
        self.env = SimEnv(config)
        self._arm_con = ur.SimConnection(self.env)
        self._hand_con = rq.SimConnection(self.env)

    def connect(self):
        self.env.connect()
        super().connect()

    def disconnect(self):
        super().disconnect()
        self.env.disconnect()

    def reset(self):
        self.env.reset()

    def update(self):
        self.env.update()

class RemoteRobotProxy():
    class PipeConnection:
        def __init__(self, pipe):
            self.pipe = pipe

        def execute(self, cmd, state):
            self.pipe.send_bytes(cmd.array)
            self.pipe.recv_bytes_into(state.array)

    def __init__(self, robot_type, config):
        self.__robot_type = robot_type
        self.__config = config

    def connect(self):
        hand_server, hand_client = Pipe(duplex=True)
        arm_server, arm_client = Pipe(duplex=True)
        self.__shutdown_event = Event()
        self.__reset_event = Event()
        self.__reset_event.set()
        self.__process = Process(target=server_loop,
                                 args=(arm_client,
                                       hand_client,
                                       self.__shutdown_event,
                                       self.__reset_event,
                                       self.__robot_type,
                                       self.__config))
        self.__process.start()
        self.arm = RemoteRobotProxy.PipeConnection(arm_server)
        self.hand = RemoteRobotProxy.PipeConnection(hand_server)
        return (self.arm, self.hand)

    def disconnect(self):
        self.__shutdown_event.set()
        self.__process.join()

    def reset(self):
        self.__reset_event.clear()
        self.__reset_event.wait()

#************************************************************************************************
# Server loop
#************************************************************************************************
def server_loop(arm_client, hand_client, shutdown_event, reset_event, robot_type, config):
    '''
    Control loop running at the same frequency as the hardware (e.g. 125Hz) (best effort but not guaranteed).
    It enables high(er)-speed closed-loop control using force and tactile sensing (but no vision).
    '''
    robot = robot_type(config)
    robot.connect()

    arm_cmd = ur.Command()
    arm_state = ur.State()
    hand_cmd = rq.Command()
    hand_state = rq.State()
    while not shutdown_event.is_set():
        #start_time = time.time()

        if not reset_event.is_set():
            # primarily in support of sim, this enables restoring the environment to an initial state
            robot.reset()
            reset_event.set()

        arm_cmd_is_new = arm_client.poll()
        if arm_cmd_is_new:
            arm_client.recv_bytes_into(arm_cmd.array) # blocking

        hand_cmd_is_new = hand_client.poll()
        if hand_cmd_is_new:
            hand_client.recv_bytes_into(hand_cmd.array) # blocking

        robot.hand.execute(hand_cmd, hand_state)
        robot.arm.execute(arm_cmd, arm_state)
        # robot.update()  # the hand (rq.sim_connection) already calls update()

        if arm_cmd_is_new:
            arm_client.send_bytes(arm_state.array)

        if hand_cmd_is_new:
            hand_client.send_bytes(hand_state.array)

        # if time.time()-start_time > 2*freq:
        #     print("Server loop lagging: " + str(time.time()-start_time))

    # Disconnect the arm and gripper.
    robot.disconnect()


from basicnn.knn import knn
from utils import load
import numpy as np

Xtr,Ytr,Xte,Yte = load.load()
knn =  knn()
knn.train(Xtr,Ytr,None,None,Xte,Yte)
labels = knn.predict(Xte[0:100,],5)
labelss = knn.predicts(Xte[0:100,])
print("{}%".format(np.mean(np.array(labels) == Yte[0:100])*100))
print("{}%".format(np.mean(labels == labelss)*100))


from __future__ import absolute_import

import collections

import threading
from typing import FrozenSet  # pylint: disable=unused-import
import time
import socket
import select
import struct

import six
from six.moves import queue  # type: ignore

from . import fsm
from . import pdu
from . import exceptions

PDU_TYPES = {
    0x01: (pdu.AAssociateRqPDU, fsm.Events.EVT_6),
    0x02: (pdu.AAssociateAcPDU, fsm.Events.EVT_3),
    0x03: (pdu.AAssociateRjPDU, fsm.Events.EVT_4),
    0x04: (pdu.PDataTfPDU, fsm.Events.EVT_10),
    0x05: (pdu.AReleaseRqPDU, fsm.Events.EVT_12),
    0x06: (pdu.AReleaseRpPDU, fsm.Events.EVT_13),
    0x07: (pdu.AAbortPDU, fsm.Events.EVT_16)
}

PDU_TO_EVENT = {
    pdu.AAssociateRqPDU.pdu_type: fsm.Events.EVT_1,  # A-ASSOCIATE Request
    pdu.AAssociateAcPDU.pdu_type: fsm.Events.EVT_7,  # A-ASSOCIATE Response (accept)
    pdu.AAssociateRjPDU.pdu_type: fsm.Events.EVT_8,  # A-ASSOCIATE Response (reject)
    pdu.AReleaseRqPDU.pdu_type: fsm.Events.EVT_11,   # A-Release Request
    pdu.AReleaseRpPDU.pdu_type: fsm.Events.EVT_14,   # A-Release Response
    pdu.AAbortPDU.pdu_type: fsm.Events.EVT_15,
    pdu.PDataTfPDU.pdu_type: fsm.Events.EVT_9
}

class DULServiceProvider(threading.Thread):
    """Implements DUL service.

    This class is responsible for low-level operations with incoming and
    outgoing PDUs.

    Service can be initialized by providing open socket that service would
    use for sending and receiving PDUs. In case if socket is not provider
    service opens a client socket by itself when sending
    :class:`~pynetdicom2.pdu.AAssociateRqPDU` instance.

    Underlying implementation relies on state machine that is defined in :doc:`fsm`

    :ivar primitive: current PDU
    :ivar dimse_gen: generator, used break current outgoing DIMSE message into P-DATA-TF PDUs
    :ivar event: current event
    :ivar max_pdu_length: maximum PDU length for incoming P-DATA-TF PDUs
    :ivar to_service_user: outgoing data queue
    :ivar from_service_user: incoming data queue
    :ivar dul_socket: socket, that service uses
    :ivar is_killed: DUL service termination flag
    """

    def __init__(
            self,
            store_in_file,  # type: FrozenSet[str]
            get_file_cb,
            dul_socket=None,  # type: socket.socket
            max_pdu_length=65536  # type: int
        ):
        """Initializes DUL service.

        If no socket is provided service will act as 'client' and will open
        new client socket when sending :class:`~pynetdicom2.pdu.AAssociateRqPDU`
        instance.

        :param store_in_file: set of SOP Class UIDs, for which incoming dataset should be stored
                              in a file.
        :param get_file_cb: callback for getting a file to store incoming dataset
        :param dul_socket: remote client socket that will be used to send and
                           receive PDUs.
        """
        super(DULServiceProvider, self).__init__()

        self.primitive = None  # current pdu
        self.dimse_gen = None
        self.event = collections.deque()
        self.max_pdu_length = max_pdu_length

        self.to_service_user = queue.Queue()
        self.from_service_user = queue.Queue()

        # Setup the timer and finite state machines
        self.timer = Timer(10)
        self.state_machine = fsm.StateMachine(self, self.timer, store_in_file, get_file_cb)
        self._is_killed = threading.Event()

        if dul_socket:  # A client socket has been given. Generate an event 5
            self.event.append(fsm.Events.EVT_5)

        self.dul_socket = dul_socket
        self.raw_pdu = b''

        self.is_killed = False
        self.start()

    @property
    def accepted_contexts(self):
        """Accepted presentation contexts in the current association"""
        return self.state_machine.accepted_contexts

    @accepted_contexts.setter
    def accepted_contexts(self, value):
        self.state_machine.accepted_contexts = value

    def send(self, primitive):
        """Puts PDU into outgoing queue.

        .. note::

            PDU is not immediately written into the socket, but rather put into
            queue that is processed by the service event loop.

        :param primitive: outgoing PDU. Possible PDU types are described in :doc:`pdu`
        """
        self.from_service_user.put(primitive)

    def receive(self, timeout):
        """Tries to get PDU from incoming queue.

        If timeout is exceeded method
        rises :class:`~pynetdicom2.exceptions.DCMTimeoutError` exception.

        :param timeout: the amount of seconds method waits for PDU to appear in incoming queue
        :return: PDU instance or a tuple containing DIMSE Message and Presentation Context ID.
                 Possible PDU types are described in :doc:`pdu`. Possible DIMSE messages are
                 described in :doc:`dimsemessages`.
        :raise exceptions.DCMTimeoutError: If specified timeout is exceeded
        """
        try:
            return self.to_service_user.get(timeout=timeout)
        except queue.Empty:
            raise exceptions.DCMTimeoutError()

    def stop(self):
        """Tries to stop service for idle association.

        If association is not in idle state, method will return ``False`` and
        association will not be stopped.

        :return: ``True`` if service termination flag was successfully set
                 (current association state was 'idle'), ``False`` otherwise
        """
        if self.state_machine.current_state == fsm.States.STA_1:
            self.is_killed = True
            return True
        return False

    def kill(self):
        """Sets termination flag for event loop and waits for thread to exit."""
        self.is_killed = True
        self._is_killed.wait()

    def run(self):
        try:
            while not self.is_killed:
                self._check_network() or self._check_outgoing_pdu() or self._check_timer()  # pylint: disable=expression-not-assigned
                try:
                    evt = self.event.popleft()
                except IndexError:
                    continue
                self.state_machine.action(evt)
        except Exception:
            self.to_service_user.put(pdu.AAbortPDU(source=0, reason_diag=0))
            raise
        finally:
            self._is_killed.set()

    def _check_network(self):
        if self.state_machine.current_state == fsm.States.STA_13:
            return self._close()

        if not self.dul_socket:
            return False

        if self.state_machine.current_state == fsm.States.STA_4:
            self.event.append(fsm.Events.EVT_2)
            return True

        if self.raw_pdu and self._process_incoming():
            return True

        # check if something comes in the client socket
        if select.select([self.dul_socket], [], [], 0.05)[0]:
            if self._check_incoming_pdu():
                return True

        return self._process_incoming()

    def _check_outgoing_pdu(self):
        try:
            if self.dimse_gen:
                try:
                    self.primitive = next(self.dimse_gen)
                    self.event.append(PDU_TO_EVENT[self.primitive.pdu_type])
                    return True
                except StopIteration:
                    self.dimse_gen = None
            incoming = self.from_service_user.get(False, None)
            if hasattr(incoming, 'pdu_type'):
                self.primitive = incoming
            else:
                self.dimse_gen = incoming
                self.primitive = next(self.dimse_gen)
            self.event.append(PDU_TO_EVENT[self.primitive.pdu_type])
            return True
        except KeyError:
            raise exceptions.PDUProcessingError(
                'Unknown PDU {0} with type {1}'.format(self.primitive, self.primitive.pdu_type)
            )
        except queue.Empty:
            return False

    def _check_timer(self):
        if self.timer.check() is False:
            self.event.append(fsm.Events.EVT_18)  # Timer expired
            return True
        return False

    def _check_incoming_pdu(self):
        # type: () -> bool
        # There is something to read
        try:
            data = self.dul_socket.recv(self.max_pdu_length)
        except socket.error:
            self.event.append(fsm.Events.EVT_17)
            self.dul_socket.close()
            self.dul_socket = None
            return True

        if not data:
            # Remote port has been closed
            self.event.append(fsm.Events.EVT_17)
            self.dul_socket.close()
            self.dul_socket = None
            return True

        self.raw_pdu += data
        return False

    def _process_incoming(self):
        if len(self.raw_pdu) < 6:
            return False

        length = self.raw_pdu[2:6]
        length = struct.unpack('>L', length)[0]
        full_length = length + 6
        if len(self.raw_pdu) < full_length:
            return False

        raw_pdu = self.raw_pdu[:full_length]
        self.raw_pdu = self.raw_pdu[full_length:]

        # Determine the type of PDU coming on remote port and set the event accordingly
        try:
            pdu_type, event = PDU_TYPES[six.indexbytes(raw_pdu, 0)]
            self.primitive = pdu_type.decode(raw_pdu)
            self.event.append(event)
        except KeyError:
            self.event.append(fsm.Events.EVT_19)
        return True

    def _close(self):
        # waiting for connection to close
        if self.dul_socket is None:
            return False

        # wait for remote connection to close
        try:
            while self.dul_socket.recv(1) != b'':
                continue
        except socket.error:
            return False

        self.dul_socket.close()
        self.dul_socket = None
        self.event.append(fsm.Events.EVT_17)
        return True

class Timer(object):
    """A small helper timer class"""

    def __init__(self, max_seconds):
        # type: (int) -> None
        self._max_seconds = max_seconds
        self._start_time = None

    def start(self):
        """Sets a timer"""
        self._start_time = time.time()

    def stop(self):
        """Stops a timer"""
        self._start_time = None

    def restart(self):
        """Restarts a timer"""
        self.stop()
        self.start()

    def check(self):
        # type: () -> bool
        """Checks if timer has expired"""
        if self._start_time and (time.time() - self._start_time > self._max_seconds):
            return False
        return True


import os
import time
import json
import logging
from multiprocessing import Pool
from preprocessors.utils import chunk_list, get_paths_of_root, get_paths_of_split
from preprocessors.nlp_pipeline import Pipeline

from preprocessors.base_processor import BaseProcessor
from preprocessors.cnndm_reader import Reader
from preprocessors.graph_builder import GraphBuilder

logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger(__name__)

dm_single_close_quote = u'\u2019'  # unicode
dm_double_close_quote = u'\u201d'
# acceptable ways to end a sentence
END_TOKENS = ['.', '!', '?', '...', "'", "`", '"', dm_single_close_quote, dm_double_close_quote, ")"]

class Processor(BaseProcessor):
    def __init__(self, config_path):
        super(Processor, self).__init__(config_path=config_path)
        self.cnn_story_root = os.path.join(self.root, self.configs["cnn_story_path"])
        self.dm_story_root = os.path.join(self.root, self.configs["dm_story_path"])

        self.split_root = self.configs["split_root"]
        self.train_path = os.path.join(self.split_root, self.configs["all_train_path"])
        self.val_path = os.path.join(self.split_root, self.configs["all_val_path"])
        self.test_path = os.path.join(self.split_root, self.configs["all_test_path"])

        self.max_n_sents = self.configs["max_n_sents"]
        self.max_n_words = self.configs["max_n_words"]

        self.reader = Reader()

    def process(self):
        logger.info("Start pre-processing")
        self._tokenize_data("cnn")
        self._tokenize_data("dm")
        self._split_data()
        self._build_vocabulary()
        self._extract_oracle()
        self._build_graphs()

    """
    Tokenize
    """

    def _tokenize_data(self, news="cnn"):
        logger.info("Tokenizing {} documents.".format(news))
        if news == "cnn":
            story_root = self.cnn_story_root
            story_file_list = os.listdir(self.cnn_story_root)
        if news == "dm":
            story_root = self.dm_story_root
            story_file_list = os.listdir(self.dm_story_root)

        logger.info("n_stories: {}".format(len(story_file_list)))
        self._mp_helper(story_root=story_root, file_list=story_file_list)

    def _mp_helper(self, story_root, file_list):
        file_list_chunks = chunk_list(list(file_list), self.n_process)
        pool = Pool(self.n_process)

        input_list = []
        for i, file_chunk in enumerate(file_list_chunks):
            input_list.append((i, story_root, file_chunk))
        pool.map(self._process_data, input_list)

        pool.close()
        pool.join()

    def _process_data(self, inputs):
        worker_id, story_root, story_file_list = inputs
        start_time = time.time()
        reader = Reader()
        pipeline = Pipeline(port=worker_id)
        for i, story_file in enumerate(story_file_list):
            if i % 1000 == 0:
                logger.info("Worker: {}, {} documents processed, {:.2f}s elapsed.".format(worker_id, i, time.time()-start_time))
            story_path = os.path.join(story_root, story_file)
            story = reader.read_story(story_path)
            story_tokenized = self._tokenize_story(story=story, pipeline=pipeline)
            if not story_tokenized:  # skip empty files
                continue
            body_sents, body_heads, summary_sents, summary_heads = story_tokenized

            data = {
                "document": body_sents,
                "document_heads": body_heads,
                "summary": summary_sents,
                "summary_heads": summary_heads,
                "hashid": story_file.split(".")[0],
            }

            path = os.path.join(self.save_file_root, data["hashid"] + ".json")
            with open(path, "w") as f:
                json.dump(data, f)
        pipeline.core_nlp.stop()

    def _tokenize_story(self, story, pipeline):
        ret = pipeline(story.strip().lower())

        if not ret:
            return False
        story_tokenized, heads = ret
        if story_tokenized[0][0] == "@highlight":
            return False

        body_sents = []
        body_heads = []
        summary_sents = []
        summary_heads = []
        summary_flag = False
        cnt_high = 0
        for i, sent in enumerate(story_tokenized):
            if sent[0] == "@highlight":
                summary_flag = True
                cnt_high += 1
            elif summary_flag:
                summary_sents.append(self._fix_missing_period(sent))
                summary_heads.append(heads[i])
            else:
                body_sents.append(self._fix_missing_period(sent))
                body_heads.append(heads[i])
        return body_sents, body_heads, summary_sents, summary_heads

    def _fix_missing_period(self, sent):
        """Adds a period to a line that is missing a period"""
        if sent[-1] in END_TOKENS:
            return sent
        return sent + ["."]

    """
    Split data
    """

    def _split_data(self):
        logger.info("Splitting data.")
        start_time = time.time()

        self._process_split_data(data_path=self.train_path, save_path=self.save_split_train)
        self._process_split_data(data_path=self.val_path, save_path=self.save_split_val)
        self._process_split_data(data_path=self.test_path, save_path=self.save_split_test)

        logger.info("Time elapsed: {:.2f}s.".format(time.time() - start_time))

    def _process_split_data(self, data_path, save_path):
        data_list = self.reader.get_split(path=data_path)

        with open(save_path, "w") as f:
            for line in data_list:
                f.writelines(line + ".json\n")

    """
    Build Tf-idf graph
    """

    def _build_graphs(self):
        self._update_stop_words()
        self.graph_builder = GraphBuilder(
            stop_word_list=self.stop_words_list,
            min_df=self.configs["min_df"],
            n_process=self.n_process)
        self._create_graph()

    def _update_stop_words(self):
        for word in END_TOKENS:
            self.stop_words_list.append(word)
        self.stop_words_list = list(set(self.stop_words_list))   # remove redundant words

if __name__ == "__main__":
    processor = Processor(config_path="../configs/cnndm_configs.yml")
    processor.process()


import cv2
import numpy as np

def main():
    image = cv2.imread("../data/house.tiff", 1)

    blue, green, red = cv2.split(image)
    rows, columns, channels = image.shape

    output = np.empty((rows, columns * 3, 3), np.uint8)

    output[:, 0:columns] = cv2.merge([blue, blue, blue])
    output[:, columns:columns * 2] = cv2.merge([green, green, green])
    output[:, columns * 2:columns * 3] = cv2.merge([red, red, red])

    hsvimage = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    hue, satr, vlue = cv2.split(hsvimage)
    hsvoutput = np.concatenate((hue, satr, vlue), axis=1)

    cv2.imshow("Sample Image", image)
    cv2.imshow("Output Image", output)
    cv2.imshow("HSV Image", hsvoutput)

    cv2.waitKey(0)
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()


import os
import pandas as pd
import cv2
from PIL import Image, ImageOps, ImageEnhance
from utils.model import freeze_blocks_resnet, CNN
from utils.img_preprocess import img_process_transform, img_process_PIL

import xgboost as xgb
import pickle

import argparse

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms as transforms
import torchvision.models as models

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-g', '--gpu_id', type=str, default="0", help='GPU ID')
    # parser.add_argument('-p', '--img_path', type=str, default='', help='img_path')

    FLAG = parser.parse_args()

    predict_kidney(FLAG)

def predict_kidney(FLAG):
    os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = FLAG.gpu_id

    torch.cuda.set_device(0)
    # specify dtype
    use_cuda = torch.cuda.is_available()
    print("use GPU: {}".format(use_cuda))
    if use_cuda:
        dtype = torch.cuda.FloatTensor
    else:
        dtype = torch.FloatTensor
    print("numbers of GPU: {}".format(torch.cuda.device_count()))

    # normalization of image
    transformations = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))])

    while True:
        img_path = input('Input image path: ')
        try:
            img = img_process_PIL(img_path, None, transformations, False, (224, 224))
            img = Variable(img).type(dtype)
            img = img.unsqueeze(0)
        except:
            print('Unable to open image! Try again!')
            continue
        else:

            if 'cnn_models' not in locals():
                print("loading trained models...")
                # create models dict
                cnn_models = {}
                extract_models = {}
                # model directory
                model_dir = 'models/'
                # model ensemble
                for i in range(10):
                    # ResNet-101 in submitted paper
                    model = CNN(drop_rate=0.5).type(dtype)
                    net = torch.nn.DataParallel(model)
                    net.load_state_dict(torch.load(model_dir+ 'model_{}.pkl'.format(i)))

                    extract_CNN = nn.Sequential(*list(net.module.base_model.children())[:-1]).type(dtype)
                    extract_net = torch.nn.DataParallel(extract_CNN)
                    print("loaded model {}/10".format(i), end = '\r')
                    cnn_models["model_{}".format(i)] = net
                    extract_models["extract_model_{}".format(i)] = extract_net
                    del net
                print("loaded trained CNN models")
            else:
                print("CNN models already loaded")

            if 'xgb_models' not in locals():
                print("loading trained XGBoost models...")

                xgb_models = {}
                xgb_dir = 'models/'
                for i in range(10):
                    xgb_model = pickle.load(open(xgb_dir + 'xgb_{}.pickle.dat'.format(i), "rb"))
                    xgb_models["xgb_{}".format(i)] = xgb_model
                print("loaded trained XGBoost models")
            else:
                print("XGBoost models already loaded")

            egfr = predict_egfr(img, cnn_models)
            print("Predictred eGFR: {}".format(egfr))
            ckd_stage = predict_ckd(img, extract_models, xgb_models)
            print("Predicted CKD stage > II: {:.2f}%".format(ckd_stage*100))

def predict_egfr(input_img, cnn_models):
    pred = []

    print("predicting egfr...")
    for i in range(10):
        test_pred = cnn_models["model_{}".format(i)].eval()(input_img)
        test_pred = test_pred.data.cpu().numpy()
        pred.append(test_pred)
        print("prediction {}/10 completed".format(i+1), end="\r")

    ens_pred = sum(pred)/len(pred)

    return(ens_pred)

def predict_ckd(input_img, extract_models, xgb_models):
    pred = []

    print("predicting CKD stage...")
    for i in range(10):
        test_features = extract_models["extract_model_{}".format(i)].eval()(input_img)
        test_features = test_features.data.cpu().numpy()
        test_features = pd.DataFrame(test_features.reshape(-1, 2048))
        test_features.columns = [str(i) for i in range(2048)]
        test_features = test_features[xgb_models["xgb_{}".format(i)].feature_names]
        test_features = xgb.DMatrix(test_features)
        test_class = xgb_models["xgb_{}".format(i)].predict(test_features)
        pred.append(test_class)
        print("prediction {}/10 completed".format(i+1), end="\r")

    ens_pred = sum(pred)/len(pred)
    # ens_pred = ens_pred > 0.5

    return(ens_pred[0])

if __name__ == '__main__':
    main()


from PyQt5 import uic
from PyQt5.QtWidgets import QMainWindow, QTableWidgetItem, QAbstractItemView, QDialog

from ECommerce.Catalogo import Catalogo
from Gestione.Database import Database
from Gestione.GestioneSessioneMarket import GestioneSessioneMarket
from view.CarrelloView import CarrelloView
from view.DettagliProdottoView import DettagliProdottoView
from view.MsgBoxView import MsgBox
from view.RicercaProdottoClienteView import RicercaProdottoClienteView

class CatalogoView(QMainWindow):
    def __init__(self, cliente):
        # inizializzazione finestra
        super(CatalogoView, self).__init__()
        self.gestione_market = GestioneSessioneMarket()

        uic.loadUi('ui/catalogo_prodotti.ui', self)

        # il prodotto che l'utente selezionerà dal catalogo sarà qui
        self.selected_prod_id = None
        # il cliente che fa acquisti è qui
        self.cliente = cliente
        self.cliente.get_carrello_prodotti().carica_carrello(self.cliente.get_id())
        # imposto la selezione a intera riga e non a cella singola nella tabella del catalogo
        self.table_catalogo.setSelectionBehavior(QAbstractItemView.SelectRows)
        # disabilito la possibilità di modificare le celle della tabella liberamente
        self.table_catalogo.setEditTriggers(QAbstractItemView.NoEditTriggers)

        # evento selezione di una riga
        self.table_catalogo.cellClicked.connect(self.get_selected_prod)
        # eventi dei pulsanti
        self.btn_visualizza_dettagli.clicked.connect(self.get_dettagli_prod)
        self.btn_search_prod.clicked.connect(self.ricerca_prodotto)
        self.btn_open_carrello.clicked.connect(self.visualizza_carrello)
        self.btn_aggiungi_al_carrello.clicked.connect(self.aggiungi_al_carrello)
        self.btn_finalizza_ordine.clicked.connect(self.finalizza_ordine)

        # caricamento del catalogo
        self.catalogo = Catalogo()
        lista_prodotti = self.catalogo.get_lista_prodotti()
        # visualizzazione catalogo a schermo
        self.carica_catalogo(lista_prodotti)

    def carica_catalogo(self, lista_prodotti):
        # pulizia tabella (per aggiornamenti, ad esempio)
        self.table_catalogo.setRowCount(0)
        # ottengo il numero di righe che vanno inserite
        self.table_catalogo.setRowCount(len(lista_prodotti))
        # inserisco effettivamente le righe
        num_riga = 0
        for prod in lista_prodotti:
            # gli item della qtablewidget devono essere obbligatoriamente delle stringhe, quindi vado a castare
            # i valori numerici in stringhe per renderli visibili in tabella
            prod_id = QTableWidgetItem(str(prod.get_id()))
            prod_title = QTableWidgetItem(prod.get_titolo())
            prod_qta = QTableWidgetItem(str(prod.get_quantita()))
            prod_price = QTableWidgetItem(str(prod.get_prezzo()) + "€")

            self.table_catalogo.setItem(num_riga, 0, prod_id)
            self.table_catalogo.setItem(num_riga, 1, prod_title)
            self.table_catalogo.setItem(num_riga, 2, prod_qta)
            self.table_catalogo.setItem(num_riga, 3, prod_price)

            num_riga += 1

    def get_selected_prod(self):
        # ottengo l'id del prodotto selezionato
        selected_row = self.table_catalogo.currentRow()
        self.selected_prod_id = int(self.table_catalogo.item(selected_row, 0).text())

        # abilito i pulsanti disabilitati
        self.btn_visualizza_dettagli.setEnabled(True)
        self.btn_aggiungi_al_carrello.setEnabled(True)

    def get_dettagli_prod(self):
        prod = self.catalogo.ricerca_per_id(self.selected_prod_id)

        if prod is not None:
            prod_details_view = DettagliProdottoView(self, prod)
            prod_details_view.exec_()
        else:
            msg = MsgBox()
            msg.show_error_msg("Il prodotto non è stato trovato, probabilmente non è stato selezionato alcun prodotto.")

    def ricerca_prodotto(self):
        # ottenimento del nome
        self.search_dialog = RicercaProdottoClienteView(self)
        dialog_return = self.search_dialog.exec_()

        if dialog_return == QDialog.Accepted:
            nome = self.search_dialog.get_query()
            prod = self.catalogo.ricerca_per_nome(nome)

            if prod is not None:
                # faccio vedere i dettagli del prodotto trovato.
                prod_details_view = DettagliProdottoView(self, prod)
                prod_details_view.exec_()
                # il prodotto trovato viene impostato come selezionato.
                self.selected_prod_id = prod.get_id()
                # abilito i pulsanti disabilitati
                self.btn_visualizza_dettagli.setEnabled(True)
                self.btn_aggiungi_al_carrello.setEnabled(True)
            else:
                msg = MsgBox()
                msg.show_error_msg("Il prodotto non è stato trovato, probabilmente è stato inserito il nome di un "
                                   "prodotto inesistente.")

    def visualizza_carrello(self):
        self.carrello_view = CarrelloView(self.cliente, self)
        self.carrello_view.show()

    def aggiungi_al_carrello(self):
        # ottengo il prodotto dal catalogo
        prod = self.catalogo.ricerca_per_id(self.selected_prod_id)

        # controllo quantità prodotto
        if prod.get_quantita() <= 0:
            msg = MsgBox()
            msg.show_error_msg("Impossibile aggiungere il prodotto nel catalogo: il prodotto selezionato è esaurito.")
        else:
            self.cliente.get_carrello_prodotti().aggiungi(prod, self.cliente.get_id())

            prod.scala_quantita()
            db_con = Database("system.db")
            db_con.modifica_quantita_prodotto(prod)

            self.carica_catalogo(self.catalogo.get_lista_prodotti())

    def finalizza_ordine(self):
        if self.gestione_market.finalizza_acquisto(self.cliente.get_carrello_prodotti(), self.cliente.get_id()):
            msg = MsgBox()
            msg.show_info_msg("L'ordine è stato correttamente inviato.")
        else:
            msg = MsgBox()
            msg.show_error_msg("L'ordine è stato respinto: controlla: di avere almeno un prodotto nel carrello, "
                               "di avere saldo sufficiente e di non aver annullato l'ordine.")


import pandas as pd
from parser import selectToString
from datetime import datetime, date
import time

def resultsToDf(doc):

  page_df = pd.DataFrame()

  # subset the document to only <div id="results_column">

  doc = doc.find("div", {"id": "results_column"} )

  # iterate through the individual results, save to data frame

  for res in doc.find_all("div", class_="row"):

    # If H2 is empty (call to action, feature box), then skip

    if res.select("div.res a > h2") != []:

      # Job title is in the H2 under div with class res

      job_title = selectToString(res,"div.res a > h2")

      # Best place to get advertiser is from a strong element only visible on XS displays

      advertiser = selectToString(res,"span.visible-xs > strong")

      # Location is in a span with class loc

      loc = selectToString(res,"span.loc")

      # Salary was reviewed during production. It's apparently a free text field for the advertiser.
      # There is no consistency.
      # Sometimes it doesn't appear, sometimes it doesn't contain a number, sometimes it's an hourly rate, sometimes it's a range.
      # We could write rules to parse and handle salaries, but as long as it remains free-text, an edge case will occur.
      # Something like this would be best handled with a manual cleanup, so we'll skip it for this exercise.
      # We'll extract anyway, because it has an unusual selection path

      # Salary is in a span with a class matching the job's location
      salary = selectToString(res,"span." + loc)

      # Date posted is in span with class date-posted

      posted = selectToString(res,"span.date-posted")

      # Amend the string to remove 'Date posted '

      posted = posted.replace("Date posted: ","")

      # Format as a usable date

      posted = datetime.strptime(posted,"%d %B")

      # Year isn't published, so assume that if date is in the future then it was published last year
      # Subtract 1 from year value if month value is greater than today's month value

      if posted.strftime("%m") > date.today().strftime("%m"):
        posted = posted.strftime("%d-%m-" + str(int(date.today().strftime("%Y")) - 1) )
      else:
        posted = posted.strftime("%d-%m-" + str(int(date.today().strftime("%Y"))) )

      # Store values in a dict object

      i_dict = {"job":job_title,"by":advertiser,"loc":loc,"date":posted}

      # Convert i_dict to a pandas dataframe

      i_df = pd.DataFrame([i_dict])

      # Append to cumulative dataframe

      page_df = page_df.append(i_df)

    else:
      pass

  return page_df


from config import *
from player import *
from plataformas import *
from serra import *
import sys
import pygame

class Inicial:
    def tela_inicial(self):
        pygame.init()
        SCREEN = screen

        f1 = 20
        f2 = 20
        escolha = 1

        score_font = pygame.font.Font('assets/PressStart2P.ttf', 44)
        score_text = score_font.render('INICIO', True, "blue", SCREEN_COLOR)
        score_text_rect = score_text.get_rect()
        score_text_rect.center = (500, 50)

        f1_font = pygame.font.Font('assets/PressStart2P.ttf', 20)
        f1_text = f1_font.render('Fase 1', True, "blue", SCREEN_COLOR)
        f1_text_rect = f1_text.get_rect()
        f1_text_rect.center = (500, 400)

        f2_font = pygame.font.Font('assets/PressStart2P.ttf', 20)
        f2_text = f2_font.render('Fase 2', True, "blue", SCREEN_COLOR)
        f2_text_rect = f2_text.get_rect()
        f2_text_rect.center = (500, 500)

        while True:
            SCREEN.fill(SCREEN_COLOR)
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()

                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_w:
                        if escolha == 1:
                            escolha += 1
                        else:
                            escolha -= 1
                    if event.key == pygame.K_s:

                        if escolha == 2:
                            escolha -= 1
                        else:
                            escolha += 1
                    if event.key == pygame.K_SPACE:
                        if escolha == 1:
                            GR.fase1()
                        else:
                            GR2.fase2()

            screen.blit(score_text, score_text_rect)
            screen.blit(f1_text, f1_text_rect)
            screen.blit(f2_text, f2_text_rect)

            if escolha == 1:
                f1_text = f1_font.render('Fase 1', True, "green", SCREEN_COLOR)
            else:
                f1_text = f1_font.render('Fase 1', True, "blue", SCREEN_COLOR)

            if escolha == 2:
                f2_text = f2_font.render('Fase 2', True, "green", SCREEN_COLOR)
            else:
                f2_text = f2_font.render('Fase 2', True, "blue", SCREEN_COLOR)

            pygame.display.flip()
        pygame.quit()

INI = Inicial()

class Game_rodas2:
    def fase2(self):
        pygame.init()

        # desenhar tela (dentro de config)
        SCREEN = screen

        # informacões do player
        player = play(self)
        player_x = 10
        player_y = 570
        direita = False
        esquerda = False
        pulo = False

        pode_pular = True
        # so vai poder pular novamente quando o player_y for igual ao ponto_inicial
        ponto_inicial = 0

        player_y_pulo = player_y

        #gravidade
        caindo = True

        armadilha = pygame.image.load("img/armadilha.png")
        armadilha = pygame.transform.scale(armadilha, (200,90))
        armadilha_x = 400
        arm_direita= True

        while True:
            SCREEN.fill(SCREEN_COLOR)
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()

                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_d:
                        direita = True
                    if event.key == pygame.K_a:
                        esquerda = True
                    if event.key == pygame.K_w:
                        pulo = True
                        ponto_inicial = player_y

                if event.type == pygame.KEYUP:
                    if event.key == pygame.K_d:
                        direita = False
                    if event.key == pygame.K_a:
                        esquerda = False

            #desenhar player
            SCREEN.blit(player, (player_x, player_y))

            #desenhar plataformas
            plataforma = pygame.draw.rect(SCREEN, "green",(0,625,1000,20))

             #desenhar armadilha
            SCREEN.blit(armadilha, (armadilha_x, 550))

            # movimentacão do player
            if (direita is True):
                player_x += 1
            if (esquerda is True):
                player_x -= 1

            # fisica para cair
            if(caindo is True):
                player_y +=1
            else:
                plyaer_y = player_y

            # colisão com as plataformas
            if player_y >= 595:
                caindo = False
            else:
                caindo = True

            # condicão para o pulo
            if(pulo is True) and (pode_pular is True):
                caindo = False
                player_y_pulo = player_y_pulo
                player_y -=1
            else:
                player_y_pulo = player_y

            # voltar do pulo
            if player_y <= player_y_pulo - 120:
                pulo=False
                pode_pular = False

            if player_y == ponto_inicial:
                pode_pular = True

            # movimentp da armadilha
            if arm_direita is True:
                armadilha_x += 1
            elif arm_direita is False:
                armadilha_x -= 1
            # direcao da armadilha
            if armadilha_x >= 700:
                arm_direita = False
            elif armadilha_x <= 400:
                arm_direita = True

            # condicão para quando perder
            if(  armadilha_x - 10 <= player_x <= armadilha_x + 190 ) and (player_y > 590):
                player_x = 10

            pygame.display.flip()
        pygame.quit()
GR2 = Game_rodas2()

class Game_rodas:

    def fase1(self):
        pygame.init()

        # desenhar tela (dentro de config)
        SCREEN = screen

        # informacões do player
        player = play(self)
        player_x = 10
        player_y = 570
        direita = False
        esquerda = False
        pulo = False

        pode_pular = True
        # so vai poder pular novamente quando o player_y for igual ao ponto_inicial
        ponto_inicial = 0

        player_y_pulo = player_y

        #gravidade
        caindo = True

        # desenhar plataformas
        plataform1 = plat_m(self)
        plataform2 = plat_m(self)
        plataform3 = plat_m(self)
        plataform4 = plat_m(self)
        plataform5 = plat_m(self)
        plataform6 = plat_m(self)
        plataform7 = plat_m(self)

        while True:
            SCREEN.fill(SCREEN_COLOR)
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()

                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_d:
                        direita = True
                    if event.key == pygame.K_a:
                        esquerda = True
                    if event.key == pygame.K_w:
                        pulo = True
                        ponto_inicial = player_y

                if event.type == pygame.KEYUP:
                    if event.key == pygame.K_d:
                        direita = False
                    if event.key == pygame.K_a:
                        esquerda = False

            #desenhar player
            SCREEN.blit(player, (player_x, player_y))

            #desenhar plataformas
            SCREEN.blit(plataform1, (0, 600))
            SCREEN.blit(plataform2, (100, 500))
            SCREEN.blit(plataform3, (200, 400))
            SCREEN.blit(plataform4, (300, 300))
            SCREEN.blit(plataform5, (500, 300))
            SCREEN.blit(plataform6, (700, 300))
            SCREEN.blit(plataform7, (900, 300))

            #desenhar passagem
            passagem = pygame.draw.rect(SCREEN, "red", (940,266,30,50))

            # movimentacão do player
            if (direita is True):
                player_x += 1
            if (esquerda is True):
                player_x -= 1

            # fisica para cair
            if(caindo is True):
                player_y +=1
            else:
                plyaer_y = player_y

            # colisão com as plataformas
            caindo = coli_plat_f1(self, player_x, player_y)

            # condicão para o pulo
            if(pulo is True) and (pode_pular is True):
                caindo = False
                player_y_pulo = player_y_pulo
                player_y -=1
            else:
                player_y_pulo = player_y

            # voltar do pulo
            if player_y <= player_y_pulo - 120:
                pulo=False
                pode_pular = False

            if player_y == ponto_inicial:
                pode_pular = True

            if  920<player_x<960:
                GR2.fase2()

            if player_y >= 700:
                player_x = 10
                player_y = 570

            pygame.display.flip()
        pygame.quit()


from flask import Flask, flash, redirect, render_template, request, session, url_for, send_file, jsonify, make_response
from flask_session import Session
from tempfile import mkdtemp
from bokeh.embed import components
from bokeh.plotting import figure, output_file, save
from graphIt import get_CGPA
from SGPA import get_SGPA
from subject import get_subject
import numpy as np
import json
from forms import FirstForm

app = Flask(__name__)

def apology(message, code=400):
	"""Render message as an apology to user."""
	def escape(s):
		"""
		Escape special characters.

		https://github.com/jacebrowning/memegen#special-characters
		"""
		for old, new in [("-", "--"), (" ", "-"), ("_", "__"), ("?", "~q"),
						 ("%", "~p"), ("#", "~h"), ("/", "~s"), ("\"", "''")]:
			s = s.replace(old, new)
		return s
	return render_template("apology.html", top=code, bottom=escape(message)), code

def getSGPA_allBranch(year = 16):
	val = get_SGPA(year)
	scripts = []
	divs = []
	k = 0
	for value in val:
		if k == len(val)-3:
			break
		p = figure(plot_width=400, plot_height=400, sizing_mode='scale_width')
		#p.y_range.end=60
		p.xaxis.axis_label='Semester '+str(int(k+1))
		temp_SGPA=[]
		temp_roll=[]
		temp_name=[]
		for idx,i in enumerate(val[len(val)-2]):
			temp_SGPA.append(value[idx])
			temp_roll.append(val[len(val)-2][idx])
			temp_name.append(val[len(val)-1][idx])
		if not temp_SGPA or len(temp_SGPA)==0:
			values = []
			return values, val
		m=max(temp_SGPA)
		t=[i for i, j in enumerate(temp_SGPA) if j == m]
		l=[i for i in range(0,len(temp_SGPA))]
		hist, edges = np.histogram(temp_SGPA, density=True, bins=10, weights=l)
		hist=[int(i*100) for i in hist]
		p.quad(top=hist, bottom=0,
			   left=edges[:-1], right=edges[1:], fill_color="#036564", line_color="#033649")
		script, div = components(p)
		scripts.append(script)
		divs.append(div)
		k=k+1
	values = list(zip(scripts, divs))
	return values, val

def getCGPA_allBranch(year=16):
	val = get_CGPA(year)
	scripts = []
	divs = []
	k = 0
	for value in val:
		if k == len(val)-3:
			break
		p = figure(plot_width=400, plot_height=400, sizing_mode='scale_width')
		#p.y_range.end=60
		p.xaxis.axis_label='Semester '+str(int(k+1))
		temp_CGPA=[]
		temp_roll=[]
		temp_name=[]
		highs=[]
		for idx,i in enumerate(val[len(val)-2]):
			temp_CGPA.append(value[idx])
			temp_roll.append(val[len(val)-2][idx])
			temp_name.append(val[len(val)-1][idx])
		if not temp_CGPA or len(temp_CGPA)==0:
			values = []
			return values, val
		m=max(temp_CGPA)
		t=[i for i, j in enumerate(temp_CGPA) if j == m]
		for x in t:
			highs.append((temp_name[x],temp_roll[x],temp_CGPA[x]))
		l=[i for i in range(0,len(temp_CGPA))]
		hist, edges = np.histogram(temp_CGPA, density=True, bins=10, weights=l)
		hist=[int(i*100) for i in hist]
		p.quad(top=hist, bottom=0,
			   left=edges[:-1], right=edges[1:], fill_color="#036564", line_color="#033649")
		script, div = components(p)
		scripts.append(script)
		divs.append(div)
		k=k+1
	values = list(zip(scripts, divs))
	return values, val, highs

def getCGPA_branchwiseValues(year=16, branch = "EC"):
	val=get_CGPA(year)
	scripts=[]
	divs=[]
	k=0
	highs=[]
	print(len(val))
	for value in val:
		if k==len(val)-3:
			break
		p=figure(plot_width=400,plot_height=400,sizing_mode='scale_width')
		p.xaxis.axis_label='Semester '+str(k+1)
		temp_CGPA=[]
		temp_roll=[]
		temp_name=[]
		for idx,i in enumerate(val[len(val)-2]):
			if i[2:4]==branch:
				temp_CGPA.append(value[idx])
				temp_roll.append(val[len(val)-2][idx])
				temp_name.append(val[len(val)-1][idx])
		if not temp_CGPA or len(temp_CGPA)==0:
			values = []
			highs=[]
			return values, highs
		m=max(temp_CGPA)
		t=[i for i, j in enumerate(temp_CGPA) if j == m]
		for v in t:
			highs.append("Semester "+str(k+1)+": "+temp_name[v]+","+temp_roll[v]+","+str(temp_CGPA[v]))
		l=[i for i in range(0,len(temp_CGPA))]
		hist, edges = np.histogram(temp_CGPA, density=True, bins=10, weights=l)
		hist=[int(i*100) for i in hist]
		p.quad(top=hist, bottom=0,
			   left=edges[:-1], right=edges[1:], fill_color="#036564", line_color="#033649")
		script, div = components(p)
		scripts.append(script)
		divs.append(div)
		k=k+1
	values = list(zip(scripts, divs))
	return values, highs

def getSGPA_branchwiseValues(year=16, branch="EC"):
	val=get_SGPA(year)
	scripts=[]
	divs=[]
	k=0
	highs=[]
	for value in val:
		if k==len(val)-3:
			break
		p=figure(plot_width=400,plot_height=400,sizing_mode='scale_width')
		p.xaxis.axis_label='Semester '+str(k+1)
		temp_SGPA=[]
		temp_roll=[]
		temp_name=[]
		for idx,i in enumerate(val[len(val)-2]):
			if i[2:4]==branch:
				temp_SGPA.append(value[idx])
				temp_roll.append(val[len(val)-2][idx])
				temp_name.append(val[len(val)-1][idx])
		if not temp_SGPA or len(temp_SGPA)==0:
			values = []
			highs=[]
			return values, highs
		m=max(temp_SGPA)
		t=[i for i, j in enumerate(temp_SGPA) if j == m]
		for v in t:
			highs.append("Semester "+str(k+1)+": "+temp_name[v]+","+temp_roll[v]+","+str(temp_SGPA[v]))
		l=[i for i in range(0,len(temp_SGPA))]
		hist, edges = np.histogram(temp_SGPA, density=True, bins=10, weights=l)
		hist=[int(i*100) for i in hist]
		p.quad(top=hist, bottom=0,
			   left=edges[:-1], right=edges[1:], fill_color="#036564", line_color="#033649")
		script, div = components(p)
		scripts.append(script)
		divs.append(div)
		k=k+1
	values = list(zip(scripts, divs))
	return values, highs

# Ensure responses aren't cached
@app.after_request
def after_request(response):
	response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
	response.headers["Expires"] = 0
	response.headers["Pragma"] = "no-cache"
	return response

app.config["SESSION_FILE_DIR"] = mkdtemp()
app.config["SESSION_PERMANENT"] = False
app.config["SESSION_TYPE"] = "filesystem"
app.config['SECRET_KEY'] = 'you-will-never-guess'
Session(app)

with open('subject_list.json') as f:
	text=f.read()
	r=json.loads(text)
suggestions=r["values"]

@app.route("/", methods=['GET', 'POST'])
def index():
	form = FirstForm()
	if request.method == 'POST':
		if form.validate() == False:
			flash('All fields are necessary')
			return render_template('firstform.html', form=form)
		else:
			yearentry = form.year.data
			branchentry = form.branch.data
			formatentry = form.format.data
			if yearentry == "14" and branchentry =="EC":
				print("ECE 14");
				flash('ECE branch does not exist in year 14')
				return render_template('firstform.html',form = form)
			elif branchentry == "ALL":
				if formatentry == "C":
					values,val,highs = getCGPA_allBranch(yearentry)
					flash (' '.join(val[len(val)-3]))
					return render_template("cgpa.html", values=values)
				else:
					values,val = getSGPA_allBranch(yearentry)
					flash (' '.join(val[len(val)-3]))
					return render_template("sgpa.html",values=values)
			else:
				if formatentry == "S":
					values, highs = getSGPA_branchwiseValues(yearentry,branchentry)
					if not values or not highs:
						flash('Could not find any entry that met your specification')
						return render_template("sgpa.html", values=values)
					flash (' '.join(highs))
					return render_template("sgpa.html", values=values)
				else:
					values, highs = getCGPA_branchwiseValues(yearentry, branchentry)
					if not values or not highs:
						flash('Could not find any entry that met your specification')
						return render_template("cgpa.html", values=values)
					flash (' '.join(highs))
					return render_template("cgpa.html", values=values)
	elif request.method == 'GET':
		return render_template('firstform.html', form = form)

@app.route("/cgpa/branch",methods=["GET"])
def cgpa_branchwise(year=16, branch="EC"):
	if request.args.get('year'):
		year=request.args.get('year')
	if request.args.get('branch'):
		branch=request.args.get('branch')
	if int(year) not in [14,16,17]:
		return apology("Sorry, no data for that year");
	values, highs = getCGPA_branchwiseValues(year, branch)
	flash (' '.join(highs))
	return render_template("cgpa.html", values=values)

@app.route("/cgpa",methods=["GET"])
def cgpa_dist():
	year=16
	if request.args.get('year'):
		year=request.args.get('year')
	if int(year) not in [14,16,17]:
		return apology("Sorry, no data for that year");
	values,val,highs = getCGPA_allBranch(year)
	flash (' '.join(val[len(val)-3]))
	return render_template("cgpa.html", values=values)

@app.route("/sgpa",methods=["GET"])
def sgpa_dist():
	year=16
	if request.args.get('year'):
		year=request.args.get('year')
	if int(year) not in [14,16,17]:
		return apology("Sorry, no data for that year");
	values,val = getSGPA_allBranch(year)
	flash (' '.join(val[len(val)-3]))
	return render_template("sgpa.html", values=values)

@app.route("/sgpa/branch",methods=["GET"])
def sgpa_branchwise(year=16, branch="EC"):
	if request.args.get('year'):
		year=request.args.get('year')
	if request.args.get('branch'):
		branch=request.args.get('branch')
	if int(year) not in [14,16,17]:
		return apology("Sorry, no data for that year");
	values,highs = getSGPA_branchwiseValues(year,branch)
	flash (' '.join(highs))
	return render_template("sgpa.html", values=values)

@app.route("/subject",methods=["GET","POST"])
def subject():
	"""TODO: Implement error checking, display suitable messages if:
	a)Some argument is missing in the get request
	b)Invalid subcode"""
	sub="ID1T001"
	if request.method=="GET":
		if request.args.get("sub"):
			sub=request.args.get("sub")
	elif request.method=="POST":
		if not request.form.get("names"):
			return apology("Sorry, must provide subcode")
		sub=request.form.get("names").split("-")[0]
	scripts=[]
	divs=[]
	x=get_subject(sub)
	for key,value in x.items():
		p=figure(plot_width=400,plot_height=400,sizing_mode='scale_width')
		p.xaxis.axis_label='Batch '+str(key)
		l=[i for i in range(0,len(value))]
		#p.line(l,value,line_width=2)
		hist, edges = np.histogram(value, density=True, bins=10, weights=l)
		p.quad(top=hist, bottom=0,
			   left=edges[:-1], right=edges[1:], fill_color="#036564", line_color="#033649")
		script, div = components(p)
		scripts.append(script)
		divs.append(div)
	values = list(zip(scripts, divs))
	#r = make_response(render_template('subject.html',values=values))
	#print(r.data)
	return render_template('subject.html',values=values)

@app.route("/<string:box>")
def process(box):
	query = request.args.get('query')
	if query is not None:
		query=query.lower()
		print(query)
		global suggestions
		#suggestions = [{'label': 'mathematics first_year maths','value': 'MA1L001-Mathematics 1'}, {'label': 'jim','value': 'jim'}]
		'''TODO: Add keywords in label to describe the subject and set value to subject code
		by replacing the second entry with something like first entry.
		Link this field to subject table so that user can click to go to that page
		Also maybe add subject name with suitable delimiter (say: MA1L001-Mathematics 1)
		and parse it in File side. Currently delimiter is -'''
		temp=[]
		for val in suggestions:
			#print(val)
			k=val['label'].split(' ')
			for i in k:
				print(i)
				if i.lower().startswith(query):
					temp.append(val)
					break
		k=temp
		return jsonify({"suggestions":k})
	else:
		return jsonify({"suggestions":[{}]})


'''
Created on 27.07.2013

@author: Mischa
'''
import os
import shutil
import sys
import threading
import sqlite3
from datetime import datetime
from time import sleep
from nsabox.application.config import Config
from nsabox.application.flow.FlowService import FlowService
from nsabox.application.config.Config import SHUTDOWN_REQUEST_FILE

class Core(object):
    def __init__(self):
        self.__shutdownRequested = False
        self.__flowService = FlowService()

    def start(self):
        self.__startServices()
        while(not self.__shutdownRequested):
            try:
                self.__writeLogEntry("Core is working...")
                sleep(10)
                if(not os.path.isfile(SHUTDOWN_REQUEST_FILE)):
                    # os.remove(SHUTDOWN_REQUEST_FILE)
                    self.__shutdown()
                    os.system("sudo poweroff")
            except KeyboardInterrupt:
                self.__writeLogEntry("shutting down...")
                self.__shutdown()
            except:
                raise

    def __writeLogEntry(self, text):
        sys.stdout.write(self.__getDateTime() + " " + text + "\n")

    def __getDateTime(self):
        time = datetime.now()
        return "[%04d-%02d-%02d %02d:%02d:%02d]" % (time.year, time.month, time.day, time.hour, time.minute, time.second)

    def __startServices(self):
        self.__flowService.start()

    def __shutdown(self):
        self.__flowService.stop()
        self.__shutdownRequested = True


"""Langevin dynamics class."""

import numpy as np
from numpy.random import standard_normal, randint
import asap3
import ase
from asap3.md.md import MolecularDynamics, ParallelMolDynMixin
from ase.md.langevin import Langevin as Langevin_ASE
import asap3.constraints
import sys

class Langevin_New(object, Langevin_ASE, ParallelMolDynMixin):
    def __init__(self, atoms, timestep, temperature, friction, fixcm=True,
                 trajectory=None, logfile=None, loginterval=1):
        assert(getattr(Langevin_ASE, "_lgv_version", 1) >= 2)
        ParallelMolDynMixin.__init__(self, "Langevin", atoms)
        Langevin_ASE.__init__(self, atoms, timestep, temperature, friction, fixcm,
                              trajectory, logfile, loginterval, communicator=None)

    sdpos = property(lambda s: s.get("sdpos"), lambda s, x: s.set("sdpos", x))
    sdmom = property(lambda s: s.get("sdmom"), lambda s, x: s.set("sdmom", x))
    c1 = property(lambda s: s.get("c1"), lambda s, x: s.set("c1", x))
    c2 = property(lambda s: s.get("c2"), lambda s, x: s.set("c2", x))
    act0 = property(lambda s: s.get("act0"), lambda s, x: s.set("act0", x))
    c3 = property(lambda s: s.get("c3"), lambda s, x: s.set("c3", x))
    c4 = property(lambda s: s.get("c4"), lambda s, x: s.set("c4", x))
    pmcor = property(lambda s: s.get("pmcor"), lambda s, x: s.set("pmcor", x))
    cnst = property(lambda s: s.get("cnst"), lambda s, x: s.set("cnst", x))
    masses = property(lambda s: s.get("masses"), lambda s, x: s.set("masses", x))

class Langevin_Fast(object, Langevin_ASE, ParallelMolDynMixin):
    def __init__(self, atoms, timestep, temperature, friction, fixcm=True,
                 trajectory=None, logfile=None, loginterval=1):
        assert(getattr(Langevin_ASE, "_lgv_version", 1) >= 2)
        ParallelMolDynMixin.__init__(self, "Langevin", atoms)
        self._uselocaldata = False # Need to store on atoms for serial simul too.
        self.calculator = atoms.get_calculator()
        self.asap_md = asap3._asap.Langevin(atoms, self.calculator, timestep,
                                            self.prefix+"sdpos", self.prefix+"sdmom",
                                            self.prefix+"c1", self.prefix+"c2",
                                            fixcm, randint(1 << 30))
        if not atoms.has('momenta'):
            atoms.set_momenta(np.zeros((len(atoms), 3), float))
        if atoms.constraints:
            assert len(atoms.constraints) == 1
            constraint = atoms.constraints[0]
            assert isinstance(constraint, asap3.constraints.FixAtoms)
            constraint.prepare_for_asap(atoms)
            # Make all constants arrays by making friction an array
            friction = friction * np.zeros(len(atoms))
        Langevin_ASE.__init__(self, atoms, timestep, temperature, friction, fixcm,
                              trajectory, logfile, loginterval, communicator=None)

    def updatevars(self):
        Langevin_ASE.updatevars(self)
        if len(self.atoms.constraints) == 1:
            # Process the FixAtoms constraint
            constr = self.atoms.constraints[0].index
            self.sdpos[constr] = 0.0
            self.sdmom[constr] = 0.0
            self.c1[constr] = 0.0
            self.c2[constr] = 0.0
            self.c3[constr] = 0.0
            self.c4[constr] = 0.0
            self.act0[constr] = 0.0
        if self._localfrict:
            self.asap_md.set_vector_constants(self.prefix+"act0", self.prefix+"c3",
                                              self.prefix+"c4", self.prefix+"pmcor",
                                              self.prefix+"cnst")
        else:
            self.asap_md.set_scalar_constants(self.act0, self.c3, self.c4,
                                              self.pmcor, self.cnst)

    def run(self, steps):
        assert(self.calculator is self.atoms.get_calculator())
        self.asap_md.run(steps, self.observers, self)

    def get_random(self, gaussian):
        return self.asap_md.get_random(gaussian)

    # Properties are not inherited, need to repeat them
    sdpos = property(lambda s: s.get("sdpos"), lambda s, x: s.set("sdpos", x))
    sdmom = property(lambda s: s.get("sdmom"), lambda s, x: s.set("sdmom", x))
    c1 = property(lambda s: s.get("c1"), lambda s, x: s.set("c1", x))
    c2 = property(lambda s: s.get("c2"), lambda s, x: s.set("c2", x))
    act0 = property(lambda s: s.get("act0"), lambda s, x: s.set("act0", x))
    c3 = property(lambda s: s.get("c3"), lambda s, x: s.set("c3", x))
    c4 = property(lambda s: s.get("c4"), lambda s, x: s.set("c4", x))
    pmcor = property(lambda s: s.get("pmcor"), lambda s, x: s.set("pmcor", x))
    cnst = property(lambda s: s.get("cnst"), lambda s, x: s.set("cnst", x))
    masses = property(lambda s: s.get("masses"), lambda s, x: s.set("masses", x))

class Langevin_Old(MolecularDynamics):
    """Langevin (constant N, V, T) molecular dynamics.

    Usage: Langevin(atoms, dt, temperature, friction)

    atoms
        The list of atoms.

    dt
        The time step.

    temperature
        The desired temperature, in energy units.

    friction
        A friction coefficient, typically 1e-4 to 1e-2.

    fixcm
        If True, the position and momentum of the center of mass is
        kept unperturbed.  Default: True.

    The temperature and friction are normally scalars, but in principle one
    quantity per atom could be specified by giving an array.

    This dynamics accesses the atoms using Cartesian coordinates."""

    def __init__(self, atoms, timestep, temperature, friction, fixcm=True,
                 trajectory=None, logfile=None, loginterval=1):
        MolecularDynamics.__init__(self, atoms, timestep, trajectory,
                                   logfile, loginterval)
        self.temp = temperature
        self.frict = friction
        self.fixcm = fixcm  # will the center of mass be held fixed?

        # We need to store some data on the atoms, so they can migrate
        # along with the atoms in parallel asap simulations.  Since
        # there could be more than one Langevin object acting on the
        # same atoms, a unique prefix to the variable names is needed.
        i = 0
        while True:
            self.prefix = "langevin%d_" % (i,)
            if not self.atoms.has(self.prefix+"sdmom"):
                break  # Found unique prefix
            i += 1
        self.updatevars()

    def set_temperature(self, temperature):
        self.set("temp", temperature)
        self.updatevars()

    def set_friction(self, friction):
        self.set("frict", friction)
        self.updatevars()

    def set_timestep(self, timestep):
        self.dt = timestep
        self.updatevars()

    def updatevars(self):
        dt = self.dt
        # If the friction is an array some other constants must be arrays too.
        frict = self.get("frict")
        temp = self.get("temp")
        self._localfrict = hasattr(frict, 'shape')
        lt = frict * dt
        masses = self.atoms.get_masses()
        sdpos = dt * np.sqrt(temp / masses * (2.0/3.0 - 0.5 * lt) * lt)
        sdpos.shape = (-1, 1)
        sdmom = np.sqrt(temp * masses * 2.0 * (1.0 - lt) * lt)
        sdmom.shape = (-1, 1)
        pmcor = np.sqrt(3.0)/2.0 * (1.0 - 0.125 * lt)
        cnst = np.sqrt((1.0 - pmcor) * (1.0 + pmcor))

        act0 = 1.0 - lt + 0.5 * lt * lt
        act1 = (1.0 - 0.5 * lt + (1.0/6.0) * lt * lt)
        act2 = 0.5 - (1.0/6.0) * lt + (1.0/24.0) * lt * lt
        c1 = act1 * dt / masses
        c1.shape = (-1, 1)
        c2 = act2 * dt * dt / masses
        c2.shape = (-1, 1)
        c3 = (act1 - act2) * dt
        c4 = act2 * dt
        del act1, act2
        if self._localfrict:
            # If the friction is an array, so are these
            act0.shape = (-1, 1)
            c3.shape = (-1, 1)
            c4.shape = (-1, 1)
            pmcor.shape = (-1, 1)
            cnst.shape = (-1, 1)
        self.set("sdpos", sdpos)
        self.set("sdmom", sdmom)
        self.set("c1", c1)
        self.set("c2", c2)
        self.set("act0", act0)
        self.set("c3", c3)
        self.set("c4", c4)
        self.set("pmcor", pmcor)
        self.set("cnst", cnst)
        self.natoms = self.atoms.get_number_of_atoms()

    def step(self, f):
        atoms = self.atoms
        p = self.atoms.get_momenta()

        random1 = standard_normal(size=(len(atoms), 3))
        random2 = standard_normal(size=(len(atoms), 3))

        rrnd = self.get("sdpos") * random1
        prnd = (self.get("sdmom") * self.get("pmcor") * random1 +
                self.get("sdmom") * self.get("cnst") * random2)

        if self.fixcm:
            rrnd = rrnd - np.sum(rrnd, 0) / len(atoms)
            prnd = prnd - np.sum(prnd, 0) / len(atoms)
            factor = np.sqrt(self.natoms / (self.natoms - 1.0))
            rrnd *= factor
            prnd *= factor

        atoms.set_positions(atoms.get_positions() +
                            self.get("c1") * p +
                            self.get("c2") * f + rrnd)
        p *= self.get("act0")
        p += self.get("c3") * f + prnd
        atoms.set_momenta(p)

        f = atoms.get_forces()
        atoms.set_momenta(atoms.get_momenta() + self.get("c4") * f)
        return f

def Langevin(atoms, *args, **kwargs):
    if (isinstance(atoms, ase.Atoms)
        and asap3.constraints.check_asap_constraints(atoms)
        and getattr(Langevin_ASE, "_lgv_version", 1) >= 2
       ):
        sys.stderr.write("Using Asap-optimized C++-Langevin algorithm\n")
        return Langevin_Fast(atoms, *args, **kwargs)
    elif getattr(Langevin_New, "_lgv_version", 1) >= 2:
        sys.stderr.write("Using ASE-based Langevin algorithm\n")
        return Langevin_New(atoms, *args, **kwargs)
    else:
        sys.stderr.write("Using reimplemented Python-Langevin algorithm\n")
        return Langevin_Old(atoms, *args, **kwargs)


import numpy as np
import collections
import copy

BeamObject = collections.namedtuple('BeamObject', 'hypothesis score state')

class BeamSearch:
    def __init__(self, beam_size, model, sample, max_length=100):
        self.beam_size = beam_size
        self.max_length = max_length

        self._model = model
        self._input = self._model.prepare_test_input(sample)
        self.search()

    def search(self):

        init_input = []
        if hasattr(self._model, 'initializer'):
            self.encoding = self._model.initializer(*self._input)
            init_word = np.array([[-1]])
            init_word_mask = np.array([[1.]], dtype='float32')
            init_input = [init_word, init_word_mask] + self.encoding

        probs, next_token_index, decoder_outputs = self._model.sampler(*init_input)

        logprobs = np.log(probs)
        vocab_size = logprobs.shape[1]
        candidates = np.argpartition(logprobs, -self.beam_size, axis=None)[-self.beam_size:]

        state_dict = {'F0':copy.copy(decoder_outputs[0,0])}

        self.active_beam = [ BeamObject(hypothesis=[self._model.decoder_vocab.get_token(index)],
                                        score=logprobs.flatten()[index],
                                        state=state_dict) for index in candidates]

        self.completed_beam = []

        active_beam_size = len(self.active_beam)

        steps = 0
        while True:
            steps += 1

            # popping beams which have been finished and putting it in the completed beams
            last_tokens = []
            last_states = dict()
            pop_indices = dict()

            for beam_index in xrange(len(self.active_beam)):

                beam_object = self.active_beam[beam_index]
                last_token = beam_object.hypothesis[-1]

                if last_token == self._model.decoder_vocab.eos:
                    # remove the end of sentence token and append to the completed beam
                    self.completed_beam.append(beam_object)
                    pop_indices[beam_index] = True
                    active_beam_size = active_beam_size - 1
                else:
                    last_tokens.append(last_token)
                    for k,v in beam_object.state.iteritems():
                        if k not in last_states:
                            last_states[k] = []
                        last_states[k].append(v)

            self.active_beam = [beam_object for beam_index, beam_object in enumerate(self.active_beam) if beam_index not in pop_indices]

            # stop beam search, if nothin exists in active beam
            if active_beam_size == 0:
                break

            #for bo in self.active_beam:

            # TODO : adjust encoding preparation to avoid repeating self.encoding
            # TODO: ensemble
            last_states['F0'] = np.array(last_states['F0'])
            sampler_input = self._model.prepare_sampler_input(last_tokens, last_states['F0'], self.encoding)

            probs, next_token_index, decoder_outputs = self._model.sampler(*sampler_input)

            #print probs

            logprobs = np.log(probs)
            hypscores = np.array([[self.active_beam[beam_index].score] for beam_index in xrange(len(self.active_beam))])
            logprobs = logprobs + hypscores #broadcasting hypscores [dim: #beamsize x 1] with logprobs [ dim: #beamsize x #vocabsize]
            candidates = np.argpartition(logprobs, -active_beam_size, axis=None)[-active_beam_size:]

            beam_indices = candidates/vocab_size
            vocab_indices = candidates % vocab_size

            self.active_beam = [ BeamObject(hypothesis=self.active_beam[beam_index].hypothesis +
                                                [self._model.decoder_vocab.get_token(vocab_index)],
                                            score=logprobs[beam_index][vocab_index],
                                            state={'F0':copy.copy(decoder_outputs[0,beam_index])})
                                for beam_index, vocab_index in zip(beam_indices, vocab_indices)]

            # stop beam search if maximum steps have reached
            if steps >= self.max_length:
                self.completed_beam += self.active_beam
                break

        self.completed_beam = sorted(self.completed_beam, key=lambda x: x.score, reverse=True)

    def best_hypothesis_sentence(self):
        if self.completed_beam[0].hypothesis[-1] == '</s>':
            return ' '.join(self.completed_beam[0].hypothesis[:-1])
        else:
            return ' '.join(self.completed_beam[0].hypothesis)

    def best_hypothesis_score(self):
        return self.completed_beam[0].score

    def nbest(self):
        return self.completed_beam

    def moses_formatted_nbest(self, encoding='utf-8'):
        raise NotImplementedError


# -*- coding: utf-8 -*-

import cStringIO
import csv
import os

import re
import tempfile

from pyspark.sql import SQLContext

from scalableor.constant import COLUMN_NAME
from scalableor.context import eval_expression, to_grel_object
from scalableor.manager import MethodsManager

from scalableor.facet import get_facet_filter

@MethodsManager.register("scalableor/import")
def sc_or_import(cmd, sc=None, **kwargs):
    """
    import data in spark context and split rows in column

    :param cmd:         import parameters
    :param sc:          spark context object
    """
    rdd = sc.textFile(cmd["path"])
    rdd_splitted = rdd.map(lambda el: el.split(cmd["separator"]))
    sql_context = SQLContext(sc)
    df = sql_context.createDataFrame(rdd_splitted)

    for i in range(len(df.columns)):
        df = df.withColumnRenamed(df.columns[i], COLUMN_NAME % (i + 1))
    return df

@MethodsManager.register("scalableor/export")
def sc_or_export(cmd, df=None, **kwargs):
    """
    export data to file system

    :param cmd:         export parameters
    :param df:          spark data frame
    """
    tmp = tempfile.mkdtemp() + ".scalable.or"
    df.write.format("com.databricks.spark.csv").save(tmp)

    with open(cmd["path"], "w") as output:
        for fpath in sorted(os.listdir(tmp)):
            if fpath.startswith("part-"):
                with open(os.path.join(tmp, fpath), "r") as result:
                    for line in result.readlines():
                        output.write(line)

@MethodsManager.register("core/column-rename")
def core_column_rename(cmd, df, **kwargs):
    """
    rename column
    """
    return df.withColumnRenamed(cmd["oldColumnName"], cmd["newColumnName"])

@MethodsManager.register("core/column-removal")
def core_column_removal(cmd, df, **kwargs):
    """
    remove column by name
    """
    return df.drop(cmd["columnName"])

@MethodsManager.register("core/column-move")
def core_column_move(cmd, df, **kwargs):
    """
    move column to index by name
    """
    columns = df.columns[:]
    current_index = columns.index(cmd["columnName"])
    columns.insert(cmd["index"], columns.pop(current_index))

    replace_order = [i for i in range(len(columns))]
    replace_order.insert(cmd["index"], replace_order.pop(current_index))

    rdd_moved = df.rdd.map(lambda row: [row[i] for i in replace_order])
    return df.sql_ctx.createDataFrame(rdd_moved, columns)

@MethodsManager.register("core/row-removal")
def core_row_removal(cmd, df=None, **kwargs):
    """
    remove rows selected by facet filter
    """
    facet_filter = get_facet_filter(cmd, df)
    result = df.rdd.filter(lambda e: facet_filter(e) is False)
    return df.sql_ctx.createDataFrame(result, df.columns)

@MethodsManager.register("core/column-split")
def core_column_split(cmd, df=None, **kwargs):
    """
    split column by separator or field length
    """
    column_names = df.columns[:]
    pos = column_names.index(cmd["columnName"])

    before_columns = column_names[:pos]
    after_columns = column_names[pos + 1:]

    if "fieldLengths" in cmd:
        func = lambda e: \
            e[:pos + 1] + \
            tuple(to_grel_object(e[pos]).splitByLengths(*cmd["fieldLengths"])) + \
            e[pos + 1:]
    else:
        # max column logic
        if "maxColumns" in cmd:
            max_column = cmd["maxColumns"]
            if max_column == 1:
                return df
            if max_column > 1:
                max_column -= 1
        else:
            max_column = 0

        # if max column doesn't defined analyse first 20 column und select maximal column count
        if max_column < 1:
            max_column = 2
            for row in df.head(20):
                if hasattr(row[pos], "split"):
                    if cmd.get("regex") is True:
                        max_column = max(len(re.split(cmd["separator"], row[pos])), max_column)
                    else:
                        max_column = max(len(row[pos].split(cmd["separator"])), max_column)
            max_column -= 1

        # generate split callback
        add_to = ["" for _ in range(max_column + 1)]
        if cmd.get("regex") is True:
            func = lambda e: \
                e[:pos + 1] + \
                tuple((re.split(cmd["separator"], e[pos], max_column) + add_to)[:max_column + 1]) + \
                e[pos + 1:]
        else:
            func = lambda e: \
                e[:pos + 1] + \
                tuple((e[pos].split(cmd["separator"], max_column) + add_to)[:max_column + 1]) + \
                e[pos + 1:]

    result = df.sql_ctx.createDataFrame(df.rdd.map(func))

    # generate new column names
    new_column_names = (
        before_columns +
        [cmd["columnName"]] +
        ["%s %s" % (
            cmd["columnName"], i + 1) for i in range(len(result.columns) - len(df.columns))] +
        after_columns)

    for index, name in enumerate(new_column_names):
        result = result.withColumnRenamed("_%d" % (index + 1), name)

    if cmd.get("removeOriginalColumn") is True:
        result = result.drop(cmd["columnName"])
    return result

@MethodsManager.register("core/column-addition")
def core_column_addition(cmd, df, **kwargs):
    """
    create new column based on existing one
    """
    names = df.columns[:]
    position_of_column = df.columns.index(cmd["baseColumnName"])

    before_columns = df.columns[:position_of_column + 1]
    after_columns = df.columns[position_of_column + 1:]

    facet_fitler = get_facet_filter(cmd, df)

    # generate spark callback
    result_rdd = df.rdd.map(lambda e: (
        e[:position_of_column + 1] +
        ((eval_expression(e,
                          position_of_column,
                          cmd["expression"],
                          names=names),) if facet_fitler(e) else ("",)) +
        e[position_of_column + 1:]))

    return df.sql_ctx.createDataFrame(
        result_rdd,
        before_columns + [cmd["newColumnName"]] + after_columns)

@MethodsManager.register("core/text-transform")
def core_text_transform(cmd, df, **kwargs):
    """
    transform row values of selected column
    """
    names = df.columns[:]
    pos_of_column = df.columns.index(cmd["columnName"])
    facet_fitler = get_facet_filter(cmd, df)

    result_rdd = df.rdd.map(lambda e: (
        e[:pos_of_column] +
        ((eval_expression(e,
                          pos_of_column,
                          cmd["expression"],
                          names=names),) if facet_fitler(e) else (e[pos_of_column],)) +
        e[pos_of_column + 1:]))

    return df.sql_ctx.createDataFrame(result_rdd, df.columns)

@MethodsManager.register("core/mass-edit")
def core_mass_edit(cmd, df, **kwargs):
    """
    change row values of selected column using filter
    """
    pos_of_column = df.columns.index(cmd["columnName"])

    def core_mass_edit_callback(e):
        current_value = e[pos_of_column]
        new_value = None
        for edit in cmd["edits"]:
            if current_value in edit["from"]:
                new_value = edit["to"]
                break
        if new_value is None:
            new_value = current_value
        return (e[:pos_of_column] +
                (new_value,) +
                e[pos_of_column + 1:])

    return df.sql_ctx.createDataFrame(df.rdd.map(core_mass_edit_callback), df.columns)

@MethodsManager.register("core/fill-down")
def core_fill_down(cmd, df, **kwargs):
    """
    dummy function
    """
    return df


# author: Jasmine Qin
# date: 2020-06-09

"""
This script performs data wrangling and synthesizing needed for
visualization of the business licence file.

Usage: src/04_visualization/licence_vis_synthesis.py
"""

# load packages
import pandas as pd
import json
import re
from joblib import load
import warnings

warnings.filterwarnings("ignore")

def main():

    # read data
    licence_df = pd.read_csv(
        "data/processed/03_cleaned_combined_licences.csv",
        low_memory=False)
    parking = pd.read_csv(
        "data/raw/parking-meters.csv", sep=';')
    disability_parking = pd.read_csv(
        "data/raw/disability-parking.csv", sep=';')

    # parking cleaning
    parking = parking[['Geom', 'Geo Local Area']].rename(
        columns={'Geo Local Area': 'LocalArea'})

    disability_parking = disability_parking[[
        'Geom', 'Geo Local Area']].rename(
        columns={'Geo Local Area': 'LocalArea'})

    # licence cleaning
    # 1. remove null geom
    licence_df = licence_df[licence_df['Geom'].notnull()]

    # 2. remove unused columns
    cols_not_used = ['business_id',
                     'LicenceRSN',
                     'LicenceNumber',
                     'LicenceRevisionNumber',
                     'Unit',
                     'UnitType',
                     'House',
                     'Street',
                     'Country',
                     'label']

    licence_df = licence_df.drop(columns=cols_not_used)

    # 3. remove null BusinessIndustry
    licence_df = licence_df[licence_df.BusinessIndustry.notnull()]

    # 4. FOLDERYEAR to int
    licence_df['FOLDERYEAR'] = [int(i) for i in licence_df['FOLDERYEAR']]
    licence_df = licence_df.sort_values('FOLDERYEAR')

    # get coordinates
    for df in [parking, disability_parking, licence_df]:
        df["coord-x"] = df['Geom'].apply(
            lambda p: json.loads(p)['coordinates'][0])
        df["coord-y"] = df['Geom'].apply(
            lambda p: json.loads(p)['coordinates'][1])

    #################
    # Aggregated df #
    #################

    df = pd.read_csv(
        "data/processed/combined_licences.csv",
        low_memory=False)

    # organize FOLDERYEAR
    df = df.loc[df.FOLDERYEAR.notnull()]
    df['FOLDERYEAR'] = [y + 2000 if y >= 0 and y <
                        90 else y + 1900 for y in df.FOLDERYEAR]
    df['ExtractDate'] = pd.to_datetime(df['ExtractDate'], errors='ignore')
    df['IssuedDate'] = pd.to_datetime(df['IssuedDate'], errors='ignore')
    df['ExpiredDate'] = pd.to_datetime(df['ExpiredDate'], errors='ignore')

    df.loc[(df['FOLDERYEAR']
            < 1997.0) & (df['IssuedDate'].dt.year == 1996.0)
           & (df['ExpiredDate'].dt.year == 1997.0), 'FOLDERYEAR'] = 1997.0

    df = df[~(df.FOLDERYEAR < 1997.0)]
    df['FOLDERYEAR'] = [int(i) for i in df['FOLDERYEAR']]

    df = df.sort_values(by=['business_id', 'FOLDERYEAR', 'ExtractDate'])
    df = df[df.groupby(['business_id'])['FOLDERYEAR'].apply(
        lambda x: ~(x.duplicated(keep='last')))]

    # only Issued licences
    df = df.query('Status == "Issued"')

    # Industry mapping
    mapping = pd.read_csv("src/02_clean_wrangle/business_mapping_dictionary.csv")
    df = df.merge(mapping, on=["BusinessType"], how="left")

    # Remove 2010 Winter games : Outlier
    df = df[df.BusinessIndustry != 'Historic']
    df = df[df.BusinessIndustry != 'Real estate and rental and leasing']
    df = df[df.LocalArea.notnull()]
    df = df[df.BusinessIndustry.notnull()]

    agg_viz = pd.DataFrame(df.groupby([
        'FOLDERYEAR', 'LocalArea',
        'BusinessIndustry', 'BusinessType'])[
        'business_id'].count()).reset_index()

    agg_viz = agg_viz[~(agg_viz.BusinessType.str.contains(
        r'\*Historic\*'))]

    #############
    # Modelling #
    #############
    train = pd.read_csv("data/processed/05_feat_eng_train.csv")
    valid = pd.read_csv("data/processed/05_feat_eng_validate.csv")
    model = load('results/final_model.joblib')

    admin_cols = ["business_id", "BusinessName",
                  "BusinessTradeName", "Status",
                  "BusinessSubType", "label",
                  "BusinessIndustry",
                  "NextYearStatus", "Geom"]

    X_train = train.drop(columns=admin_cols)
    X_valid = valid.drop(columns=admin_cols)

    train["predict"] = model.predict(X_train)
    train['predict_proba'] = [
        max(i) for i in model.predict_proba(X_train)]
    valid["predict"] = model.predict(X_valid)
    valid['predict_proba'] = [
        max(i) for i in model.predict_proba(X_valid)]

    train['type'] = ['train']*len(train)
    valid['type'] = ['valid']*len(valid)

    vis_model = pd.concat([train, valid])

    vis_model['predicted_right'] = list(vis_model.label == vis_model.predict)
    vis_model['predicted_right'] = [1 if i else -
                                    1 for i in vis_model['predicted_right']]
    vis_model['predict_proba'] = vis_model['predict_proba'] * \
        vis_model['predicted_right']

    # prepare shapely geom
    vis_model = vis_model[vis_model.Geom.notnull()]
    vis_model['coord-x'] = vis_model['Geom'].apply(
        lambda p: json.loads(p)['coordinates'][0])
    vis_model['coord-y'] = vis_model['Geom'].apply(
        lambda p: json.loads(p)['coordinates'][1])

    # save to files
    vis_model.to_csv("data/processed/vis_model.csv", index=False)
    licence_df.to_csv("data/processed/vis_licence.csv", index=False)
    agg_viz.to_csv("data/processed/vis_agg_licence.csv", index=False)
    parking.to_csv("data/processed/vis_parking.csv", index=False)
    # disability_parking.to_csv(
    #    "data/processed/vis_disability_parking.csv", index=False)

if __name__ == "__main__":
    main()