import asyncio, argparse
import json
import logging
import random
import websockets
import queue
import sys
import uuid
import operator

from datetime import datetime

from blockchain import Blockchain
from transactions import Transaction
from block import Block
from TransactionJSONEncoder import TransactionJSONEncoder

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--listen', type=str,
                    help='ip and port: example: 127.0.0.1:8000')
args = parser.parse_args()

class RaftServer(object):
    """
    This class implements a blockchain based on the raft consensus algorithm. Therefore, it starts a websocket server and
    connects to the other raft nodes, setting up a peer2peer connection between all raft nodes. The consensus across all
    nodes is achieved by determining a leader, who is then responsible for mining and sharing new block. Instances that
    are followers will verify the newly mined block and append it to their blockchain. After a period of five minutes
    a new leadership election will automatically start.
    """
    STATE_FOLLOWER = 'follower'
    STATE_CANDIDATE = 'candidate'
    STATE_LEADER = 'leader'

    KEY_FIRST_VOTE = 'not elected'

    FORWARDED_KEY = "forwarded"

    HEARTBEAT_INTERVAL = 0.5

    ELECTION_TIMEOUT = 0.3

    LEADER_HEARTBEATS = 300 / 0.5  # 5 Minutes

    votes = {}

    _connections_to_servers = {}

    connected_clients = set()
    raft_nodes = set()
    control_connection = set()
    confirmed_commits = {}

    node_connections = {}

    def __init__(self, host, port, nodes, control_channels, blockchain_path):
        self.host = host
        self.port = port
        self.identifier = "{}:{}".format(self.host, self.port)

        self.nodes = nodes
        self.control_channels = control_channels

        self.blockchain = Blockchain(blockchain_path)
        self.blockchain.load()

        self._state = self.STATE_FOLLOWER
        self._currentTerm = 0
        self._commitIndex = 0
        self._lastApplied = 0
        self.votes[self.identifier] = self.identifier
        self._leader = self.KEY_FIRST_VOTE
        self._last_heartbeat = datetime.now()
        self._is_syncronizing = False
        self._backlog = queue.Queue()
        self.messages_to_save = {}
        self.heartbeats_left = self.LEADER_HEARTBEATS
        self.has_voted = False

    async def __aenter__(self):
        """
        Start websocket server and raft management.
        :return:
        """
        # Start Websocket Server
        await self.start_server()

        # Start raft state management
        asyncio.ensure_future(self.manage_raft_state(), loop=asyncio.get_event_loop())

        # Connect to other servers
        asyncio.ensure_future(self.periodic_connect_task(), loop=asyncio.get_event_loop())

        return self

    async def __aexit__(self, *args, **kwargs):
        await self._ws_server.__aexit__(*args, **kwargs)

    async def start_election(self):
        """
        Start election process
        :return:
        """
        self._currentTerm += 1
        self._state = self.STATE_CANDIDATE
        self.votes[self.identifier] = self.identifier
        self._leader = None
        self.has_voted = True
        await self.broadcast_to_clients({
            "type": "vote",
            "identifier": self.votes[self.identifier]
        })

    async def manage_raft_state(self):
        """
        Manages raft states.
        """
        await asyncio.sleep(2)
        while True:
            # No leader and missing heartbeat -> start new election
            if self._state == self.STATE_LEADER and self.heartbeats_left > 0:
                self.heartbeats_left -= 1
                # TODO: uncomment  + check if queue is empty
                self._last_heartbeat = datetime.now()
                await self.broadcast_to_clients({
                    "type": "heartbeat",
                    "identifier": self.identifier
                })

            # Force reelection after 20 heartbeats
            if self.heartbeats_left == 0:
                self._state = self.STATE_CANDIDATE
                self.heartbeats_left = self.LEADER_HEARTBEATS
                election_timeout = random.randint(100, 200) / 1000
                # election_timeout = random.randint(1, 3)
                await asyncio.sleep(election_timeout)

            if self.heartbeat_timed_out() and self._state != self.STATE_LEADER and not self._is_syncronizing:
                election_timeout = random.randint(150, 300) / 1000
                # election_timeout = random.randint(1, 3)
                await asyncio.sleep(election_timeout)
                await self.start_election()

            logger.debug("State: {}".format(self._state))
            logger.debug("Elected leader: {}".format(self._leader))
            await asyncio.sleep(self.HEARTBEAT_INTERVAL)

    def heartbeat_timed_out(self):
        return (datetime.now() - self._last_heartbeat).total_seconds() > self.HEARTBEAT_INTERVAL * 1.5

    async def start_server(self):
        """
        Start websocket server.
        :return:
        """
        logger.info("Starting websocket server")
        self._ws_server = websockets.serve(lambda ws, msg: self.server_on_message(ws, msg), self.host, self.port)
        asyncio.ensure_future(self._ws_server, loop=asyncio.get_event_loop())

    async def force_clients_to_sync_with_me(self):
        """
        Force connected raft nodes to sync their blockchain to the local one.
        :return:
        """
        for ws in self.raft_nodes:
            await self.sync_client(ws)

    async def client_recv_handler(self, connection):
        """
        Handler to communicate with the other servers
        """
        try:
            while True:
                data = await connection.recv()
                message = json.loads(data)

                await self.handle_message_exchange(message, connection)
        except Exception as e:
            pass

    async def connect_to_server(self, uri, handler):
        """
        Connect to other raft server if no connection has been made so far.
        :param uri:
        :param handler:
        :return:
        """
        if not "{}:{}".format(self.host, self.port) in uri:
            # Do not connect to yourself
            try:
                ws = websockets.connect(uri)
                connection = await ws.__aenter__()
                self._connections_to_servers["{}:{}".format(*connection.remote_address)] = connection
                asyncio.ensure_future(handler(connection), loop=asyncio.get_event_loop())
                await self.on_connect(connection)
            except Exception as e:
                logger.error("Connection failed: {} {}".format(uri, e))

    async def register_as_raft_node(self, websocket):
        await websocket.send(json.dumps({
            'type': 'register',
            'data': 'node',
            'identifier': self.identifier
        }))

    async def control_recv_handler(self, connection):
        """
        Handler to communicate with the other servers
        """
        while True:
            data = await connection.recv()
            message = json.loads(data)
            logger.info('Received message at control from {}:{} > {}'.format(*connection.remote_address, message))
            await self.handle_message_exchange(message, connection)

    async def periodic_connect_task(self):
        """
        Try to connect to configured nodes every 5 seconds.
        """
        self.ws_control_channel = None
        while True:
            for node in self.nodes:
                if node not in self._connections_to_servers and node not in self.node_connections:
                    await self.connect_to_server("ws://{}".format(node), self.client_recv_handler)
            for control_channel in self.control_channels:
                # await self.connect_to_server("ws://{}".format(control_channel), self.control_recv_handler)
                if self.ws_control_channel is None:
                    self.ws_control_channel = True
                    await self.connect_to_server("ws://{}".format(control_channel), self.control_recv_handler)

            await asyncio.sleep(2)

    async def send_heartbeat(self):
        """
        Send heartbeat fo connected raft nodes.
        :return:
        """
        await self.broadcast_to_clients({
            "type": "heartbeat",
            "identifier": self.identifier
        })

    async def sync_client(self, ws):
        """
        Send the local blockchain to the connected raft node.
        :param ws:
        :return:
        """
        await ws.send(json.dumps({
            "type": "start_sync"
        }))

        await ws.send(json.dumps({
            "type": "sync",
            "data": self.blockchain.export_as_json()
        }))

    async def server_on_message(self, websocket, path):
        """
        Register new connections to server, manages a constant connection and disconnect of clients.
        Incoming messages will be passed on the handle_message_echange function.
        :param websocket:
        :param path:
        :return:
        """
        # Register websocket
        self.connected_clients.add(websocket)
        try:
            await self.on_connect(websocket)

            while True:
                try:
                    data = await websocket.recv()
                    message = json.loads(data)

                    await self.handle_message_exchange(message, websocket)
                except Exception as e:
                    logger.error(e)
                    if websocket in self.connected_clients:
                        self.connected_clients.remove(websocket)
                    if websocket in self.raft_nodes:
                        self.raft_nodes.remove(websocket)
                    if websocket in self.control_connection:
                        self.control_connection.remove(websocket)
                    if websocket in self.node_connections:
                        self.node_connections.remove(websocket)
                    break
        finally:
            if websocket in self.connected_clients:
                self.connected_clients.remove(websocket)

    async def sync_last_block(self, message_uuid):
        """
        Send the last block of the local blockchain to all connected raft nodes
        :param message_uuid:
        :return:
        """
        self.confirmed_commits[self._currentTerm] = {'confirmations': set(),
                                                     'data': self.blockchain.last_block().pending_messages[0]}
        await self.broadcast_to_clients({
            "type": "commit",
            "data": json.dumps(self.blockchain.last_block(), ensure_ascii=False, cls=TransactionJSONEncoder),
            "term": self._currentTerm,
            "uuid": message_uuid
        })

    async def broadcast_to_clients(self, message):
        """
        Broadcast a message to all connected clients.
        """
        message['term'] = self._currentTerm
        self._currentTerm += 1

        # Send to raft nodes that this instance is connected to as a client
        for ws in self.raft_nodes:
            try:
                await ws.send(json.dumps(message))
            except Exception as e:
                logger.error("Error while sending to client: {}".format(e))

    async def handle_message_exchange(self, message, websocket):
        """
        Maps arriving messages to their local handler.
        The value of the key 'type' will determine which message will be called.
        The message will then forwared to the selected handler.
        :param message:
        :param websocket:
        :return:
        """
        logger.info('Received message from {}:{} > {}'.format(*websocket.remote_address, message))

        handler = getattr(self, "on_{}".format(message['type']), None)

        if handler is not None:
            await handler(message, websocket)
        else:
            logger.error("No handler found: {}".format(message))

    async def on_connect(self, websocket):
        """
        Sending newly connected clients immediately an heartbeat to put them into follower mode.
        :param websocket:
        :return:
        """
        logger.info("New connection from: {}:{}".format(*websocket.remote_address))
        await self.register_as_raft_node(websocket)
        if self._state == self.STATE_LEADER:
            # Sending heartbeat to new connection to set me as leader
            await self.send_heartbeat()
            await self.sync_client(websocket)

    async def on_heartbeat(self, message, websocket):
        """
        Handles heartbeat messages. Current instance will go into follower mode and updates last heartbeat timestamp.
        :param message:
        :param websocket:
        :return:
        """
        self._state = self.STATE_FOLLOWER
        self._leader = message['identifier']
        self._last_heartbeat = datetime.now()

    async def on_vote(self, message, websocket):
        self.votes["{}:{}".format(*websocket.remote_address)] = message['identifier']

        if self._state == self.STATE_CANDIDATE:
            if self.heartbeat_timed_out():
                self.votes[self.identifier] = message['identifier']
                self._currentTerm = message['term']
                self._last_heartbeat = datetime.now()

                await self.broadcast_to_clients({
                    "term": self._currentTerm,
                    "type": "vote",
                    "identifier": self.votes[self.identifier]
                })

        # We subtract 1 of len(self.votes), because the current instance also voted but is not part of self.node_connections
        everyone_has_voted = (len(self.votes) - 1) == len(self.node_connections)
        if everyone_has_voted:
            leader = self.get_leader()
            if leader is not None:
                if leader == self.identifier:
                    logger.info("I am the new leader! \o/")
                    first_election = self._leader == self.KEY_FIRST_VOTE
                    # self.votes = {}
                    self._state = self.STATE_LEADER
                    self._leader = self.identifier
                    await self.send_heartbeat()
                    if first_election:
                        # System starts up -> sync chain
                        await self.force_clients_to_sync_with_me()
                else:
                    self._state = self.STATE_FOLLOWER

    def get_websocket_identifier(self, websocket):
        """
        Build a string identifier from websocket object.
        :param websocket:
        :return:
        """
        return "{}:{}".format(*websocket.remote_address)

    async def on_start_sync(self, message, websocket):
        """
        Instance goes into sync mode and queues broadcasted blocks until sync is done.
        :param message:
        :param websocket:
        :return:
        """
        logging.info("Start sync")
        self._is_syncronizing = True

    async def on_sync(self, message, webssocket):
        """
        Receives blockchain from the leader, validates that the shared blockchain is an instance of the local blockchain.
        After sync is done, all queued blocks will be appended to the local blockchain.
        :param message:
        :param webssocket:
        :return:
        """
        logger.info("Sync in progress")
        json_chain = json.loads(message['data'])
        blockchain_candidate = Blockchain(None)
        blockchain_candidate.import_chain_from_json(json_chain)
        if self.blockchain.is_empty():
            # Blockchain is empty -> Import existing one
            logger.info("Importing new blockchain")
            self.blockchain.import_blockchain_from_blockchain_obj(blockchain_candidate)
        else:
            # Node was already part of a raft group with a blockchain
            # -> Verify that imported blockchain is extension of saved one

            # TOOD: What to do, if old chain is longer -> Force sync with me
            if blockchain_candidate.length >= self.blockchain.length:
                # Just try to import if longer
                if self.blockchain.is_partial_blockchain(blockchain_candidate):
                    logger.info("Importing blockchain")
                    self.blockchain.import_blockchain_from_blockchain_obj(blockchain_candidate)
                    logger.info("Blockchain successfully synced")
                else:
                    logger.error(
                        "The loaded blockchain from does not belong to this network. Aborting!!!")
                    sys.exit()
            else:
                logger.info("My blockchain is longer. Do not accept it.")
                await self.force_clients_to_sync_with_me()

        self.blockchain.save()

        # Save messages from backlog
        await self.commit_messages_from_backlog()

        # sync is done
        self._is_syncronizing = False

    async def on_commit(self, message, websocket):
        """
        Shares data that should be stored in the blockchain with the network so that every node can verify the content was not changed.
        If the current instance is leader, a new block will be mined and synced across the network.
        A follower will append the newly mined block to it's local blockchain.
        :param message:
        :param websocket:
        :return:
        """
        if "uuid" not in message:
            # Add id and broadcast to all nodes
            message_uuid = str(uuid.uuid4())
            message['uuid'] = message_uuid

            update_message = {**message}
            update_message['type'] = 'message_broadcast'

            await self.broadcast_to_clients(update_message)

        message_uuid = message['uuid']
        self.messages_to_save[message_uuid] = {**message}

        if self._state == self.STATE_LEADER:
            # Leader is to mine the next block
            logger.info("Mining Block and broadcasting.")
            data = json.loads(message["data"])
            if self.blockchain.is_empty():
                # init blockchain
                self.blockchain.initialize_chain()
                await self.sync_last_block(message_uuid='None')

            # Calculate blockchain
            self.blockchain.add_transaction(**data)
            self.blockchain.mine_block()
            self.blockchain.save()

            del self.messages_to_save[message_uuid]
            await self.sync_last_block(message_uuid=message_uuid)
        elif self._state == self.STATE_FOLLOWER:
            # term will be None if messages comes from control connection
            term = message.get("term", None)
            if term is not None:
                self._backlog.put(message)
                self._currentTerm = term
                if not self._is_syncronizing:
                    await self.commit_messages_from_backlog()

    async def on_message_broadcast(self, message, websocket):
        """
        Saves shared unmined messages to local chache.
        :param message:
        :param websocket:
        :return:
        """
        self.messages_to_save[message['uuid']] = message
        if self._state == self.STATE_LEADER:
            await self.on_commit(message, websocket)

    async def commit_messages_from_backlog(self):
        """
        Validates a block send from the leader and appends it to the local blockchain.
        :return:
        """
        while not self._backlog.empty():
            message = self._backlog.get()
            term = message.get('term')
            broadcasted_message = json.loads(self.messages_to_save[message['uuid']]['data'])
            data = json.loads(message['data'])
            message_integrity = broadcasted_message == data
            if message_integrity:
                new_transaction = Transaction(**json.loads(data['pending_messages'][0]))
                data['pending_messages'] = [new_transaction]
                new_block = Block(**data)
                self.blockchain.add_block(new_block)
                self.blockchain.save()
                await self.broadcast_to_clients({
                    'type': 'client_commit_confirm',
                    'source_term': term
                })
            else:
                await self.start_election()

    async def on_register(self, message, websocket):
        """
        Every newly connected node must register to state its role. A connection can either be part of the raft network
        or a control connection to saves messages in the network.
        :param message:
        :param websocket:
        :return:
        """
        if message['data'] == 'node':
            # append to node list -> node is part of raft
            self.raft_nodes.add(websocket)
            self.node_connections[message['identifier']] = websocket
        elif message['data'] == 'control':
            # append to control
            self.control_connection.add(websocket)

    async def on_client_commit_confirm(self, message, websocket):
        """
        Broadcast a message to all control connections after a message has been successfully committed by all
        nodes.
        :param message:
        :param websocket:
        :return:
        """
        if self._state == self.STATE_LEADER:
            try:
                term = message['source_term']
                self.confirmed_commits[term]['confirmations'].add(self.get_websocket_identifier(websocket))
                if len(self.confirmed_commits[term]['confirmations']) == len(self.raft_nodes):
                    # send confirm to control channel

                    data = json.dumps(self.confirmed_commits[term]['data'], ensure_ascii=False,
                                      cls=TransactionJSONEncoder)
                    del self.confirmed_commits[term]
                    await self.broadcast_to_control_connections({
                        'type': 'commit_confirm',
                        'data': data
                    })
            except Exception as e:
                print("Error", e)

    async def on_commit_confirm(self, message, websocket):
        """
        Forward confirmation message to all connectec control connections
        :param message:
        :param websocket:
        :return:
        """
        await self.broadcast_to_control_connections(message)

    def get_leader(self, ):
        """
        Determines based on the votes the current leader
        :return:
        """
        leader = None

        # count votes
        votes_recieved = {}
        for k, v in self.votes.items():
            if v not in votes_recieved.keys():
                votes_recieved[v] = 0
            votes_recieved[v] += 1

        # determine leader
        sorted_votes = sorted(votes_recieved.items(), key=operator.itemgetter(1))
        max_votes = sorted_votes[-1][1]
        if max_votes >= len(self.votes.keys()) / 2.0:
            leader = sorted_votes[-1][0]
        logger.debug("New leader is: {}".format(leader))
        return leader

    async def broadcast_to_control_connections(self, message):
        """
        Broadcast a message to all connected control connections.
        """
        # Send to raft nodes that this instance is connected to as a client
        message['type'] = 'commit_confirm'
        for k, ws in self._connections_to_servers.items():
            try:
                await ws.send(json.dumps(message))
            except Exception as e:
                logger.error("Error while sending to control connection: {}".format(e))

nodes = [
    "127.0.0.1:8007",
    "127.0.0.1:8009",
    "127.0.0.1:8008",
    "127.0.0.1:8010",
    "127.0.0.1:8011",
]

control_channels = [
    "127.0.0.1:8000/ws/blockchain/"
]

async def main():
    ip, port = args.listen.split(":")
    blockchain_path = "/tmp/blockchain_{}:{}.json".format(ip, port)
    async with RaftServer(ip, int(port), nodes, control_channels, blockchain_path) as _:
        pass

if __name__ == '__main__':
    asyncio.get_event_loop().run_until_complete(main())
    asyncio.get_event_loop().run_forever()


import requests
import json
from rely.rely_outputLogs import *
logger = FinalLogger.get_logger()

class RequestMethod(object):

    @staticmethod
    def request_method(method, url, header, parameter):
        print('The request url is : ' + str(url))
        print('The request header is : ' + str(header))
        logger.info('The request url is : ' + str(url))
        logger.info('The request header is : ' + str(header))
        if parameter != '':
            print('The request data is ' + str(method) + ': ' + str(parameter))
            logger.info('The request data is' + str(method) + ': ' + str(parameter))

        if method == '':
            pass
        else:
            if parameter == '':
                data = requests.request(method, url, headers=header, data={})
                if data.text == '':
                    return data
                else:
                    print('The response data is: ' + str(json.loads(data.text)))
                    return data
            else:
                data = requests.request(method, url, headers=header, data=parameter)
                if data.text == '':
                    return data
                else:
                    print('The response data is: ' + str(json.loads(data.text)))
                    return data


import logging

from yodeploy.hooks.daemon import DaemonApp
from yodeploy.hooks.data import DataDirApp
from yodeploy.hooks.django import ApacheHostedDjangoApp
from yodeploy.hooks.htpasswd import AuthenticatedApp
from yodeploy.util import touch

log = logging.getLogger(__name__)

class Hooks(ApacheHostedDjangoApp, AuthenticatedApp, DaemonApp, DataDirApp):
    migrate_on_deploy = True
    has_static = True

    def prepare(self):
        super(Hooks, self).prepare()
        touch(
            self.config['yolapi'].path.uwsgi_log,
            self.log_user,
            self.log_group,
            0o640,
        )

    def apache_hosted_prepare(self):
        super(Hooks, self).apache_hosted_prepare()
        self.template(
            'uwsgi/yolapi.ini.template',
            '/etc/uwsgi/yolapi.ini'
        )

hooks = Hooks


from __future__ import print_function
import datetime
import json
import os
import sys
import requests

import shutil

from tqdm import tqdm
from requests_toolbelt.threaded import pool
from fifascraper.utils import extract_file_name_from_url, threaded

# TODO Move structure to the following
#

_FIFA_CONFIG = {
    '2018': dict(
        guid='B1BA185F-AD7C-4128-8A64-746DE4EC5A82',
        max=9999999,
        zfill=7
    ),
    '2017': dict(
        guid='CC8267B6-0817-4842-BB6A-A20F88B05418',
        max=9999999,
        zfill=7
    )
}

def build_url(type, year, item_id):
    valid_years = [int(year_key) for year_key in _FIFA_CONFIG.keys()]
    if year not in valid_years:
        raise ValueError(
            "Year must be one of the following: {0}".format(','.join(str(_valid_year) for _valid_year in valid_years)))

    if year is 2018:
        # todo implement the rest of things in 2018
        if type.lower() not in ['set', 'challenge']:
            raise Exception("Unable to build a url outside of sets and challenges for 2018")

    year_guid = _FIFA_CONFIG[str(year)]

    base_url = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/{guid}/{year}/fut/sbc/companion/{type}s/images/sbc_{type}_image_{id}.png"
    return base_url.format(
        guid=year_guid,
        year=year,
        type=type,
        id=item_id
    )

# buildUrl(Item.SET,2017,min_range=000000,max_range=999999)

_FIFA_18_SET_URL = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/B1BA185F-AD7C-4128-8A64-746DE4EC5A82/2018/fut/sbc/companion/sets/images/sbc_set_image_{id}.png"
_FIFA_18_SET_MAX_ID = 9999999
_FIFA_18_SET_START_ID = 0000000
_FIFA_18_ZFILL = 7
_FIFA_18_CHALLENGE_URL = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/B1BA185F-AD7C-4128-8A64-746DE4EC5A82/2018/fut/sbc/companion/challenges/images/sbc_challenge_image_{id}.png"

_FIFA_17_SET_URL = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/CC8267B6-0817-4842-BB6A-A20F88B05418/2017/fut/sbc/companion/sets/images/sbc_set_image_{id}000.png"
_FIFA_17_CHALLENGE_URL = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/CC8267B6-0817-4842-BB6A-A20F88B05418/2017/fut/sbc/companion/challenges/images/sbc_challenge_image_1023{id}.png"
_FIFA_17_BACKGROUND_URL = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/CC8267B6-0817-4842-BB6A-A20F88B05418/2017/fut/packs/images/companion/packs_backgrounds_{id}.png"
_FIFA_17_POTM_URL = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/CC8267B6-0817-4842-BB6A-A20F88B05418/2017/fut/sbc/companion/challenges/images/sbc_challenge_image_1027{id}.png"
_FIFA_17_LEGENDS_URL = "https://fifa17.content.easports.com/fifa/fltOnlineAssets/CC8267B6-0817-4842-BB6A-A20F88B05418/2017/fut/sbc/companion/challenges/images/sbc_challenge_image_1021{id}.png"

CONCURRENT_REQUESTS = 100

class Scraper(object):
    def __init__(self):
        self.app_name = "FifaScraper"
        self.config_file = "results.json"
        self.files_folder_name = "images"
        self.backup_folder_name = "backups"
        self.config_folder = None
        self.config_location = None
        self.files_folder = None
        self.results = {}

        self.configuration_backup_folder_location = None

        self.stop_requested = False

        self.collected_data = {}

    def init_config(self):
        if sys.platform == "win32":
            app_data_folder = os.path.join(os.environ.get('APPDATA'), self.app_name)
        else:
            app_data_folder = os.path.expanduser(os.path.join("~", ".{name}".format(name=self.app_name)))

        self.configuration_backup_folder_location = os.path.join(app_data_folder, "backups")

        if not os.path.exists(app_data_folder):
            os.makedirs(app_data_folder)
            print("Created app data folder @ {location}".format(location=app_data_folder))

        # Create the configursation backup folder!
        if not os.path.exists(self.configuration_backup_folder_location):
            os.makedirs(self.configuration_backup_folder_location)
            print("Created backups folder!")

        self.files_folder = os.path.join(app_data_folder, self.files_folder_name)

        if not os.path.exists(self.files_folder):
            os.makedirs(self.files_folder)
            print("Created files folder @ {location}".format(location=self.files_folder))

        self.config_location = os.path.join(app_data_folder, self.config_file)

        if not os.path.exists(self.config_location):
            print("No existing results.json file. Will be created upon run.")
            return

        with open(self.config_location, encoding='utf-8') as data_file:
            self.results = json.load(data_file)

        print("Read results from json file")

    def save_config(self):
        if self.results is None:
            raise TypeError("Unable to save null results to file")

        with open(self.config_location, 'w', encoding='utf-8') as config_file:
            json.dump(self.results, config_file, ensure_ascii=False, indent=4,
                      separators=(',', ':'), sort_keys=True)

        print("Configuration saved to {name}".format(name=self.config_location))

    def create_configuration_backup(self):
        if self.results is None:
            raise TypeError("Unable to save null results backup")

        timestamp = datetime.datetime.now()

        backup_name = os.path.join(self.configuration_backup_folder_location,
                                   "results-backup-{0}.json".format(timestamp))
        shutil.copy(self.config_location, backup_name)
        print("Results Backup Created @ {0}".format(backup_name))

        return backup_name

    def event_exists(self, id):
        return id in self.results['sets'] or id in self.results['challenges']

    def email_results(self, collected=None):
        from fifascraper.envelopes import Envelope, GMailSMTP

        envelope = Envelope(
            from_addr=("fifascraperresults@gmail.com", "Fifa Results"),
            to_addr=("fifascraperresults@gmail.com", 'Steven Hayward'),
            # to_addr=("Steven.hayward89@gmail.com", 'Steven Hayward'),
            subject="Fifa Scraper Results {:%B %d, %Y}".format(datetime.datetime.now()),
            text_body="Collected Ids:\n{ids}".format(ids="\n".join(collected.keys()))
        )

        if len(collected) <= 0:
            print("No new results, skipping email")
            return

        for image in collected.values():
            print("Attaching image @ {0}".format(image))
            envelope.add_attachment(image)

        # Create a backup of the configuration file.
        backup_name = self.create_configuration_backup()

        envelope.add_attachment(backup_name)

        print("Sending Email")
        gmail = GMailSMTP('fifascraperresults@gmail.com', 'upwork2017')
        gmail.send(envelope)
        print("Email Sent")

    def collect_set_data(self, year=2018):
        # Iterate Sets
        print("Generating Set Requests")

        set_urls = []

        set_data_expected = {}

        collected = {

        }

        _max = _FIFA_CONFIG[str(year)]['max']
        _zfill = _FIFA_CONFIG[str(year)]['zfill']

        for set_id in tqdm(range(0, _max)):
            if self.stop_requested is True:
                break

            item_id = str(set_id).zfill(_FIFA_18_ZFILL)
            set_url = build_url(type='set', year=2018, item_id=item_id)

            if self.event_exists(item_id):
                continue

            data = {
                'event_id': item_id,
                'url': set_url,
                'type': 'sets'
            }

            set_urls.append(set_url)
            set_data_expected[set_url] = data

        set_pool = pool.Pool.from_urls(set_urls, num_processes=20)
        print("Awaiting Set Requests to Finish")
        set_pool.join_all()

        for response in set_pool.responses():
            if response.status_code != requests.codes.ok:
                continue

            url = response.request_kwargs['url']

            data = set_data_expected[url]

            if self.event_exists(data['event_id']):
                continue

            image_file_location = os.path.join(self.files_folder,
                                               extract_file_name_from_url(response.request_kwargs['url']))

            with open(image_file_location, 'wb') as f:
                f.write(response.content)

            # image = Image.open(BytesIO(response.content))
            # image.save(image_file_location)
            self.results['sets'][data['event_id']] = {
                'url': url,
                'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
                'image': image_file_location
            }

            collected[data['event_id']] = image_file_location

        return collected

    def collect_challenge_data(self, year=2018):
        collected = {}

        challenge_pool = None

        challenge_urls = []

        challenge_data_expected = {}

        config = _FIFA_CONFIG[str(year)]

        zfill = config['zfill']
        _max = config['max']

        print("Generating Challenge Reqeusts")
        for cid in tqdm(range(0, _max)):
            if self.stop_requested is True:
                break

            challenge_id = str(cid).zfill(zfill)

            challenge_url = build_url('challenge', year=2018, item_id=challenge_id)

            if self.event_exists(challenge_id):
                continue

            data = {
                'event_id': challenge_id,
                'url': challenge_url,
                'type': 'challenges'
            }

            challenge_urls.append(challenge_url)
            challenge_data_expected[challenge_url] = data

        challenge_pool = pool.Pool.from_urls(challenge_urls, num_processes=20)
        print("Awaiting Challenge Requests to Finish")

        challenge_pool.join_all()

        for response in challenge_pool.responses():
            if response.status_code != requests.codes.ok:
                continue

            url = response.request_kwargs['url']

            data = challenge_data_expected[url]

            if self.event_exists(data['event_id']):
                continue

            image_file_location = os.path.join(self.files_folder,
                                               extract_file_name_from_url(response.request_kwargs['url']))

            with open(image_file_location, 'wb') as f:
                f.write(response.content)

            # image = Image.open(BytesIO(response.content))
            # image.save(image_file_location)
            self.results['challenges'][data['event_id']] = {
                'url': url,
                'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
                'image': image_file_location
            }

            collected[data['event_id']] = image_file_location
        return collected

    def collect_player_of_the_month_data(self):
        collected = {}

        potm_urls = []

        potm_data_expected = {}

        print("Generating Player of the Month Requests")
        for pid in tqdm(range(0, 999)):
            if self.stop_requested is True:
                break

            if pid < 10:
                potm_id = "00{0}".format(pid)

            elif 10 <= pid < 100:
                potm_id = "0{0}".format(pid)

            elif pid >= 100:
                potm_id = "{0}".format(pid)

            potm_url = self._create_player_of_the_month_url(potm_id)

            if self.event_exists(potm_id):
                continue

            data = {
                'event_id': potm_id,
                'url': potm_url,
                'type': 'potm'
            }

            potm_urls.append(potm_url)
            potm_data_expected[potm_url] = data

        potm_pool = pool.Pool.from_urls(potm_urls, num_processes=20)
        print("Awaiting Player of the Month Requests to Finish")

        potm_pool.join_all()

        for response in potm_pool.responses():
            if response.status_code != requests.codes.ok:
                continue

            url = response.request_kwargs['url']

            data = potm_data_expected[url]

            if self.event_exists(data['event_id']):
                continue

            image_file_location = os.path.join(self.files_folder,
                                               extract_file_name_from_url(response.request_kwargs['url']))

            with open(image_file_location, 'wb') as f:
                f.write(response.content)

            # image = Image.open(BytesIO(response.content))
            # image.save(image_file_location)
            self.results['potm'][data['event_id']] = {
                'url': url,
                'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
                'image': image_file_location
            }

            collected['1027{0}'.format(data['event_id'])] = image_file_location

        return collected

    def collect_legends_data(self):
        collected = {}

        legends_urls = []

        legends_data_expected = {}

        print("Generating Legends Requests")
        for lid in tqdm(range(0, 999)):
            if self.stop_requested is True:
                break

            if lid < 10:
                legend_id = "00{0}".format(lid)

            elif 10 <= lid < 100:
                legend_id = "0{0}".format(lid)

            elif lid >= 100:
                legend_id = "{0}".format(lid)

            legends_url = self._create_legends_url(legend_id)

            if self.event_exists(legend_id):
                continue

            data = {
                'event_id': legend_id,
                'url': legends_url,
                'type': 'legends'
            }

            legends_urls.append(legends_url)
            legends_data_expected[legends_url] = data

        legends_pool = pool.Pool.from_urls(legends_urls, num_processes=20)
        print("Awaiting Legends Requests to Finish")

        legends_pool.join_all()

        for response in legends_pool.responses():
            if response.status_code != requests.codes.ok:
                continue

            url = response.request_kwargs['url']

            data = legends_data_expected[url]

            if self.event_exists(data['event_id']):
                continue

            image_file_location = os.path.join(self.files_folder,
                                               extract_file_name_from_url(response.request_kwargs['url']))

            with open(image_file_location, 'wb') as f:
                f.write(response.content)

            # image = Image.open(BytesIO(response.content))
            # image.save(image_file_location)
            self.results['legends'][data['event_id']] = {
                'url': url,
                'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
                'image': image_file_location
            }

            collected['1021{0}'.format(data['event_id'])] = image_file_location

        return collected

    def collect_background_data(self):
        collected = {}

        background_pool = None

        background_urls = []

        background_data_expected = {}

        for background_id in tqdm(range(0, 99)):
            if self.stop_requested is True:
                break

            background_url = self._create_background_url(background_id)

            if self.event_exists(background_id):
                continue

            data = {
                'event_id': background_id,
                'url': background_url,
                'type': 'backgrounds'
            }

            background_urls.append(background_url)
            background_data_expected[background_url] = data

        background_pool = pool.Pool.from_urls(background_urls, num_processes=20)
        print("Awaiting background Requests to Finish")
        background_pool.join_all()

        for response in background_pool.responses():
            if response.status_code != requests.codes.ok:
                continue

            url = response.request_kwargs['url']

            data = background_data_expected[url]

            if self.event_exists(data['event_id']):
                continue

            image_file_location = os.path.join(self.files_folder,
                                               extract_file_name_from_url(response.request_kwargs['url']))

            with open(image_file_location, 'wb') as f:
                f.write(response.content)

            # image = Image.open(BytesIO(response.content))
            # image.save(image_file_location)
            self.results['backgrounds'][data['event_id']] = {
                'url': url,
                'time': "{:%B %d, %Y}".format(datetime.datetime.now()),
                'image': image_file_location
            }

            collected['{0}'.format(data['event_id'])] = image_file_location

        return collected

    def start(self):
        self.stop_requested = False
        self.collected_data = {}

        if 'sets' not in self.results:
            self.results['sets'] = {}

        if 'challenges' not in self.results:
            self.results['challenges'] = {}

        if 'backgrounds' not in self.results:
            self.results['backgrounds'] = {}

        if 'potm' not in self.results:
            self.results['potm'] = {}

        if 'legends' not in self.results:
            self.results['legends'] = {}

        collected = self.collect_set_data(year=2018)

        collected.update(self.collect_challenge_data(year=2018))

        # collected.update(self.collect_background_data())

        # collected.update(self.collect_player_of_the_month_data())

        # collected.update(self.collect_legends_data())

        self.collected_data = collected
        self.finish(collected=self.collected_data)

    def finish(self, collected=None):
        """
        Finish the execution, though allow the script to continue execution if the client wishes to do so.
        Will save the configuration, and email results.
        """

        self.save_config()
        self.email_results(collected=collected if collected is not None else self.collected_data)

    def stop(self, collected=None):
        self.finish(collected=collected if collected is not None else self.collected_data)
        self.stop_requested = True


from django.conf.urls.defaults import *
from django.conf import settings
from django.contrib import admin
from django.contrib.auth.views import login, logout, logout_then_login, password_change
from django.contrib.auth.forms import AuthenticationForm
from haystack.views import SearchView
from wehaveweneed.search.forms import PostSearchForm
from wehaveweneed.web.forms import RegistrationForm

admin.autodiscover()

urlpatterns = patterns('',
    url(r'^accounts/activate/(?P<activation_key>\w+)/$',
        'registration.views.activate',
        {'extra_context': {'auth_form': AuthenticationForm()}},
        name='registration_activate'),
    #url(r'^accounts/register/$', 'registration.views.register', {'form_class': RegistrationForm}),
    url(r'^accounts/settings/$', 'wehaveweneed.web.views.account_settings'),
    url(r'^accounts/', include('registration.urls')),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^api/', include('wehaveweneed.api.urls')),
    url(r'^feeds/', include('wehaveweneed.api.feedurls')),
    url(r'^login/', login, { 'template_name': 'registration/login.html' }),
    url(r'^logout/', logout_then_login ),
    url(r'^haves/(?P<category>[-\w]+)?', 'wehaveweneed.web.views.viewhaves', name='web_viewhaves'),
    url(r'^needs/(?P<category>[-\w]+)?', 'wehaveweneed.web.views.viewneeds', name='web_viewneeds'),
    url(r'^post/', 'wehaveweneed.web.views.post_create', name='web_postcreate'),
    url(r'^search/', SearchView(form_class=PostSearchForm)),
    url(r'^top_needs/$', 'wehaveweneed.web.views.top_needs', name='top_needs'),
    url(r'^view/(?P<id>\d+)/$', 'wehaveweneed.web.views.view_post', name='view_post'),
    url(r'^$', 'wehaveweneed.web.views.home', name="home"),
)

if (settings.DEBUG):
    urlpatterns += patterns('',
        url(r'^%s/(?P<path>.*)$' % settings.MEDIA_URL.strip('/'), 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT}),
    )


import tensorflow as tf
mysession=tf.Session()
import os
import pandas
import h5py
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn import preprocessing
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model, Sequential
from keras import optimizers
import keras.backend as K
from keras.layers import LSTM, Dense, Reshape, TimeDistributed
from keras.utils.layer_utils import print_summary

MAIN_FOLDER = './Data/'
VERIFY_OTHERS= './verify_others/'
VERIFY_VIDEO= './verify_video/'

def load_df(full_filename):
    dtype = {'ip.src': str, 'ip.dst': str, 'ip.proto': float, 'ip.flags': str ,'ip.len': float , 'tcp.srcport': float, 'tcp.dstport': float,'tcp.flags': str, 'udp.dstport':float, 'udp.srcport': float }
    dataset = pandas.read_csv(full_filename, dtype=dtype, sep=",")
    print(dataset.shape)
    #tcp.srcport,tcp.dstport,tcp.flags,udp.dstport,udp.srcport
    dataset[['ip.src']]=dataset[['ip.src']].fillna('NaN')
    dataset[['ip.dst']]=dataset[['ip.dst']].fillna('NaN')
    dataset[['ip.proto']]=dataset[['ip.proto']].fillna(0)
    dataset[['ip.flags']]=dataset[['ip.flags']].fillna('NaN')
    dataset[['ip.len']]=dataset[['ip.len']].fillna(0)
    dataset[['tcp.srcport']]=dataset[['tcp.srcport']].fillna(0)
    dataset[['tcp.dstport']]=dataset[['tcp.dstport']].fillna(0)
    dataset[['tcp.flags']]=dataset[['tcp.flags']].fillna('NaN')
    dataset[['udp.dstport']]=dataset[['udp.dstport']].fillna(0)
    dataset[['udp.srcport']]=dataset[['udp.srcport']].fillna(0)
    dataset.fillna(0)
    le = preprocessing.LabelEncoder()
    #print(dataset)
    dataset = dataset.apply(le.fit_transform)
    #dataset=pandas.get_dummies(dataset, sparse=True)
    dataset.head()
    input_cols = ['ip.src', 'ip.dst', 'ip.proto', 'ip.flags', 'ip.len','tcp.srcport', 'tcp.dstport', 'tcp.flags','udp.dstport', 'udp.srcport']
    return np.array(dataset[input_cols])

def get_training_data(length,path):
    samples=list()
    files = os.listdir(path)
    files_full_path = [path + file for file in files]

    for file in files_full_path:
        df_train = load_df(file)
        #print (df_train)
        # step over the 5,000 in jumps of 200
        for i in range(0, df_train.shape[0], length):
            #grab from i to i + 200
            sample = df_train[i:i + length]
            #print(sample[0])
            #split data into samples with length 200, if there are leftovers smaller than
            # 200 drop them
            if sample.shape[0] == length:
                samples.append(sample)
        print(len(samples))

    sample_array = np.array(samples)
    print(sample_array.shape)
    print(sample_array[0])
    sample_array = sample_array.reshape(len(samples), length, samples[0].shape[1])
    print(sample_array.shape)
    return sample_array

#print(dataset['ip.src'])
#print("data labeled")
"""
y, X = dataset['msg'], dataset[['timestamp', 'sig_generator', 'sig_id', 'sig_rev', 'proto', 'src', 'srcport', 'dst', 'dstport', 'ethsrc', 'ethdst', 'ethlen', 'tcpflags', 'tcpseq', 'tcpack', 'tcplen', 'tcpwindow', 'ttl', 'tos', 'id', 'dgmlen', 'iplen', 'icmptype', 'icmpcode', 'icmpid', 'icmpseq']]
#y, X = dataset['Classification'],dataset[['DGMLEN', 'IPLEN', 'IP']].fillna(0)
X_Modeltrain, X_Modeltest, y_Modeltrain, y_Modeltest = train_test_split(X, y, test_size=0.1, random_state=39) #, random_state=39
print("training done")
lr = LogisticRegression()
print("regression done")
lr.fit(X_Modeltrain, y_Modeltrain)
print (accuracy_score(y_Modeltest, lr.predict(X_Modeltest)))

"""
def main():
    X_Modeltrain=get_training_data(100, MAIN_FOLDER)
    y_Modeltrain=[1] * (X_Modeltrain.shape[0]-420) + [0]*420
    y_Modeltrain=np.array(y_Modeltrain)
    #X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    #print(X.shape)
    X_predict_others = list(get_training_data(100, VERIFY_OTHERS))
    y_predict_others = [0]*len(X_predict_others)
    X_predict_video = list(get_training_data(100, VERIFY_VIDEO))
    y_predict_video = [1] * len(X_predict_video)
    X_Modeltest = np.array(X_predict_others + X_predict_video)
    y_Modeltest = np.array(y_predict_others + y_predict_video)
    print("Xtrain shape:", X_Modeltrain.shape)
    print("ymodeltest shape: " ,y_Modeltrain.shape)
    #lr = LogisticRegression()
    #lr.fit(X_Modeltrain, y_Modeltrain)
    input_shape = (X_Modeltrain.shape[1], X_Modeltrain.shape[2])
    #clf = LinearDiscriminantAnalysis(solver='lsqr', shrinkage='auto')
    #clf.fit(X_Modeltrain, y_Modeltrain)
    #print(accuracy_score(y_Modeltest, clf.predict(X_Modeltest)))
    model= create_LSTM_model(300, input_shape)
    history=fit_model(model,X_Modeltrain, y_Modeltrain, 1, 3)
    # serialize model to JSON

    #history.save('./LSTM_first_try.h5')
    scores = model.evaluate(X_Modeltest, y_Modeltest, verbose=0)
    print("Accuracy: %.2f%%" % (scores[1] * 100))
    prediction = history.model.predict(X_Modeltest[0].reshape(1,X_Modeltrain.shape[1],X_Modeltrain.shape[2]), verbose=1)
    print("prediction: ", prediction)
    print("actually: ", y_Modeltest[0])

def create_LSTM_model(number_of_units,input_shape):
    model = Sequential()

    RMSprop = optimizers.RMSprop(lr=0.05)
    # sgd=optimizers.sgd(lr=0.05)

    model.add(LSTM(number_of_units, return_sequences=False, input_shape=(input_shape), dropout=0.02))
    # Try LSTM with n layers and reshape output
    # model.add(Reshape((2, )))

    ####################  Change Dense according to desired output dimensions #################
    model.add(Dense(1))
    model.compile(loss='mean_squared_error',
                  optimizer='sgd',
                  metrics=['accuracy'],
                  )
    #print(samples.shape)
    print_summary(model)
    return model

def fit_model(model, X_train, y_train, batch_size, nb_epoch):
    model.fit(X_train, y_train,
                        batch_size=batch_size, nb_epoch=nb_epoch,
                        verbose=1)
    model_json = model.to_json()
    with open("model_test_reduce_overfitting.json", "w") as json_file:
        json_file.write(model_json)
    # serialize weights to HDF5
    model.save_weights("test_model_reduce_overfitting.h5")
    print("Saved model to disk")
    return model

if __name__ == "__main__":
    main()


#!/usr/bin/python2.7.6

__credits__=  "Dr.Anil prabhakar"
__date__=     " April/June 2018"
#######################Description #####################
'''
    This Class room clicker software is designed for interactive class room learning, This software is developed for getting data from msp430
    microcontroller through RF module and plot bar graph in GUI.

    ~~~~~~~~~~~~~~~~~ This code is for Windows 10( x64 ) supporting BLE ~~~~~~~~~~~~~~~~~~~~

'''
################Loding Modules###########################
import os
# import pprint
# import random
import time
import wx
# import sys
# import matplotlib.pyplot as plt
# import serial
import numpy as np
# import matplotlib
from pylab import figure, table, show, axis, title
#matplotlib.use('WXAgg')
from matplotlib.figure import Figure
from matplotlib.backends.backend_wxagg import \
    FigureCanvasWxAgg as FigCanvas, \
    NavigationToolbar2WxAgg as NavigationToolbar

###########################modified  ###########
#import Tkinter
#from Tkinter import *
#import thread
#from threading import Thread
# import traceback
#import tkMessageBox
# from bluepy.btle import Scanner, DefaultDelegate
# from bluepy.btle import UUID, Peripheral
# import time
#from tkFileDialog import askopenfilename
# import os, sys
import subprocess as sp

tot_devices=['e6:c6:15:3d:db:df','dc:8a:19:f8:14:aa', 'f4:36:67:20:f8:6d','fc:77:48:ab:85:c6','ee:46:b2:5f:3d:c4']
tot={'e6:c6:15:3d:db:df':'Click1','dc:8a:19:f8:14:aa':'Click2','f4:36:67:20:f8:6d':'Click3','fc:77:48:ab:85:c6':'Click4','ee:46:b2:5f:3d:c4':'Click5'} ##Name for each Device
found_devices=[]
avail_devices = []
#root=Tk()
global answer

def bt_add_from_ull( device_address_ull ):
    device_address_hex = hex( device_address_ull )[ 2: ]        # removed the '0x' from left end of hex address
    # print( device_address_hex )

    # device_address -> address of device in bluetooth address format as required
    device_address = str()
    for i in range( int( len(device_address_hex) / 2 ) ):
        device_address += device_address_hex[ i*2 : i*2 + 2 ] + ':'
    device_address = device_address[ : -1 ]                         # removed the extra ':' from right end
    # print( device_address )
    return device_address

def ull_from_bt_add( bt_add ):
    hex_add = bt_add.replace( ':', '' )
    ull_add = int( hex_add, 16 )
    return ull_add

class BarsFrame(wx.Frame):                                        #Creating bar graph frame for 20 users As of now we have 5 devices only
    title = 'Class Room Clicker'
    def btscan(self,event):
        print('scan_button_pressed')
        global avail_devices
        global found_devices

        ############################################################################
        # class ScanDelegate(DefaultDelegate):
        #     def _init_(self):
        #         DefaultDelegate._init_(self)
        #     def handleDiscovery(self, dev, isNewDev, isNewData):
        #         if isNewDev:
        #             print ("Discovered device", dev.addr)
        #         elif isNewData:
        #             print ("Received new data from", dev.addr )
        # scanner = Scanner().withDelegate(ScanDelegate())
        # devices = scanner.scan(10.0)
        # for dev in devices:
        #     for (adtype, desc, value) in dev.getScanData():
        #         #print ("  %s = %s" % (desc, value) )
        #         if 'RFduino' in value:      #Filter Only RFduino Device
        #             found_devices.append(dev.addr)

        discover_exe = 'Modules\\exe\\discover.exe'
        child_process = sp.Popen( discover_exe, stdout = sp.PIPE )
        output_bytes = child_process.communicate()[0]
        # print( output_bytes )
        output_str = output_bytes.decode('utf-8')
        # print( output_str )

        # split into separate device addresses
        device_list_str = output_str.splitlines()
        print( device_list_str )

        device_list_ull = [ int( i ) for i in device_list_str ]         # list of device addresses in unsigned long long format

        found_devices = [ bt_add_from_ull( i ) for i in device_list_ull ]

        #####################################################

        print('Found Devices are :',found_devices )
        print('Total Devices list are \n',tot_devices )
        # print('scanned devices list are \n',found_devices )
        avail_devices = list( set(tot_devices) & set(found_devices) )
        print('Available Devices are \n',avail_devices )
        wx.MessageBox('Scan completed', 'Info',
                wx.OK | wx.ICON_INFORMATION)

    def __init__(self):#Initialize variables
        wx.Frame.__init__(self, None, -1, self.title)

        #Labels creat for table

        self.labelc = [ 'A', 'B', 'C', 'D', 'LOP','Vmon']
        self.labelr = ['ID', '1', '2', '3', '4', '5', '6', '7', '8', '9',
              '10', '11', '12', '13', '14', '15', '16', '17', '18', '19','20']
        self.chars = [['' for c in range(6)] for r in range(21)]
        self.colors = [[(0.95, 0.95, 0.95) for c in range(6)] for r in range(21)]
        self.qqq=1

        self.wloop=1
        self.totg = [0, 0, 0, 0]
        self.ao=[0]*20
        self.bo=[0]*20
        self.co=[0]*20
        self.do=[0]*20
        self.vv=[0]*20
        self.lastopt=[0]*20

        self.create_menu()
        self.create_status_bar()
        self.create_main_panel()
        self.textbox.SetValue(' '.join(map(str, 't(sec)')))
        self.textbox2.SetValue(' '.join(map(str, 'question')))

        self.draw_figure()

    def serread(self,event):#creating file,read till timeout and write to CRC.log file
        self.logfile=open("CRC.log","a")                   #log file is created
        self.logfile.write("\n##")
        self.logfile.write( time.strftime("%d/%m/%Y"))
        self.logfile.write("   ")

        self.logfile.write( time.strftime("%H:%M:%S"))
        self.logfile.write("##\n")
        self.logfile.write("#Q")
        self.logfile.write(str(self.qqq))
        self.logfile.write(":")

        self.logfile.write(self.textbox2.GetValue())
        self.logfile.write("?")
        #print ((time.strftime("%d/%m/%Y")))
        #print ((time.strftime("%H:%M:%S")))
        num_user = float(self.textbox.GetValue())
        #global ser
        #self.question=self.textbox2.GetValue()
        #print(self.question)
        global avail_devices
        try:

            ####################################
            print ( avail_devices )
        except Exception:
            answer=0
        except:
            print ("Device was not connected")

        timeout = int(time.time()) + int(num_user)
        while(self.wloop==1):
            if(time.time()>timeout):
                self.wloop=0
            else:
            #    print (avail_devices )
                for i in range(len(avail_devices)):
                    # print (avail_devices[i])
                    #####################################################
                    # read_uuid = UUID(0x2221)
                    # write_wuuid = UUID(0x2222)
                    # p = Peripheral(avail_devices[i],"random")
                    # ch = p.getCharacteristics(uuid=read_uuid)[0]
                    # chw = p.getCharacteristics(uuid=write_wuuid)[0]

                    # val=0
                    # if (ch.supportsRead()):
                    #     val=0
                    #     answer=''
                    #     val=chw.write('ff') #command for read the data from the device.
                    #     time.sleep(2)
                    #     val = ch.read()
                    #     value = ord(val)

                    ##### NOTE: as coded in exe file, the exe sleeps for 1 sec

                    answer = ''
                    write_to_characteristic = 'ff'              # The string parameter that will be written to the write characteristic

                    address_ull = ull_from_bt_add( avail_devices[ i ] )

                    connectAndRead_exe = 'Modules\\exe\\ConnectAndRead.exe'
                    child_process = sp.Popen( [ connectAndRead_exe, str( address_ull ), write_to_characteristic ], stdout = sp.PIPE )
                    output_bytes = child_process.communicate()[0]
                    # print( output_bytes )
                    output_str = output_bytes.decode('utf-8')
                    # print( output_str )                       # is of type '3\r\n'

                    value = int( output_str )

                    ####################################

                    if value==1:
                        answer= 'A'
                    elif value==2:
                        answer= 'B'
                    elif value==3:
                        answer= 'C'
                    elif value==4:
                        answer= 'D'
                    else:
                        answer='0'
                    print (tot.get(avail_devices[i]),answer )

                    volt=float(4.4)/10
                    #self.data = [int(self.serdata[0:2]), str(self.serdata[2]),volt]
                    self.data = [int(i), str(answer),volt]   #Device name, Option, volt
                    #print (self.data[0] )
                    #print(type(self.data[1]))
                    if(self.data[1]=='A'):
                        self.ao[((self.data[0])-1)]=self.ao[((self.data[0])-1)]+1
                        self.lastopt[((self.data[0])-1)]=self.data[1]
                        self.vv[((self.data[0])-1)]=self.data[2]

                    if(self.data[1]=='B'):
                        self.bo[((self.data[0])-1)]=self.bo[((self.data[0])-1)]+1
                        self.lastopt[((self.data[0])-1)]=self.data[1]
                        self.vv[((self.data[0])-1)]=self.data[2]

                    if(self.data[1]=='C'):
                        self.co[((self.data[0])-1)]=self.co[((self.data[0])-1)]+1
                        self.lastopt[((self.data[0])-1)]=self.data[1]
                        self.vv[((self.data[0])-1)]=self.data[2]

                    if(self.data[1]=='D'):
                        self.do[((self.data[0])-1)]=self.do[((self.data[0])-1)]+1
                        self.lastopt[((self.data[0])-1)]=self.data[1]
                        self.vv[((self.data[0])-1)]=self.data[2]

        self.atot=sum(self.ao)
        self.btot=sum(self.bo)
        self.ctot=sum(self.co)
        self.dtot=sum(self.do)
        self.totg=[self.atot,self.btot,self.ctot,self.dtot]
        #self.ser.flushInput()
        #self.ser.close()
        self.draw_figure()
        #print('10sec')
        print(self.bo)
                #print ("Reopen this gui" )
        for kkkk in range(0,20):
            lev=kkkk
            self.logfile.write("\n")
            self.logfile.write(str(lev+1))
            self.logfile.write(",")
            self.logfile.write(str(self.lastopt[kkkk]))
        #self.logfile.write("\n")

        self.logfile.close()
        self.qqq=self.qqq+1

    def create_menu(self):###Menu bar is created
        self.menubar = wx.MenuBar()

        menu_file = wx.Menu()
        m_expt = menu_file.Append(-1, "&Save plot\tCtrl-S", "Save plot to file")
        self.Bind(wx.EVT_MENU, self.on_save_plot, m_expt)
        menu_file.AppendSeparator()
        m_exit = menu_file.Append(-1, "E&xit\tCtrl-X", "Exit")
        self.Bind(wx.EVT_MENU, self.on_exit, m_exit)

        menu_help = wx.Menu()
        m_about = menu_help.Append(-1, "&About\tF1", "About the classe room clicker")
        self.Bind(wx.EVT_MENU, self.on_about, m_about)

        self.menubar.Append(menu_file, "&File")
        self.menubar.Append(menu_help, "&Help")
        self.SetMenuBar(self.menubar)

    def create_main_panel(self):# Crearing the elements of the GUI

        self.panel = wx.Panel(self)

        # Create the mpl Figure and FigCanvas objects.
        # 5x4 inches, 100 dots-per-inch
        #
        self.dpi = 100
        self.fig = Figure((5.0, 4.0), dpi=self.dpi)
        self.canvas = FigCanvas(self.panel, -1, self.fig)
        self.textbox2 = wx.TextCtrl(
            self.panel,
            size=(900,-1),
            style=wx.TE_PROCESS_ENTER)

        self.axes = self.fig.add_subplot(111)

        # Bind the 'pick' event for clicking on one of the bars
        #
        self.canvas.mpl_connect('pick_event', self.on_pick)

        #self.Bind(wx.EVT_TEXT_ENTER, self.on_text_enter, self.textbox2)
        self.drawbutton1 = wx.Button(self.panel, -1, "Scan")
        self.Bind(wx.EVT_BUTTON,  self.btscan, self.drawbutton1)
        self.textbox = wx.TextCtrl(
            self.panel,
            size=(60,-1),
            style=wx.TE_PROCESS_ENTER)
        self.Bind(wx.EVT_TEXT_ENTER, self.on_text_enter, self.textbox)
        self.drawbutton = wx.Button(self.panel, -1, "Start")
        self.Bind(wx.EVT_BUTTON, self.serread, self.drawbutton)
        self.attbutton = wx.Button(self.panel, -1, "Init")
        self.Bind(wx.EVT_BUTTON, self.init, self.attbutton)
        self.donebutton = wx.Button(self.panel, -1, "Result")
        self.Bind(wx.EVT_BUTTON, self.result, self.donebutton)
        self.closebutton = wx.Button(self.panel, -1, "Close")
        self.Bind(wx.EVT_BUTTON, self.closebtt, self.closebutton)
        self.plotlopbut = wx.Button(self.panel, -1, "L_opt")
        self.Bind(wx.EVT_BUTTON, self.plotlop, self.plotlopbut)
        self.totbt = wx.Button(self.panel, -1, "Total")
        self.Bind(wx.EVT_BUTTON, self.totfun, self.totbt)
        #self.cb_grid = wx.CheckBox(self.panel, -1,
        #    "Show Grid",
        #    style=wx.ALIGN_RIGHT)
        #self.Bind(wx.EVT_CHECKBOX, self.on_cb_grid, self.cb_grid)

        #self.slider_label = wx.StaticText(self.panel, -1,
        #    "width")
        #self.slider_width = wx.Slider(self.panel, -1,
        #    value=20,
        #    minValue=1,
        #    maxValue=100,
        #   style=wx.SL_AUTOTICKS | wx.SL_LABELS)
        #self.slider_width.SetTickFreq(10, 1)
        #self.Bind(wx.EVT_COMMAND_SCROLL_THUMBTRACK, self.on_slider_width, self.slider_width)

        self.toolbar = NavigationToolbar(self.canvas)

        #
        # Layout with box sizers
        #
        self.qbox = wx.BoxSizer(wx.HORIZONTAL)
        self.vbox = wx.BoxSizer(wx.VERTICAL)
        flags = wx.ALIGN_LEFT | wx.ALL | wx.ALIGN_CENTER_VERTICAL

        self.vbox.Add(self.canvas, 1, wx.TOP | wx.GROW)
        self.vbox.Add(self.toolbar, 0, wx.EXPAND)
        self.vbox.AddSpacer(10)

        self.hbox = wx.BoxSizer(wx.HORIZONTAL)

        self.qbox.AddSpacer(10)

        self.hbox.Add(self.drawbutton1, 0, border=3, flag=flags)
        self.hbox.Add(self.textbox, 0, border=3, flag=flags)
        self.hbox.Add(self.drawbutton, 0, border=3, flag=flags)
        self.hbox.Add(self.donebutton, 0, border=3, flag=flags)
        self.hbox.Add(self.totbt, 0, border=3, flag=flags)

        self.hbox.Add(self.plotlopbut, 0, border=3, flag=flags)
        self.hbox.Add(self.attbutton, 0, border=3, flag=flags)
        self.hbox.Add(self.closebutton, 0, border=3, flag=flags)
        self.qbox.Add(self.textbox2, 0, border=3, flag=flags)

        #self.hbox.Add(self.cb_grid, 0, border=3, flag=flags)
        self.hbox.AddSpacer(30)
        #self.hbox.Add(self.slider_label, 0, flag=flags)
        #self.hbox.Add(self.slider_width, 0, border=3, flag=flags)

        self.vbox.Add(self.hbox, 0, flag = wx.ALIGN_LEFT | wx.TOP)
        self.vbox.Add(self.qbox, 0, flag = wx.ALIGN_LEFT | wx.TOP)
        self.panel.SetSizer(self.vbox)
        self.vbox.Fit(self)

    def totfun(self,event):#Button event for raw option
        self.draw_figure()
    def init(self,event):# To reuse the GUI Every variables need to be reinit
        self.chars = [['' for c in range(6)] for r in range(21)]
        self.wloop=1
        self.totg = [0, 0, 0, 0]
        self.ao=[0]*20
        self.bo=[0]*20
        self.co=[0]*20
        self.do=[0]*20
        self.vv=[0]*20
        self.lastopt=[0]*20

        app.frame = BarsFrame()
        print ("att")
        x = range(4)
        #xy=['a','b','c','d']
        ind = np.arange(4)
        width=0
        self.totg=[0,0,0,0]
        # clear the axes and redraw the plot anew
        #
        self.axes.clear()
        #self.axes.xticks(range(len(xy)),xy,size='small')
        #self.axes.grid(self.cb_grid.IsChecked())

        self.axes.bar(left=x,height=self.totg,
            width=1,
            align='center',color='y')
        self.axes.set_title('Histogram')
        self.axes.set_xlabel('Options')
        self.axes.set_ylabel('level')
        self.axes.set_xticks(ind+width)
        self.axes.set_xticklabels( ('A', 'B', 'C', 'D') )
        #self.ser.close()
        #ab=self.axes.set_xticklabels(['.','A', '.', 'B','.','C', '.','D'])

        self.canvas.draw()
    def plotlop(self,event):# To find the last option given by all the 20 users
        print('enter')
        x = range(4)
        self.lastoptA=0
        self.lastoptB=0
        self.lastoptC=0
        self.lastoptD=0
        #xy=['a','b','c','d']
        ind = np.arange(4)
        width=0

        # clear the axes and redraw the plot anew
        #
        self.axes.clear()

        for ii in range(0,20):
            if(self.lastopt[ii]=='A'):
                self.lastoptA=self.lastoptA+1
            if(self.lastopt[ii]=='B'):
                self.lastoptB=self.lastoptB+1
            if(self.lastopt[ii]=='C'):
                self.lastoptC=self.lastoptC+1
            if(self.lastopt[ii]=='D'):
                self.lastoptD=self.lastoptD+1

        self.lastoptt=[self.lastoptA,self.lastoptB,self.lastoptC,self.lastoptD]

        #self.axes.xticks(range(len(xy)),xy,size='small')
        #self.axes.grid(self.cb_grid.IsChecked())

        self.axes.bar(left=x,height=self.lastoptt, width=.35, align='center',color='y')
        self.axes.set_title('Histogram')
        self.axes.set_xlabel('Options')
        self.axes.set_ylabel('level')
        self.axes.set_xticks(ind+width)
        self.axes.set_xticklabels( ('A', 'B', 'C', 'D') )

        #ab=self.axes.set_xticklabels(['.','A', '.', 'B','.','C', '.','D'])

        self.canvas.draw()

    def closebtt(self,event):# Closs serial port and other events
        #global ser
        self.Close(True)
        #self.ser.close()

    def create_status_bar(self):#status bar
        self.statusbar = self.CreateStatusBar()

    def draw_figure(self):#Redraws the figure

        #str = self.textbox.GetValue()
        #self.data = map(int, str.split())

        x = range(4)
        #xy=['a','b','c','d']
        ind = np.arange(4)
        width=0

        # clear the axes and redraw the plot anew
        #
        self.axes.clear()
        #self.axes.xticks(range(len(xy)),xy,size='small')
        #self.axes.grid(self.cb_grid.IsChecked())

        self.axes.bar(left=x,height=self.totg, width=.35,  align='center',color='y')
        self.axes.set_title('Histogram')
        self.axes.set_xlabel('Options')
        self.axes.set_ylabel('level')
        self.axes.set_xticks(ind+width)
        self.axes.set_xticklabels( ('A', 'B', 'C', 'D') )

        #ab=self.axes.set_xticklabels(['.','A', '.', 'B','.','C', '.','D'])
        # global ser
        self.canvas.draw()

    def on_cb_grid(self, event):# Grid draw
        self.draw_figure()

    def on_slider_width(self, event):#slider
        self.draw_figure()

    def on_draw_button(self, event):# draw button
        self.figure(figsize=(8, 4), dpi=120)
        #self.data = [3, 1, 0, 1]
        self.draw_figure()
    def result(self, event):# Present the result of 20 user in a table
        #print ("result" )
        for x3 in range(0, 20):

            self.chars[x3+1][0] = self.ao[x3]
        for xx in range(0, 20):

            self.chars[xx+1][1] = self.bo[xx]

        for yy in range(0, 20):

            self.chars[yy+1][2] = self.co[yy]

        for zz in range(0, 20):

            self.chars[zz+1][3] = self.do[zz]

        for x1 in range(0, 20):

            self.chars[x1+1][4] = self.lastopt[x1]

        for x4 in range(0, 20):

            self.chars[x4+1][5] = self.vv[x4]

        for x5 in range(0, 20):
            if(self.vv[x5]>=2.6):

                self.colors[x5+1][5]=(0, 1, 0)
            elif((self.vv[x5]<2.6) and (self.vv[x5]>1)):
                self.colors[x5+1][5]=(1, 0, 0)
            else:
                self.colors[x5+1][5]=(.95, .95, 0.95)

        #for x2 in range(0, 20):

        #    self.chars[x2][5] = self.lastopt[x2]
        #self.chars[19][4] = '0'

        lightgrn = (0.5, 0, .9)
        title('Result')
        tab = table(cellText=self.chars,
                    rowLabels=self.labelr,
                    colLabels=self.labelc,
                    rowColours=[lightgrn]*21,
                    colColours=[lightgrn]*6,
                    cellColours=self.colors,
                    cellLoc='center',
                    loc='upper left')

        axis('off')
        show()

    def on_pick(self, event):#Pick single bar for info

        box_points = event.artist.get_bbox().get_points()
        msg = "You've clicked on a bar with coords:\n %s" % box_points

        dlg = wx.MessageDialog(
            self,
            msg,
            "Click!",
            wx.OK | wx.ICON_INFORMATION)

        dlg.ShowModal()
        dlg.Destroy()

    def on_text_enter(self, event):# This function will be called when the text event occer to display in GUI
        self.draw_figure()

    def on_save_plot(self, event):# To save the plot
        file_choices = "PNG (*.png)|*.png"

        dlg = wx.FileDialog(
            self,
            message="Save plot as...",
            defaultDir=os.getcwd(),
            defaultFile="plot.png",
            wildcard=file_choices,
            style=wx.SAVE)

        if dlg.ShowModal() == wx.ID_OK:
            path = dlg.GetPath()
            self.canvas.print_figure(path, dpi=self.dpi)
            self.flash_status_message("Saved to %s" % path)

    def on_exit(self, event):#  Exit button
        self.Destroy()

    def on_about(self, event):#Help menu
        msg = """ Class Room Clicker:

         * This Application is created for intrective class room learning

        """
        dlg = wx.MessageDialog(self, msg, "About", wx.OK)
        dlg.ShowModal()
        dlg.Destroy()

    def flash_status_message(self, msg, flash_len_ms=1500):# Life time of status message
        self.statusbar.SetStatusText(msg)
        self.timeroff = wx.Timer(self)
        self.Bind(
            wx.EVT_TIMER,
            self.on_flash_status_off,
            self.timeroff)
        self.timeroff.Start(flash_len_ms, oneShot=True)

    def on_flash_status_off(self, event):
        self.statusbar.SetStatusText('')

if __name__ == '__main__':
    app = wx.App()
    app.frame = BarsFrame()
    app.frame.Show()
    app.MainLoop()


import argparse
from urllib.request import urlopen
import urllib.request
from urllib.parse import urlparse
from bs4 import BeautifulSoup
import re

'''
Input :  url
Output : formatted url with prefixes that fits urllib library (format : http://www.)
'''

def get_valid_url(url):
    if url.find("www.") == -1:  # case prefix www. missing
        url = "www." + url
    if not urlparse(url).scheme:  # case prefix http:// missing
        url = "http://" + url
    return url

'''
Input : user input - string
Output : validate format return error if or wrong format, if successful return 0 for URL, return 1 for valid file
'''

def check_file_or_url(parameter):
    try:
        with open(parameter, "r"):
            return 1
    except IOError:
        parameter = get_valid_url(parameter)
    try:
        with urllib.request.urlopen(parameter) as web_page:
            if web_page.getcode() != 200:  # Status not OK
                print("Unsuccessful request to server status code is : " + web_page.getcode() + "Please try again")
                exit(0)
        # case invalid URL
    except Exception:
        print("Invalid parameters please enter a valid URL or file path.")
        exit(0)
    return 0

'''
Implement by DFS for with depth of "max_depth, write a href links to result_file
Input : URL with prefixes (http://www.), depth of DFS,
Output : scrap that URL page with maximum level of "max_depth" write all hyperlinks to text
        ASSUMPTION : all hyperlinks has prefix http// or https//
        also ignore all possible errors and move to next link.
'''

def dfs_recursive_by_level(url, max_depth, current_level, result_file):
    if current_level < max_depth:
        try:
            with urllib.request.urlopen(url) as web_page:
                if web_page.getcode() == 200:  # Status code OK
                    all_links = BeautifulSoup(web_page.read(), 'lxml')
                    for link in all_links.findAll('a', attrs={'href': re.compile("^http://www.|^https://www.")}):
                        href_link = link.get('href')
                        result_file.write(href_link + "\n")
                        dfs_recursive_by_level(href_link, max_depth, current_level + 1, result_file)
        except Exception:
            pass

'''
Used to call DFS recursive by level function, with default depth is 2
Input : URL with prefixes (http://www.), result file to write href links, depth of DFS,
Output : result file with all href links
'''

def scrap_url_page(url, result_file, max_depth=2):
    dfs_recursive_by_level(url, max_depth, 0, result_file)

def main():
    # get arguments from user
    parser = argparse.ArgumentParser("user_parameters")
    parser.add_argument("user_param", help="text file or web address")
    args = parser.parse_args()
    # check if file or website url
    result = check_file_or_url(args.user_param)
    # create result file for scrap hyperlinks
    with open("scrap_result.txt", "w") as result_file:
        # Case URL
        if result == 0:
            # case missing prefix in url
            url = get_valid_url(args.user_param)
            # scrap this url
            print("scraping for website: " + url)
            scrap_url_page(url, result_file)

        # Case file
        if result == 1:
            # read from file
            with open(args.user_param, "r") as scrap_file:
                line = scrap_file.readline()
                # scrap using url file
                while line:
                    if line == '\n':  # case newline
                        line = scrap_file.readline()
                        continue
                    line = line.replace(' ', '')  # remove spaces
                    if check_file_or_url(line) == 0:
                        url = get_valid_url(line)
                        print("currently scraping for website = " + line)
                        scrap_url_page(url, result_file)
                        line = scrap_file.readline()
    print("Scrap finished please check scrap_result.txt located in python file dir for results")

# run main from terminal
if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 16:42:45 2019

@author: ZİŞAN
"""
import pandas as pd

#%% dataset eklenir
data=pd.read_csv(r"gender-classifier.csv", encoding="latin1")
data=pd.concat([data.gender,data.description],axis=1)

#%% eksik ve anlamsız verileri çıkartma
data.dropna(axis=0, inplace=True)#nan degerleri çıkartma
#gruplandırıken string ifadeyi sayısala çevrime
#female -> 1, male ->0
data.gender=[1 if each == "female" else 0 for each in data.gender]

#%% cleaning data
#regular expression RE
import re

first_description=data.description[4]
#a dan z ye ve A dan Z yekadar olan harfleri değiştirme geri
#kalanını boşluğa dönüştür
description=re.sub("[^a-zA-Z]"," ",first_description)
#her harfin boyutunu küçült
description=description.lower()

#%% stopwords (irrelavent words) gereksiz kelimeler
import nltk #natural languafe tool kit
nltk.download("stopwords") #corpus diye klasöre indiriliyor
#from nltk.corpus import stopwords #corpusdan import ediliyor
#description=description.split()
#split yerine tokanizer kullanılıyor
description = nltk.word_tokenize(description)

#%%
#gereksiz kelimleri çıkart
description= [word for word in description if not word in set(stopwords.words("english"))]

#%%
#lemmatization loved->love kelime kökleri
import nltk as nlp
lemma=nlp.WordNetLemmatizer()
description=[lemma.lemmatize(word) for word in description]
#kelimeleri tekrar birleştirdi
description=" ".join(description)
#%%tüm datalar için yapma
description_list=[]
for description in data.description:
    description=re.sub("[^a-zA-Z]"," ",description)
    description=description.lower()
    description = nltk.word_tokenize(description)
    description= [word for word in description if not word in set(stopwords.words("english"))]
    lemma=nlp.WordNetLemmatizer()
    description=[lemma.lemmatize(word) for word in description]
    description=" ".join(description)
    description_list.append(description)

#%%bag of words

from sklearn.feature_extraction.text import CountVectorizer #bag of words yazdırmak için kullanılan vektör
max_features=5000

count_vectorizer=CountVectorizer(max_features=max_features) #parametre olarak stop_words="english" vb seyler alabilr

sparce_matrix=count_vectorizer.fit_transform(description_list).toarray() #x

print("en sık kullanılan {} kelimler: {}".format(max_features,count_vectorizer.get_feature_names()))

#%%
y= data.iloc[:,0].values #male or female
x=sparce_matrix
from sklearn.model_selection import train_test_split #train test split
x_train, x_test, y_train, y_test=train_test_split(x,y, test_size=0.1, random_state=42)

#%%navie bayes

from sklearn.naive_bayes import GaussianNB

nb=GaussianNB()
nb.fit(x_train,y_train)

#prediction
y_pred=nb.predict(x_test)
print("accuracy: ",nb.score(y_pred.reshape(-1,1),y_test))


"""
Plot mimic-related stuff
"""
import sys
import argparse
import logging
import numpy as np
import pandas as pd

import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
import seaborn as sns

from decision_interval_aggregator import DecisionIntervalAggregator
from decision_interval_recalibrator import DecisionIntervalRecalibrator
from dataset import Dataset
from common import pickle_to_file, pickle_from_file, process_params, get_normal_ci, load_model, get_interval_width
from plot_common import *

def plot_accept_prob_vs_missing(fitted_models, dataset, args):
    num_missing_indicators = int((dataset.x.shape[1] - 3)/2)
    print('num missing indicators', num_missing_indicators)
    missingness = np.mean(dataset.x[:, -num_missing_indicators:], axis=1)

    all_dfs = []
    for model_idx, fitted_model in enumerate(fitted_models):
        accept_probs = fitted_model.get_accept_prob(dataset.x)
        df = pd.DataFrame.from_dict({
            "missingness": missingness,
            "accept_prob": accept_probs.flatten(),
            "model_idx": model_idx})
        all_dfs.append(df)
    all_dfs = pd.concat(all_dfs)

    plt.clf()
    sns.lineplot(
            x="missingness",
            y="accept_prob",
            hue="model_idx",
            data=all_dfs)
    plt.savefig(args.out_missing_plot)

def plot_accept_prob_vs_age(fitted_models, dataset, age, args):
    # Age is the first column!
    all_dfs = []
    for model_idx, fitted_model in enumerate(fitted_models):
        accept_probs = fitted_model.get_accept_prob(dataset.x)
        df = pd.DataFrame.from_dict({
            "age": age.flatten(),
            "accept_prob": accept_probs.flatten(),
            "Fold": model_idx})
        all_dfs.append(df)
    all_dfs = pd.concat(all_dfs)

    plt.clf()
    plt.figure(figsize=(4,4))
    sns.set(style="white")
    sns.despine()
    g = sns.lmplot(
            x="age",
            y="accept_prob",
            hue="Fold",
            data=all_dfs,
            lowess=True,
            scatter=False)
    plt.xlabel("Age")
    plt.ylabel("Acceptance probability")
    plt.tight_layout()
    plt.savefig(args.out_age_plot)

def plot_prediction_vs_age(
        fitted_models,
        dataset,
        age,
        args,
        y_label,
        pred_func=None,
        accept_prob_thres=0.5,
        max_predict_plot=False):
    """
    @param pred_func: whether to transform the prediction before plotting
    @param max_predict_plot: whether to only plot things within one std dev of the median
    """
    # Age is the first column!
    all_dfs = []
    for model_idx, fitted_model in enumerate(fitted_models):
        if not fitted_model.has_density:
            alpha_PIs = fitted_model.get_prediction_interval(dataset.x, args.alpha)
            prediction = np.mean(alpha_PIs, axis=1)
        elif fitted_model.density_parametric_form == "bernoulli":
            prediction = fitted_model.get_prediction_probs(dataset.x)[:,0]
        else:
            raise ValueError("not an option")

        accept_probs = fitted_model.get_accept_prob(dataset.x).flatten()
        if max_predict_plot:
            accepted_mask = (accept_probs > accept_prob_thres) * (prediction < np.median(prediction) + np.sqrt(np.var(prediction)))
        else:
            accepted_mask = (accept_probs > accept_prob_thres)
        df = pd.DataFrame.from_dict({
            "Age": age.flatten()[accepted_mask],
            "prediction": prediction[accepted_mask],
            "Fold": model_idx})
        all_dfs.append(df)
    all_dfs = pd.concat(all_dfs)
    if pred_func is not None:
        all_dfs.prediction = pred_func(all_dfs.prediction)

    plt.clf()
    plt.figure(figsize=(4,4))
    sns.set(style="white")
    sns.despine()
    sns.lmplot(
            x="Age",
            y="prediction",
            hue="Fold",
            lowess=True,
            scatter=False,
            data=all_dfs)
    plt.ylabel(y_label)
    plt.tight_layout()
    plt.savefig(args.out_age_pred_plot)

#def plot_local_coverages(aggregator, dataset, args, fontsize=14):
#    """
#    Plot local coverages....
#    """
#    all_local_coverages = []
#    for model_idx, fitted_model in enumerate(aggregator.fitted_models):
#        local_coverages = assess_local_coverage(
#                fitted_model,
#                dataset,
#                alpha=args.alpha,
#                num_rand=args.num_rand,
#                k=args.num_nearest_neighbors)
#        print("local coverages assessed at %d points" % len(local_coverages))
#        all_local_coverages.append(pd.DataFrame.from_dict({
#            "model_idx": model_idx,
#            "local_coverage": local_coverages}))
#    local_agg_coverages = assess_local_agg_coverage(
#            aggregator,
#            dataset,
#            alpha=args.alpha,
#            num_rand=args.num_rand,
#            k=args.num_nearest_neighbors)
#    print(local_agg_coverages)
#    all_local_coverages.append(pd.DataFrame.from_dict({
#        "model_idx": "Agg",
#        "local_coverage": local_coverages}))
#
#    plt.clf()
#    sns.violinplot(
#            x="model_idx",
#            y="local_coverage",
#            data=pd.concat(all_local_coverages),
#            cut=0)
#    plt.xlabel("Fitted Model", fontsize=fontsize)
#    plt.ylabel("Local coverage", fontsize=fontsize)
#    plt.savefig(args.out_local_coverage_plot)


import logging
import os
from threading import Thread

pypath = os.path.dirname(os.path.abspath(__file__))

def get_logger(name):
    log_file = os.path.join(pypath, "log/%s.log" % name)

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    f_handler = logging.FileHandler(log_file,mode='a')
    #s_handler = logging.StreamHandler()

    f_handler.setFormatter(formatter)
    #s_handler.setFormatter(formatter)

    level = logging.DEBUG
    logger = logging.getLogger(name)
    logger.setLevel(level)
    logger.addHandler(f_handler)
    #logger.addHandler(s_handler)

    return logger

def run(name):
    logger = get_logger(name)
    logger.warning('warning from thread_%s' % name)

for i in range(1, 10):
    thread = Thread(target=run, args=("thread_%d" %(i),))
    thread.start()
    thread.join()

print('main thread')


from app import app
from app import db
from flask import render_template, request, redirect, url_for
from forms import *
import datetime
#from werkzeug.utils import secure_filename
#import os
#from tcx_parser import parsetcx
from modules import Moves, UserDetails, Plan
#import pymysql
#pymysql.install_as_MySQLdb

#@app.route('/', methods=['GET', 'POST'])
#def index():
#    form = Uploadform()
#    if request.method == "POST":
##TODO Here to do check of file type and file exist
#            filename = secure_filename(form.fileName.data.filename)
#            form.fileName.data.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
#
#            parsetcx(os.path.join(app.config['UPLOAD_FOLDER'] + filename), 1)
#            return redirect(url_for('uploaded_file'))
##Test moves table.
#    moves =  Moves.query.order_by('Date desc', 'Time desc').limit(10).all()
#    return render_template('index.html', moves = moves, form = form, title = 'Mymoves', Header = 'Mymoves', Small = 'upload .tcx file')

# This is a sample chart with sample.js
#http://localhost/sample_chart
@app.route('/sample_chart')
def sample_chart():
    legend = 'Planned Weekly TSS'
#    labels = ['1', '2', '3', '4', '5', '6', '7', '8', '9']
#    values = [160, 190, 210, 160, 190, 210, 230, 160, 210]

    values = Plan.query.all()

    return render_template('sample_chart.html', values = values, legend = legend)

@app.route('/')
#http://localhost/plan
@app.route('/plan', methods=['POST', 'GET'])
def plan():
    if request.method == 'POST':
        weeknumber = request.form['weeknumber']
        tsscompleted = request.form['tsscompleted']
        try:
            db.session.query(Plan).filter_by(week=weeknumber).update({'tss_compl': tsscompleted})
            db.session.commit()
        except:
            print('can not do this')
        return redirect(url_for('plan'))
    form = UpdatePlan()

    values = Plan.query.all()
    output = []
    total_plantss = 0
    total_compltss = 0
    for v in values:
        total_plantss += v.tss_plan
        total_compltss += v.tss_compl
        atl_plan = round(v.tss_plan / 7)
        atl_compl = round(v.tss_compl / 7)
        tss_plan_sum = 0
        tss_compl_sum = 0
        if v.week > 6:
            for k in range(6):
                tss_plan_sum += values[v.week - k -1].tss_plan
                tss_compl_sum += values[v.week - k -1].tss_compl
            ctl_plan = round(tss_plan_sum / 42)
            ctl_compl = round(tss_compl_sum / 42)
        else:
            for k in range(v.week):
                tss_plan_sum += values[v.week - k -1].tss_plan
                tss_compl_sum += values[v.week - k -1].tss_compl
            ctl_plan = round(tss_plan_sum / 42)
            ctl_compl = round(tss_compl_sum / 42)

        entry = {'week': v.week, 'date': v.date, 'tss_plan': v.tss_plan, 'ctl_plan': ctl_plan, \
                 'atl_plan': atl_plan, 'tss_compl': v.tss_compl, 'ctl_compl': ctl_compl, \
                 'atl_compl': atl_compl, 'training_phase': v.training_phase }
        output.append(entry)

    legend = 'Weekly TSS plan'
    date_today = datetime.date.today()

    return render_template('plan.html', n = output, d = date_today, legend = legend, form=form, tptss=total_plantss, tctss=total_compltss)

#http://localhost/create_plan
@app.route('/create_plan', methods=['POST', 'GET'])
def create_plan():
    if request.method == 'POST':
        startdate = request.form['startdate']
        truestartdate = datetime.datetime.strptime(startdate, '%Y-%m-%d').date()
        tssplanweek = []
        weekphase = []
        for x in range(52):
            tssplanweek.append(request.form['tssplanweek' + str(x)])
            weekphase.append(request.form['weekphase' + str(x)])
        try:

            db.session.query(Plan).delete()
            db.session.commit()
            for x in range(52):
                plan = Plan(date=(truestartdate + datetime.timedelta(days=7)*x), week= x + 1, tss_plan= tssplanweek[x], tss_compl=0, training_phase=weekphase[x])
                db.session.add(plan)
                db.session.commit()

        except:
            print('Can not do this')
        return redirect(url_for('plan'))
    form = CreatePlan()

    return render_template( 'create_plan.html', form=form)

#http://localhost/user
@app.route('/user')
def user():
    userdetail = UserDetails.query.first()
    return render_template('User.html', title = 'User Details', userdetail = userdetail)

#@app.route('/Edit', methods=['POST', 'GET'])
#def Edit():
#    form = UserEdit()
#    if form.validate_on_submit():
#        g.user.Name = form.Name.data
#        g.user.Age = form.Age.data
#        g.user.Weight = form.Weight.data
#        db.session.add(g.user)
#        db.session.commit()
#        flash('Your changes have been saved.')
#        return redirect(url_for('edit'))
#    else:
#        form.Name.data = g.user.Name
#        form.Age.data = g.user.Age
#        form.Weight.data = g.user.Weight
#    return render_template('Edit.html', form = form)

#@app.route('/login')
#def login():
#
#    return render_template("login.html", title = 'Login page', Header = 'Login page', Small = ' test login' )

#@app.route('/upload', methods=['POST', 'GET'])
#def upload():
#    form = Uploadform()
#    if request.method == "POST":
##TODO Here to do check of file type and file exist
#            filename = secure_filename(form.fileName.data.filename)
#            form.fileName.data.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
#
#            parsetcx(os.path.join(app.config['UPLOAD_FOLDER'] + filename), 1)
#            return redirect(url_for('uploaded_file'))
#
#    return render_template("upload.html", form = form, title = 'Upload page', Header = 'Upload file', Small = 'upload .tcx file')

#@app.route('/uploaded_file')
#def uploaded_file():
#    return render_template('uploaded_file.html', title = 'upload successfull')

#def allowed_file(filename):
#    return '.' in filename and \
#        filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS


import random, argparse

from pythonosc import osc_message_builder
from pythonosc import udp_client

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--ip", default="127.0.0.1",
        help="The ip of the OSC server")
    parser.add_argument("--port", type=int, default=7400,
        help="The port the OSC server is listening on")
    args = parser.parse_args()
    client = udp_client.SimpleUDPClient(args.ip, args.port)

freq_range = range(100, 1001) # max val?
freq_number = 16

def make_freqs():
    freqs = random.sample(freq_range, freq_number)
    up_fifths = [i * 1.5 for i in freqs]
    down_fifths = [i / 1.5 for i in freqs]
    up_thirds = [i * 1.25 for i in freqs]
    down_thirds = [i / 1.25 for i in freqs]
    up_sevenths = [i * 1.75 for i in freqs]
    down_sevenths = [i / 1.75 for i in freqs]
    up_elevenths = [i * 1.375 for i in freqs]
    down_elevenths = [i / 1.375 for i in freqs]
    all_freqs = [freqs, up_fifths, down_fifths, up_thirds, down_thirds,
                 up_sevenths, down_sevenths, up_elevenths, down_elevenths]
    return all_freqs

all_freqs = make_freqs()

options = '1. Original freq list 2. 1:1 3. 3:2 4. 2:3 5. 5:4 6. 4:5 7. 7:4 ' \
          '8. 4:7 9. 11:8 10. 8:11 11. reset freqs'

print(all_freqs[0])

while True:

    send_freqs = input(options + " ")
    if send_freqs == '1': # send original freq list
        client.send_message('freqs', all_freqs[0])
    elif send_freqs == '2': # send unison freqs
        client.send_message('harm', all_freqs[0])
    elif send_freqs == '3': # send up_fifths
        client.send_message('harm', all_freqs[1])
    elif send_freqs == '4': # send down_fifths
        client.send_message('harm', all_freqs[2])
    elif send_freqs == '5': # send up_thirds
        client.send_message('harm', all_freqs[3])
    elif send_freqs == '6': # send down_thirds
        client.send_message('harm', all_freqs[4])
    elif send_freqs == '7': # send up_sevenths
        client.send_message('harm', all_freqs[5])
    elif send_freqs == '8': # send down_sevenths
        client.send_message('harm', all_freqs[6])
    elif send_freqs == '9': # send up_elevenths
        client.send_message('harm', all_freqs[7])
    elif send_freqs == '10': # send down_elevenths
        client.send_message('harm', all_freqs[8])
    elif send_freqs == '11': # reset freqs
        all_freqs = make_freqs()
        print(all_freqs[0])
    else:
        break


from django.urls import path

from .views import StatusListAPIView, StatusDetailAPIView, StatusCreateAPIView, StatusUpdateAPIView, StatusDeleteAPIView

urlpatterns = [
  path('', StatusListAPIView.as_view()),
  path('<int:pk>/', StatusDetailAPIView.as_view()),
  path('create/', StatusCreateAPIView.as_view()),
  path('<int:pk>/update/', StatusUpdateAPIView.as_view()),
  path('<int:pk>/delete/', StatusDeleteAPIView.as_view()),
]


from django.db import models
from datetime import date
from vendas.models import Venda

class Caixa(models.Model):
    data = models.DateField(auto_now=True)
    valor_abertura = models.DecimalField(max_digits=7, decimal_places=3)

    @property
    def valor_especie(self):
        hj = date.today()
        vendas_do_dia = Venda.objects.filter(data=hj)
        total = 0
        vendas_dinheiro = vendas_do_dia.filter(pagamento='Dinheiro')
        for venda in vendas_dinheiro:
            total += venda.total
        sum = total + self.valor_abertura
        return sum