from copy import deepcopy
import itertools
import json
from urllib.parse import urlsplit
import uuid

import datetime
from aniso8601 import parse_date
import cassandra.util
import valideer as V

from coreapis import cassandra_client
from coreapis.crud_base import CrudControllerBase
from coreapis.scopes import is_gkscopename, has_gkscope_match
from coreapis.scopes.manager import ScopesManager
from coreapis.authproviders import AUTHPROVMGR, REGISTER_CLIENT
from coreapis.utils import (
    LogWrapper, timestamp_adapter, ValidationError, ForbiddenError,
    valid_url, valid_name, valid_description, userinfo_for_log,
    get_platform_admins, get_approved_creators, PRIV_PLATFORM_ADMIN, public_userinfo,
    public_orginfo)

USER_SETTABLE_STATUS_FLAGS = {'Public'}
INVALID_URISCHEMES = {'data', 'javascript', 'file', 'about'}
FEIDE_REALM_PREFIX = 'feide|realm|'
MAX_DAYS = 14

def is_valid_uri(uri):
    parsed = urlsplit(uri)
    scheme = parsed.scheme
    if not scheme or scheme in INVALID_URISCHEMES:
        return False
    if scheme in ['http', 'https']:
        return bool(parsed.netloc)
    return True

def filter_client_status(attrs_new, attrs_old, privileges):
    try:
        status_old = set(attrs_old['status'])
    except (KeyError, TypeError):
        status_old = set()
    try:
        status_requested = set(attrs_new['status'])
    except (KeyError, TypeError):
        status_requested = set()
    if PRIV_PLATFORM_ADMIN in privileges:
        status_allowed = status_requested
    else:
        status_allowed = status_requested.intersection(USER_SETTABLE_STATUS_FLAGS)
    attrs_new['status'] = list(status_old.union(status_allowed))

def client_sortkey(client):
    sortkey = (client.get('count_tokens') or 0) + (client.get('count_users') or 0)
    if client.get('organization'):
        sortkey += 10000000
    return sortkey

class ClientAdmController(CrudControllerBase):
    schema = {
        # Required
        '+name': V.AdaptBy(valid_name, traps=ValueError),
        '+redirect_uri': V.HomogeneousSequence(item_schema=V.Condition(is_valid_uri), min_length=1),
        '+scopes_requested':  V.HomogeneousSequence(item_schema='string', min_length=1),
        # Maintained by clientadm API
        'created': V.AdaptBy(timestamp_adapter),
        'id': V.Nullable(V.AdaptTo(uuid.UUID)),
        'owner': V.AdaptTo(uuid.UUID),
        'organization': V.Nullable('string', None),
        'updated': V.AdaptBy(timestamp_adapter),
        'orgauthorization': V.Nullable({}),
        # Other attributes
        'client_secret': V.Nullable('string', ''),
        'descr': V.Nullable(V.AdaptBy(valid_description, traps=ValueError), ''),
        'scopes': V.Nullable(['string'], ()),
        'authproviders': V.Nullable(['string'], ()),
        'status': V.Nullable(['string'], ()),
        'type': V.Nullable('string', ''),
        'systemdescr': V.Nullable(V.AdaptBy(valid_description, traps=ValueError), ''),
        'privacypolicyurl': V.Nullable(valid_url),
        'homepageurl': V.Nullable(valid_url),
        'loginurl': V.Nullable(valid_url),
        'supporturl': V.Nullable(valid_url),
        'authoptions': V.Nullable({}),
        'admins': V.Nullable(['string'], ()),
        'admin_contact': V.Nullable('string', None),
    }
    public_attrs = ['id', 'name', 'descr', 'redirect_uri', 'owner', 'organization', 'authproviders',
                    'systemdescr', 'privacypolicyurl', 'homepageurl', 'loginurl', 'supporturl',
                    'scopes', 'status']
    scope_attrs = ['scopes', 'scopes_requested']
    platformadmin_attrs = ['owner', 'scopes', 'orgauthorization']
    platformadmin_attrs_update = ['organization']
    protected_attrs = ['created', 'updated']
    protected_attrs_update = ['id']

    def __init__(self, settings):
        contact_points = settings.get('cassandra_contact_points')
        keyspace = settings.get('cassandra_keyspace')
        authz = settings.get('cassandra_authz')
        maxrows = settings.get('clientadm_maxrows')
        super(ClientAdmController, self).__init__(maxrows, 'client')
        self.session = cassandra_client.Client(contact_points, keyspace, authz=authz)
        platformadmins_file = settings.get('platformadmins_file')
        self.platformadmins = get_platform_admins(platformadmins_file)
        approved_creators_file = settings.get('approved_creators_file')
        self.approved_creators = set(get_approved_creators(approved_creators_file))
        self.scopemgr = ScopesManager(settings, self.session, self.get_public_info, False)
        self.log = LogWrapper('clientadm.ClientAdmController')
        self.groupengine_base_url = settings.get('groupengine_base_url')

    @staticmethod
    def adapt_client(client):
        adapted = deepcopy(client)
        for key, val in adapted.items():
            if key == 'orgauthorization':
                adapted[key] = {}
                if val:
                    adapted[key] = {k2: json.loads(v2) for k2, v2 in val.items()}
            elif key == 'authoptions':
                adapted[key] = {}
                if val:
                    adapted[key] = json.loads(val)
            elif isinstance(val, cassandra.util.SortedSet):
                adapted[key] = list(val)
        return adapted

    def _list(self, selectors, values, scope):
        if scope:
            selectors.append('scopes contains ?')
            values.append(scope)
        res = self.session.get_clients(selectors, values, self.maxrows)
        return [self.adapt_client(client) for client in res]

    def list_by_owner(self, owner, scope=None):
        selectors = ['owner = ?']
        values = [owner]
        return self._list(selectors, values, scope)

    def list_personal(self, owner, scope=None):
        return [c for c in self.list_by_owner(owner, scope) if c['organization'] is None]

    def list_by_admin(self, admin, scope=None):
        selectors = ['admins contains ?']
        values = [admin]
        clients = self._list(selectors, values, scope)
        return [c for c in clients]

    def list_delegated(self, userid, scope, token):
        delegated = []
        delegated_ids = set()
        groupids = set(self.get_my_groupids(token))
        for groupid in groupids:
            clients = self.list_by_admin(groupid, scope)
            for client in clients:
                clientid = client['id']
                if clientid not in delegated_ids:
                    delegated.append(client)
                    delegated_ids.add(clientid)
        return delegated

    def list_by_organization(self, organization, scope=None):
        selectors = ['organization = ?']
        values = [organization]
        return self._list(selectors, values, scope)

    def list_all(self, scope=None):
        clients = self._list([], [], scope)
        self.session.add_client_counters(clients)
        return sorted(clients,
                      key=client_sortkey, reverse=True)

    def get_public_client_list(self, clients):
        owner_ids = list({c['owner'] for c in clients})
        users = {uid: public_userinfo(user)
                 for uid, user in self.session.get_users(owner_ids).items()}
        org_ids = list({c['organization'] for c in clients if c.get('organization')})
        orgs = {oid: public_orginfo(org) for oid, org in self.session.get_orgs(org_ids).items()}
        return [self.get_public_info(c, users, orgs) for c in clients if c]

    def public_clients(self, orgauthorization, status):
        selectors = []
        values = []
        if orgauthorization:
            selectors = ['orgauthorization contains key ?']
            values = [orgauthorization]
        if status:
            selectors.append('status contains ?')
            values.append(status)
        clients = self._list(selectors, values, None)
        self.session.add_client_counters(clients)

        clients = sorted(clients,
                         key=client_sortkey, reverse=True)
        return self.get_public_client_list(clients)

    def is_owner(self, user, client):
        if client['owner'] == user['userid']:
            return True
        return False

    def is_delegated_admin(self, client, token):
        admins = set(client.get('admins') or [])
        if not admins:
            return False
        groupids = set(self.get_my_groupids(token))
        return bool(admins.intersection(groupids))

    def has_permission(self, client, user, token):
        if user is None:
            return False
        if self.is_platform_admin(user):
            return True
        org = client.get('organization', None)
        if ((org and self.is_org_admin(user, org)) or
                (not org and self.is_owner(user, client))):
            return True
        return self.is_delegated_admin(client, token)

    def has_add_permission(self, user, token):
        groupids = set(self.get_my_groupids(token))
        allowed_for_id_provider = AUTHPROVMGR.has_user_permission(user, REGISTER_CLIENT)
        approved_creator = bool(set(self.approved_creators).intersection(groupids))
        return allowed_for_id_provider or approved_creator

    def get(self, clientid):
        self.log.debug('Get client', clientid=clientid)
        return self.adapt_client(self.session.get_client_by_id(clientid))

    def insert_client(self, client):
        sessclient = deepcopy(client)
        orgauthz = sessclient.get('orgauthorization', None)
        if orgauthz:
            for key, val in orgauthz.items():
                orgauthz[key] = json.dumps(val)
        self.session.insert_client(sessclient)

    # Used both for add and update.
    # By default CQL does not distinguish between INSERT and UPDATE
    def _insert(self, client, privileges):
        self.scopemgr.handle_update(client, privileges)
        AUTHPROVMGR.check_client_update(self.session, client)
        self.insert_client(client)
        self.scopemgr.notify_moderators(client)
        return client

    def add(self, item, user, privileges):
        filter_client_status(item, {}, privileges)
        res = super(ClientAdmController, self).add(item, user, privileges)
        self.log.info('adding client',
                      audit=True, clientid=res['id'], user=userinfo_for_log(user))
        return res

    def update(self, itemid, attrs, user, privileges):
        client = self.get(itemid)
        filter_client_status(attrs, client, privileges)
        client = self.validate_update(itemid, attrs)
        res = self._insert(client, privileges)
        self.log.info('updating client',
                      audit=True, clientid=res['id'], attrs=attrs, user=userinfo_for_log(user))
        return res

    def delete(self, clientid, user):
        self.log.info('delete client',
                      audit=True, clientid=clientid, user=userinfo_for_log(user))
        self.session.delete_client(clientid)

    def get_logo(self, clientid):
        return self.session.get_client_logo(clientid)

    def _save_logo(self, clientid, data, updated):
        self.session.save_logo('clients', clientid, data, updated)

    # Return clients for each scope as dict { <scope>: set(clientids)}
    def get_scope_clients(self):
        res = {}
        for client in self._list([], [], None):
            clientscopes = client.get('scopes', None)
            if clientscopes:
                for scope in [scope for scope in clientscopes if is_gkscopename(scope)]:
                    clientids = set(res.get(scope, []))
                    clientids.add(str(client['id']))
                    res[scope] = clientids
        return res

    # Return targets for a scope with subscopes as dict { <scope>: set(realms)}
    def get_scope_targets(self, name, scopedef):
        res = {}
        if scopedef:
            try:
                res[name] = set(scopedef['policy']['orgadmin']['target'])
            except KeyError:
                pass  # scopedef does not have orgadmin targets
            subscopes = scopedef.get('subscopes', None)
            if subscopes:
                for subname, subscopedef in subscopes.items():
                    res.update(self.get_scope_targets('{}_{}'.format(name, subname), subscopedef))
        return res

    # Return dict { <scope>: set(realms)} including all scopes, subscopes and their realms
    def get_scope_targets_all(self):
        res = {}
        for apigk in self.session.get_apigks([], [], 999999):
            res.update(self.get_scope_targets('gk_' + apigk['id'], apigk['scopedef']))
        return res

    # Return all clients which have been assigned a scope targeting this realm
    # as dict { <client id>: [<scope>, ..]}
    def get_realmclient_scopes(self, realm):
        # Get dict { <scope>: set(clientids)} including all scopes, subscopes and their clients
        clientids = self.get_scope_clients()
        # Get dict { <scope>: set(realms)} including all scopes, subscopes and their realms
        scope_targets = self.get_scope_targets_all()
        # For scopes which target the realm, add list of clients to result dict
        res = {}
        for scopename, realms in scope_targets.items():
            if FEIDE_REALM_PREFIX + realm in realms and scopename in clientids.keys():
                for clientid in clientids[scopename]:
                    scopenames = set(res.get(clientid, []))
                    scopenames.add(scopename)
                    res[clientid] = list(scopenames)
        return res

    def get_realmclient(self, realm, clientid, scopes):
        client = self.get(uuid.UUID(clientid))
        orgauthz = client.get('orgauthorization', {})
        scopeauthz = {scope: scope in orgauthz.get(realm, '[]') for scope in list(scopes)}
        pubclient = self.get_public_info(client)
        pubclient['scopeauthorizations'] = scopeauthz
        return pubclient

    def get_realmclients(self, realm):
        return [self.get_realmclient(realm, key, val)
                for key, val in self.get_realmclient_scopes(realm).items()]

    def get_gkscope_client(self, client, gkscopes, users=None, orgs=None):
        gkclient = self.get_public_info(client, users, orgs)
        orgauthz = client['orgauthorization']
        if orgauthz:
            orgauthz = {key: json.loads(val) for key, val in orgauthz.items()}
        gkclient.update({'orgauthorization': orgauthz})
        gkclient.update({attr: [] for attr in self.scope_attrs})
        gkclient.update({'admin_contact': self.get_admin_contact(client)})
        for attr in self.scope_attrs:
            clientscopes = client[attr]
            if clientscopes:
                gkclient[attr] = [scope for scope in clientscopes
                                  if has_gkscope_match(scope, gkscopes)]
        return gkclient

    def get_gkscope_clients(self, gkscopes):
        users = {}
        orgs = {}
        clientdict = {}
        for gkscope in gkscopes:
            for client in (itertools.chain(self.session.get_clients_by_scope(gkscope),
                                           self.session.get_clients_by_scope_requested(gkscope))):
                if client['id'] not in clientdict:
                    clientdict[client['id']] = self.get_gkscope_client(client, gkscopes,
                                                                       users, orgs)
        return list(clientdict.values())

    def list_scopes(self):
        self.log.debug('List scopes')
        return self.scopemgr.list_scopes()

    def validate_gkscope(self, user, scope, token):
        if not is_gkscopename(scope):
            raise ForbiddenError('{} is not an API Gatekeeper'.format(scope))
        gk = self.scopemgr.scope_to_gk(scope)
        if not gk or not self.has_permission(gk, user, token):
            raise ForbiddenError('User does not have access to manage API Gatekeeper')

    def add_gkscopes(self, client, user, scopes_add, token):
        for scope in [scope for scope in scopes_add if scope not in client['scopes']]:
            if scope not in client['scopes_requested']:
                raise ForbiddenError('Client owner has not requested scope {}'.format(scope))
            self.validate_gkscope(user, scope, token)
            client['scopes'].append(scope)
        return client

    def remove_gkscopes(self, client, user, scopes_remove, token):
        for scope in scopes_remove:
            self.validate_gkscope(user, scope, token)
            if scope in client['scopes']:
                client['scopes'].remove(scope)
            if scope in client['scopes_requested']:
                client['scopes_requested'].remove(scope)
        return client

    def update_gkscopes(self, clientid, user, scopes_add, scopes_remove, token):
        client = self.get(clientid)
        client = self.add_gkscopes(client, user, scopes_add, token)
        client = self.remove_gkscopes(client, user, scopes_remove, token)
        self.log.info('updating gkscopes for client',
                      audit=True, clientid=clientid,
                      scopes_add=scopes_add, scopes_remove=scopes_remove,
                      user=userinfo_for_log(user))
        self.insert_client(client)

    def add_scopes(self, client, scopes_add, privileges):
        for scope in [scope for scope in scopes_add if scope not in client['scopes']]:
            if scope not in client['scopes_requested']:
                raise ForbiddenError('Client owner has not requested scope {}'.format(scope))
            self.scopemgr.handle_scope_request(client, scope, privileges)
        return client

    def remove_scopes(self, client, scopes_remove):
        for scope in scopes_remove:
            if scope in client['scopes']:
                client['scopes'].remove(scope)
            if scope in client['scopes_requested']:
                client['scopes_requested'].remove(scope)
        return client

    def update_scopes(self, client, user, scopes_add, scopes_remove):
        client = self.add_scopes(client, scopes_add, self.get_privileges(user))
        client = self.remove_scopes(client, scopes_remove)
        self.log.info('updating scopes for client',
                      audit=True, clientid=client['id'],
                      scopes_add=scopes_add, scopes_remove=scopes_remove,
                      user=userinfo_for_log(user))
        self.insert_client(client)
        self.scopemgr.notify_moderators(client)
        return client

    def has_realm_permission(self, realm, user):
        org = self.session.get_org_by_realm(realm)
        return self.is_admin(user, org['id'])

    @staticmethod
    def get_orgauthorization(client, realm):
        return client['orgauthorization'].get(realm, [])

    def update_orgauthorization(self, client, realm, scopes, user):
        clientid = client['id']
        self.log.info('updating orgauthorization for client',
                      audit=True, clientid=clientid, realm=realm, scopes=scopes,
                      user=userinfo_for_log(user))
        self.session.insert_orgauthorization(clientid, realm, json.dumps(scopes))

    def delete_orgauthorization(self, client, realm, user):
        clientid = client['id']
        self.log.info('deleting orgauthorization for client',
                      audit=True, clientid=clientid, realm=realm, user=userinfo_for_log(user))
        self.session.delete_orgauthorization(clientid, realm)

    def get_policy(self, user, token):
        approved = self.has_add_permission(user, token)
        return dict(register=approved)

    def get_logins_stats(self, clientid, end_date, num_days, authsource):
        if end_date:
            try:
                end_date = parse_date(end_date)
            except Exception:
                raise ValidationError('end_date not a valid date: {}'.format(end_date))
        else:
            end_date = datetime.date.today()
        try:
            num_days = int(num_days)
        except ValueError:
            raise ValidationError('num_days not an integer: {}'.format(num_days))
        if num_days < 1 or num_days > MAX_DAYS:
            msg = 'num_days should be an integer: 1 <= num_days <= {}'.format(MAX_DAYS)
            raise ValidationError(msg)
        start_date = end_date - datetime.timedelta(num_days - 1)
        dates = [start_date + datetime.timedelta(i) for i in range(num_days)]
        return list(self.session.get_logins_stats(clientid, dates, authsource, None))


# deploy_front/app.py
#
import os.path
from flask import Flask
import os
import json
import run_backend

import time

app = Flask(__name__)

def get_predictions():

    videos = []

    novos_videos_json = "novos_videos.json"
    if not os.path.exists(novos_videos_json):
        run_backend.update_db()

    last_update = os.path.getmtime(novos_videos_json) * 1e9

    #if time.time_ns() - last_update > (720*3600*1e9): # aprox. 1 mes
    #    run_backend.update_db()

    with open("novos_videos.json", 'r') as data_file:
        for line in data_file:
            line_json = json.loads(line)
            videos.append(line_json)

    predictions = []
    for video in videos:
        #print(video)
        #print(video['video_id'])
        predictions.append((video['video_id'], video['title'], float(video['score'])))

    predictions = sorted(predictions, key=lambda x: x[2], reverse=True)[:30]

    predictions_formatted = []
    for e in predictions:
        #print(e)
        predictions_formatted.append("
{title}	{score}
".format(title=e[1], link=e[0], score=e[2]))

    return '\n'.join(predictions_formatted), last_update

@app.route('/')
def main_page():
    preds, last_update = get_predictions()
    return """
Recomendador de Vídeos do Youtube

    Segundos desde a última atualização: {}

             {}

    """.format((time.time_ns() - last_update) / 1e9, preds)

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


import os
import asyncio
import moteus
import moteus_pi3hat
from utils import (prepare_empty_arrays,
                   rad2rev,
                   rev2rad,
                   read_data,
                   prepare_data,
                   wrap_z2pi,
                   zero_offset,
                   save_data,
                   generate_path,
                   plot_data,
                   send_rad_command,
                   get_theta1)
import time
import numpy as np
from datetime import datetime
import math

def simple_print(th1,th2,imu):
    print(f"imu_angle: {np.rad2deg(imu)}")
    print(f"[theta1,theta2]: [{np.rad2deg(th1),np.rad2deg(th2)}]")
    print(f'imu-theta2: {np.rad2deg(imu) - np.rad2deg(th2)}')
def wrapped_z2pi_print(th1,th2,imu):
    print(f"imu_angle: {np.rad2deg((imu))}")
    print(f"[theta1,theta2]: [{np.rad2deg((th1)),np.rad2deg((th2))}]")
    print(f"wrapped_imu_angle: {np.rad2deg(wrap_z2pi(imu))}")
    print(f"wrapped[theta1,theta2]: [{np.rad2deg(wrap_z2pi(th1)),np.rad2deg(wrap_z2pi(th2))}]")
    print(f'imu-theta2: {np.rad2deg(wrap_z2pi(imu)) - np.rad2deg(wrap_z2pi(th2))}')
def plot_custom_data(meas_time,
                     shoulder_meas_pos,
                     shoulder_meas_vel,
                     elbow_meas_pos,
                     elbow_meas_vel):
    date = datetime.now().strftime("%Y%m%d-%I%M%S-%p")
    folder_name = 'results/imu_validation'
    file_name = date
    directory = generate_path(folder_name, file_name, 1)
    print(f'save to {directory}')
    os.makedirs(directory)
    plot_data(x_data=[meas_time, meas_time],
              y_data=[shoulder_meas_pos, elbow_meas_pos],
              labels=["Time (s)", "Position (rad)"],
              title="Position (rad) vs Time (s)",
              legends=['shoulder', 'elbow'],
              save_name=directory + '/pos.pdf')

    plot_data(x_data=[meas_time, meas_time],
              y_data=[shoulder_meas_vel, elbow_meas_vel],
              labels=["Time (s)", "Velocity (rad/s)"],
              title="Velocity (rad/s) vs Time (s)",
              legends=['shoulder', 'elbow'],
              save_name=directory + '/vel.pdf')
    return date, directory

async def main():
    print("Motor Enabled.")
    if len(bus_number) == 1:
        transport = moteus_pi3hat.Pi3HatRouter(
            servo_bus_map={
                bus_number[0]: motor_id,
            },
        )
        print(f"check Bus Numbers and motor id: {bus_number[0]}:[{motor_id[0]},{motor_id[1]}]")
    else:
        transport = moteus_pi3hat.Pi3HatRouter(
            servo_bus_map={
#                bus_number[0]: [motor_id[0]],
#                bus_number[1]: [motor_id[1]],
                 bus_number:motor_id,
            },
        )
        print(f"check Bus Numbers and motor id: {bus_number[0]}:{[motor_id[0]]},{bus_number[1]}:{[motor_id[1]]}")
    # create a moteus_pi3hat.Pi3HatRouter for IMU reading
    pr = moteus_pi3hat.Pi3HatRouter(mounting_deg={'roll': 90,
                                                'pitch': 0,
                                                'yaw': 0})
    # wait until convergence of IMU reading
    time.sleep(0.5)

    # create a moteus.Controller object for interaction with controller
    c_shoulder = moteus.Controller(id=motor_id[0], transport=transport)
    c_elbow = moteus.Controller(id=motor_id[1], transport=transport)
    await c_shoulder.set_stop()
    await c_elbow.set_stop()

    index = 0
    t = 0
    meas_dt = 0
    #    start_time = time.time()
    tau_limit = 3
    pos_sh = 0.
    pos_el = 0.
    # (pos_sh,
    #     _,
    #     _) = await send_rad_command(controller_obj=c_shoulder,
    #                                             pos=pos_sh,
    #                                             vel=vel,
    #                                             tau_limit=tau_limit)
    # (pos_el,
    #     _,
    #     _) = await send_rad_command(controller_obj=c_elbow,
    #                                             pos=pos_el,
    #                                             vel=vel,
    #                                             tau_limit=tau_limit)
    while index < n:
        start_loop = time.time()
        t += meas_dt
        # send commands to motor in radians
        (_,
            _,
            _) = await send_rad_command(controller_obj=c_shoulder,
                                                    pos=pos_sh,
                                                    vel=vel,
                                                    tau_limit=tau_limit)
        (_,
            _,
            _) = await send_rad_command(controller_obj=c_elbow,
                                                    pos=pos_el,
                                                    vel=vel,
                                                    tau_limit=tau_limit)
        # meas_time[index] = t
        # read the data from the motor
        # sh_state = await c_shoulder.set_position(position=rad2rev(pos_sh),
        #                              velocity=vel,
        #                              kp_scale=1,
        #                              kd_scale=1,
        #                              stop_position=None,
        #                              feedforward_torque=None,
        #                              maximum_torque=tau_limit,
        #                              watchdog_timeout=None,
                                    #  query=True)
        # el_state = await c_elbow.set_position(position=rad2rev(pos_el),
        #                              velocity=vel,
        #                              kp_scale=1,
        #                              kd_scale=1,
        #                              stop_position=None,
        #                              feedforward_torque=None,
        #                              maximum_torque=tau_limit,
        #                              watchdog_timeout=None,
        #                              query=True)
        # imu = await pr.cycle([], request_attitude=True)
        # quaternion_obj = imu[0]
        # w, x, y, z = quaternion_obj.attitude.w, \
        #              quaternion_obj.attitude.x, \
        #              quaternion_obj.attitude.y, \
        #              quaternion_obj.attitude.z
        # '''
        # denom = math.sqrt(1 - w * w)
        # print(f"denom: {denom}")
        # '''
        # denom = 1
        # ax = x / denom
        # ay = y / denom
        # az = z / denom
        # angle_axis = 2 * math.acos(w)
        # theta2 = angle_axis if ay < 0 else -angle_axis
        # # store data
        # shoulder_meas_pos[index] = rev2rad(sh_state.values[moteus.Register.POSITION])
        # shoulder_meas_vel[index] = rev2rad(sh_state.values[moteus.Register.VELOCITY])
        # elbow_meas_pos[index] = rev2rad(el_state.values[moteus.Register.POSITION])
        # elbow_meas_vel[index] = rev2rad(el_state.values[moteus.Register.VELOCITY])
        # meas_time[index] = t
        # print(f"imu_angle: {theta2}")
        # print(f"[theta1,theta2]: [{shoulder_meas_pos[index],elbow_meas_pos[index]}]")
        index += 1
        pos_el += final_pos_elbow / n
        pos_sh += final_pos_shoudler / n
        '''
        while time.time() - start_loop < dt:
            pass
        '''
        # keep the final position
        # print(f'freez for {freez_time} seconds at pos [{np.rad2deg(final_pos_shoudler)},{np.rad2deg(final_pos_elbow)}].')
        # start_time = time.time()
        # while time.time() - start_time < freez_time:
    print(f'freez for {freez_loop} loops at pos [{np.rad2deg(final_pos_shoudler)},{np.rad2deg(final_pos_elbow)}].')
    index = 0
    t = 0
    meas_dt = 0
    try:
        while index < freez_loop:
            start_loop = time.time()
            t += meas_dt
            imu = await pr.cycle([], request_attitude=True)
            quaternion_obj = imu[0]
            w, x, y, z = quaternion_obj.attitude.w, \
                        quaternion_obj.attitude.x, \
                        quaternion_obj.attitude.y, \
                        quaternion_obj.attitude.z
            '''
            denom = math.sqrt(1 - w * w)
            print(f"denom: {denom}")
            '''
            denom = math.sqrt(1-w*w)
            ax = x / denom
            ay = y / denom
            az = z / denom
            angle_axis = 2 * math.acos(w)
            angle_axis = angle_axis if ay < 0 else -angle_axis
            # send commands to motor in radians
            (shoulder_meas_pos[index],
            shoulder_meas_vel[index],
            _) = await send_rad_command(controller_obj=c_shoulder,
                                                    pos=pos_sh,
                                                    vel=vel,
                                                    tau_limit=tau_limit)
            (elbow_meas_pos[index],
            elbow_meas_vel[index],
            _) = await send_rad_command(controller_obj=c_elbow,
                                                    pos=pos_el,
                                                    vel=vel,
                                                    tau_limit=tau_limit)
            theta1 = get_theta1(angle_axis, elbow_meas_pos[index])
            # print(f"imu_angle: {np.rad2deg(angle_axis)}")
            print(f"[theta1,theta2]: [{np.rad2deg(shoulder_meas_pos[index]),np.rad2deg(elbow_meas_pos[index])}]")
            print(f'imu-theta1: {np.rad2deg(theta1)}')
            meas_time[index] = t
            index += 1

            meas_dt = time.time() - start_loop

    except BaseException as e:
        print("**Exeptoin Block**")
        print(f"Error= {e}")
    finally:
      # Save Trajectory to a csv file to be sent to the motor.
        data = {"time": meas_time[:index],
                "shoulder_pos": shoulder_meas_pos[:index],
                "shoulder_vel": shoulder_meas_vel[:index],
                "elbow_pos": elbow_meas_pos[:index],
                "elbow_vel": elbow_meas_vel[:index]}
        # date, directory = plot_custom_data(meas_time[:index],
        #                                    shoulder_meas_pos[:index],
        #                                    shoulder_meas_vel[:index],
        #                                    elbow_meas_pos[:index],
        #                                    elbow_meas_vel[:index])
        # save_data(data, directory+'/measured.csv')
        # Disabling the motor
        await c_shoulder.set_stop()
        await c_elbow.set_stop()

if __name__ == '__main__':
#    motor id and bus number mapping
    bus_number = [2]
    motor_id = [8, 9]
    # zero offset the motors
    zero_offset(bus_number[0], motor_id[0])
    zero_offset(bus_number[0], motor_id[1])
    n = 500
    # freez_time = 20.
    freez_loop = 5000
    final_pos_shoudler = np.deg2rad(0.)
    final_pos_elbow = np.deg2rad(0)
    vel =  0.2
    # (meas_time,
    #  shoulder_meas_pos,
    #  shoulder_meas_vel,
    #  elbow_meas_pos,
    #  elbow_meas_vel,
    #  _,
    #  _,
    #  _,
    #  _) = prepare_empty_arrays(n)
    (meas_time,
     shoulder_meas_pos,
     shoulder_meas_vel,
     elbow_meas_pos,
     elbow_meas_vel,
     _,
     _,
     _,
     _) = prepare_empty_arrays(freez_loop)
    if input('continue with real time experiment?') == 'y':
        asyncio.run(main())
    print('end of script')


from models.KF_wrapper import *
from training.KF_train_utils import train_loop_KF
from datasets.utils.get_data import get_mimic_data
from sacred import Experiment
import seml

ex = Experiment()
seml.setup_logger(ex)

@ex.post_run_hook
def collect_stats(_run):
    seml.collect_exp_stats(_run)

@ex.config
def config():
    overwrite = None
    db_collection = None
    if db_collection is not None:
        ex.observers.append(seml.create_mongodb_observer(db_collection, overwrite=overwrite))

@ex.automain
def run(epochs, device, dim, ldim, batch_size, lr, seed, cov_dim, hiddendim, model_name, norm, reg_lambda, ds_seed):

    train, val, _, name = get_mimic_data(norm=norm, small_ds=False, random_state=ds_seed)

    if model_name == "KF":
        model = KF(seed=seed, dim=dim, latent_dim=ldim)
    elif model_name == "NKF":
        model = NKF(seed=seed, dim=dim, latent_dim=ldim)
    elif model_name == "RKF-O":
        model = RKF_O(seed=seed, dim=dim, latent_dim=ldim)
    elif model_name == "RKF-F":
        model = RKF_F(seed=seed, dim=dim, latent_dim=ldim, hidden_dim=hiddendim)
    elif model_name == "RCKF":
        model = RCKF(seed=seed, dim=dim, latent_dim=ldim, hidden_dim=hiddendim, cov_dim=cov_dim)

    experiment_name = train_loop_KF(model, model_name, name, train, val, epochs=epochs, lr=lr, batch_size=batch_size,
                                    device=device, reg_lambda=reg_lambda)

    return experiment_name


from __future__ import division

import numpy as np

from scipy import optimize
from deap.benchmarks import schwefel
from deap.benchmarks import griewank
from deap.benchmarks import schaffer

from abc import ABCMeta
from abc import abstractmethod
from six import add_metaclass

from sklearn.model_selection import train_test_split
from sklearn.datasets import load_digits
from keras.models import Sequential
from keras import layers

@add_metaclass(ABCMeta)
class ObjectiveFunction(object):

    def __init__(self, name, dim, minf, maxf):
        self.name = name
        self.dim = dim
        self.minf = minf
        self.maxf = maxf

    def sample(self):
        return np.random.uniform(low=self.minf, high=self.maxf, size=self.dim)

    def custom_sample(self):
        return np.repeat(self.minf, repeats=self.dim) \
               + np.random.uniform(low=0, high=1, size=self.dim) *\
               np.repeat(self.maxf - self.minf, repeats=self.dim)

    @abstractmethod
    def evaluate(self, x):
        pass

@add_metaclass(ABCMeta)
class PartitionalClusteringObjectiveFunction(ObjectiveFunction):

    def __init__(self, dim, n_clusters, data):
        super(PartitionalClusteringObjectiveFunction, self)\
            .__init__('PartitionalClusteringObjectiveFunction', dim, 0.0, 1.0)
        self.n_clusters = n_clusters
        self.centroids = {}
        self.data = data

    def decode(self, x):
        centroids = x.reshape(self.n_clusters, self.data.shape[1])
        self.centroids = dict(enumerate(centroids))

    @abstractmethod
    def evaluate(self, x):
        pass

class QuantizationError(PartitionalClusteringObjectiveFunction):

    def __init__(self, dim, n_clusters, data):
        super(QuantizationError, self).__init__(dim, n_clusters, data)
        self.name = 'QuantizationError'

    def evaluate(self, x):
        self.decode(x)

        clusters = {key: [] for key in self.centroids.keys()}
        for instance in self.data:
            distances = [np.linalg.norm(self.centroids[idx] - instance)
                         for idx in self.centroids]
            clusters[np.argmin(distances)].append(instance)

        outer_sum = 0.0
        for centroid in self.centroids:
            inner_sum = 0.0
            if len(clusters[centroid]) > 0:
                for instance in clusters[centroid]:
                    inner_sum += np.linalg.norm(instance - self.centroids[centroid])
                inner_sum /= len(clusters[centroid])
            outer_sum += inner_sum
        return outer_sum/len(self.centroids)

class SumOfSquaredErrors(PartitionalClusteringObjectiveFunction):

    def __init__(self, dim, n_clusters, data):
        super(SumOfSquaredErrors, self).__init__(dim, n_clusters, data)
        self.name = 'SumOfSquaredErrors'

    def evaluate(self, x):
        self.decode(x)

        clusters = {key: [] for key in self.centroids.keys()}
        for instance in self.data:
            distances = [np.linalg.norm(self.centroids[idx] - instance)
                         for idx in self.centroids]
            clusters[np.argmin(distances)].append(instance)

        sum_of_squared_errors = 0.0
        for idx in self.centroids:
            distances = [np.linalg.norm(self.centroids[idx] - instance)
                         for instance in clusters[idx]]
            sum_of_squared_errors += sum(np.power(distances, 2))
        return sum_of_squared_errors

class DeepNeuralNetworkObjectiveFunction(ObjectiveFunction):

    def __init__(self, dim):
        super(DeepNeuralNetworkObjectiveFunction, self).__init__('DeepNeuralNetworkObjectiveFunction', dim, 0, 2)
        digits = load_digits()
        self.data = digits['data']
        self.target = digits['target']

    def evaluate(self, x):
        model = Sequential()
        model.add(layers.Dense(12, input_dim=self.data.shape[1], activation='relu'))
        print('evaluating ...')
        for idx, layer in enumerate(np.round(x)):
            if layer == 0:
                model.add(layers.LSTM(150, return_sequences=True))
                print('adding layer LSTM ...')
            elif layer == 1:
                model.add(layers.MaxPooling2D())
                print('adding layer MaxPooling ...')
            elif layer == 2:
                model.add(layers.Conv2D(32, kernel_size=(3, 3), activation='relu'))
                print('adding layer Conv2D ...')
        model.add(layers.Dense(1, activation='softmax'))

        model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
        train_x, test_x, train_y, test_y = train_test_split(self.data, self.target, stratify=self.target)
        results = model.fit(train_x, train_y, epochs=2, batch_size=500, validation_data=(test_x, test_y))
        return -1 * results.history["val_acc"]

class Sphere(ObjectiveFunction):

    def __init__(self, dim):
        super(Sphere, self).__init__('Sphere', dim, -100.0, 100.0)

    def evaluate(self, x):
        return sum(np.power(x, 2))

class Rosenbrock(ObjectiveFunction):

    def __init__(self, dim):
        super(Rosenbrock, self).__init__('Rosenbrock', dim, -30.0, 30.0)

    def evaluate(self, x):
        return optimize.rosen(x)

class Rastrigin(ObjectiveFunction):

    def __init__(self, dim):
        super(Rastrigin, self).__init__('Rastrigin', dim, -5.12, 5.12)

    def evaluate(self, x):
        return 10 * len(x)\
               + np.sum(np.power(x, 2) - 10 * np.cos(2 * np.pi * np.array(x)))

class Schwefel(ObjectiveFunction):

    def __init__(self, dim):
        super(Schwefel, self).__init__('Schwefel', dim, -500.0, 500.0)

    def evaluate(self, x):
        return schwefel(x)[0]

class Griewank(ObjectiveFunction):

    def __init__(self, dim):
        super(Griewank, self).__init__('Griewank', dim, -600.0, 600.0)

    def evaluate(self, x):
        return griewank(x)[0]

class Schaffer(ObjectiveFunction):

    def __init__(self, dim):
        super(Schaffer, self).__init__('Schaffer', dim, -100.0, 100.0)

    def evaluate(self, x):
        return schaffer(x)[0]


import torch
import torch.nn as nn
import torch.nn.functional as F
from mmcv.cnn import normal_init

from ..builder import HEADS
from .sim_head import SimHead
from sim.datasets.utils import denormalize
from sim.models.utils import FFN

@HEADS.register_module()
class ParticleHead(SimHead):
    """Particle based simulation head.

    Args:
        in_channels (int): Number of channels in the input feature map.
        loss (dict): Config of classification loss.
    """  # noqa: W605

    def __init__(self,
                 out_channels=3,
                 in_channels=0,
                 loss=dict(type='MSELoss', loss_weight=1.0),
                 dt=1/60.0,
                 seperate=True,
                 rotation_dim=4,
                 weighted=False,
                 eps=1e-7):
        super(ParticleHead, self).__init__(loss=loss)
        # nf_effect
        self.in_channels = in_channels
        # position_dim
        self.out_channels = out_channels
        self.dt = dt
        self.rotation_dim = rotation_dim
        self.weighted = weighted
        self.seperate = seperate
        self.eps = eps

        if self.out_channels <= 0:
            raise ValueError(
                f'out_channels={out_channels} must be a positive integer')

        self._init_layers()

    def _init_layers(self):
        out_channels = self.out_channels + self.rotation_dim

        if self.seperate:
            self.rigid_predictor = FFN([self.in_channels, self.in_channels, self.in_channels, out_channels])
            out_channels = self.out_channels
        self.fluid_predictor = FFN([self.in_channels, self.in_channels, self.in_channels, out_channels])

    def evaluate(self, vel_dist, inputs, gt_label):
        # bs, n_particles, state_dim
        gt_label = gt_label['label'].squeeze(1).transpose(-1, -2)
        # bs, n_particles, state_dim
        prev_state = inputs['state'].squeeze(1).transpose(-1, -2)
        # bs, n_particles
        output_mask = inputs['output_mask'].squeeze(1)
        # bs, 2, 3, 3
        stat = inputs['stat']
        # bs, 3
        mean_p = stat[:, 0, :, 0]
        std_p = stat[:, 0, :, 1]
        mean_v = stat[:, 1, :, 0]
        std_v = stat[:, 1, :, 1]
        # bs, 3, 2
        pos_stat = torch.stack([mean_p, std_p], dim=2)
        vel_stat = torch.stack([mean_v, std_v], dim=2)

        state_dim = prev_state.shape[-1]
        valid_pos_dim = self.out_channels
        if state_dim == 2*self.out_channels:
            prev_pos = prev_state[:, :, :self.out_channels]
            prev_vel = prev_state[:, :, self.out_channels:]
            vel_label = gt_label[:, :, self.out_channels:]
            pos_label = gt_label[:, :, :self.out_channels]
        else:
            assert self.out_channels % 2 == 0
            valid_pos_dim = int(self.out_channels/2)
            prev_pos = prev_state[:, :, valid_pos_dim:self.out_channels]
            pos_label = gt_label[:, :, valid_pos_dim:self.out_channels]

            prev_vel = prev_state[:, :, self.out_channels+valid_pos_dim:2*self.out_channels]
            # compute loss first
            vel_label = gt_label[:, :, self.out_channels:]
            # prev: bs, 6, 2
            # after: bs, 3, 2
            vel_stat = vel_stat[:, :valid_pos_dim, :]
            pos_stat = pos_stat[:, :valid_pos_dim, :]

        vel_loss = self.loss(vel_dist, vel_label, inputs=inputs, reduction_override='none')

        if state_dim != 2*self.out_channels:
            # Ricegrip
            vel_label = vel_label[:, :, valid_pos_dim:]
            vel_dist = vel_dist[:, :, valid_pos_dim:]

        prev_pos, vel_dist = denormalize(
            [prev_pos, vel_dist], [pos_stat, vel_stat], mask=output_mask)
        pos_results = vel_dist * self.dt + prev_pos
        pos_label = denormalize([pos_label], [pos_stat], mask=output_mask)[0]
        pos_loss = self.loss(pos_results, pos_label, inputs=inputs, reduction_override='none')

        # Parse loss
        # bs, n_particles, 3/6
        vel_loss = vel_loss['accuracy']['Batch std']
        # bs, n_particles, 3
        pos_loss = pos_loss['loss']
        return vel_loss, pos_loss, pos_results

    def simple_test(self, particle_emb, gt_label=None, **kwargs):
        """Test without augmentation."""
        inputs = kwargs.get('inputs', None)
        assert isinstance(inputs, dict)
        vel_dist = self.predict(particle_emb, inputs)
        # bs, n_particles, 1
        output_mask = inputs['output_mask'].transpose(-1, -2)

        rst = torch.cat([vel_dist, output_mask], dim=2)

        if gt_label is not None:
            # bs, n_particles, 3
            vel_loss, pos_loss, pred_pos = self.evaluate(vel_dist, inputs, gt_label)
            # bs, n_particles, 1
            vel_loss = vel_loss.mean(dim=2, keepdim=True)
            pos_loss = pos_loss.mean(dim=2, keepdim=True)
            rst = torch.cat([rst, vel_loss, pos_loss, pred_pos], dim=2)
        if torch.onnx.is_in_onnx_export():
            return rst
        rst = list(rst.detach().cpu().numpy())
        return rst

    def forward_train(self, particle_emb, gt_label, **kwargs):
        inputs = kwargs.get('inputs', None)
        assert isinstance(inputs, dict)
        vel_dist = self.predict(particle_emb, inputs)

        label = gt_label['label'].squeeze(1).transpose(-1, -2)
        label_mask = inputs['output_mask'].squeeze(1)
        rigid_mask = inputs['rigid_mask'].squeeze(1)
        fluid_mask = inputs['fluid_mask'].squeeze(1)

        vel_dist = vel_dist * label_mask.unsqueeze(2)
        # bs, nparticles, [pos, vel]; vel in 3
        vel_label = label[:, :, self.out_channels:] * label_mask.unsqueeze(2)
        weight = None
        if self.weighted:
            weight_factor = fluid_mask.sum(dim=-1, keepdim=True) / (rigid_mask.sum(dim=-1, keepdim=True) + self.eps)
            weight = rigid_mask * weight_factor + fluid_mask
            weight = weight.unsqueeze(-1)
        losses = self.loss(vel_dist, vel_label, weight=weight, inputs=inputs)
        return losses

    def rotation_matrix_from_quaternion(self, params):
        # params dim - 4: bs, (w, x, y, z)

        # multiply the rotation matrix from the right-hand side
        # the matrix should be the transpose of the conventional one

        # Reference
        # http://www.euclideanspace.com/maths/geometry/rotations/conversions/quaternionToMatrix/index.htm

        params = params / (torch.norm(params, dim=1, keepdim=True) + self.eps)
        # bs, 1
        w, x, y, z = params[:, 0], params[:, 1], params[:, 2], params[:, 3]
        one = torch.ones_like(w)

        rot = torch.stack((
            torch.stack((1-y*y*2-z*z*2, x*y*2+z*w*2, x*z*2-y*w*2), dim=1),
            torch.stack((x*y*2-z*w*2, 1-x*x*2-z*z*2, y*z*2+x*w*2), dim=1),
            torch.stack((x*z*2+y*w*2, y*z*2-x*w*2, 1-x*x*2-y*y*2), dim=1)), dim=1)

        return rot

    def predict(self, particle_emb, inputs):
        # Init inputs
        state = inputs['state'].squeeze(1).transpose(-1, -2)
        rigid_mask = inputs['rigid_mask'].squeeze(1)
        fluid_mask = inputs['fluid_mask'].squeeze(1)
        # bs, 2, 3, 3
        stat = inputs['stat']
        # bs, 3
        mean_p = stat[:, 0, :, 0]
        std_p = stat[:, 0, :, 1]
        mean_v = stat[:, 1, :, 0]
        std_v = stat[:, 1, :, 1]

        # Predict
        rigid_mask = rigid_mask.unsqueeze(2)
        fluid_mask = fluid_mask.unsqueeze(2)
        rigid_particles = particle_emb * rigid_mask
        # fluid_particles = particle_emb * fluid_mask
        # Pred fluid
        pred_fluid = self.fluid_predictor(particle_emb)

        ## Rigid
        if torch.sum(rigid_mask) > 0:
            if self.seperate:
                pred_rigid = self.rigid_predictor(rigid_particles)
            else:
                pred_rigid = pred_fluid

            if self.rotation_dim > 0:
                # bs, 7
                pred_rigid = torch.sum(pred_rigid * rigid_mask, dim=1)/(torch.sum(rigid_mask, dim=1)+self.eps)
                # R: bs, 3, 3
                R = self.rotation_matrix_from_quaternion(pred_rigid[:, :4])
                b = (pred_rigid[:, 4:] * std_v + mean_v) * self.dt
                # bs, n_p, 3
                p_0 = (state[:, :, :3] * std_p.unsqueeze(1) + mean_p.unsqueeze(1)) * rigid_mask
                # bs, 3
                c = torch.sum(p_0, dim=1) / (torch.sum(rigid_mask, dim=1) + self.eps)
                # bs, n_p, 3   *   bs, 3, 3 = bs, n_p, 3
                p_1 = torch.bmm(p_0 - c.unsqueeze(1), R) + b.unsqueeze(1) + c.unsqueeze(1)
                v = (p_1 - p_0) / self.dt
                pred_rigid = (v-mean_v.unsqueeze(1)) / std_v.unsqueeze(1)

                pred_fluid = pred_fluid[:, :, -self.out_channels:]

            pred = pred_rigid * rigid_mask + pred_fluid * fluid_mask
        else:
            if self.rotation_dim > 0:
                pred_fluid = pred_fluid[:, :, -self.out_channels:]
            pred = pred_fluid * fluid_mask

        return pred


import numpy as np

X = np.linspace(0, 1)
Y = 3 *X + 5
X1 = [[x - 0.1] for x in X]
import matplotlib.pyplot as plt
import pandas as pd
from sklearn import datasets, linear_model

lr = linear_model.LinearRegression()
lr.fit(X1, Y)
print(lr.coef_)
print(lr.intercept_)
def show_lr_plot():
    # show scatter
    plt.scatter(X1, Y)

    X2 = [[1.1], [1.3], [1.5], [1.11]]
    plt.plot(X2, lr.predict(X2), color ='red')

    plt.show()

show_lr_plot()


"""
Basic skeleton of a mitmproxy addon.

Run as follows: mitmproxy -s anatomy.py
"""
import os
import pathlib
import signal
import zmq
from typing import Any, cast, Optional
import simplejson as json
import time
import threading
from mitmproxy.http import Headers, Response as MitmResponse, HTTPFlow as MitmHTTPFlow
from common_types import ProxyRequest, ProxyResponse, ProxyWebsocketMessage
from home_page_html import HOME_PAGE_HTML
from sys import argv
from argparse import ArgumentParser

TIMEOUT_AFTER_SECONDS_NO_POLL = 3

# Example of argv:
#
# [
#   '/Users/evan/Code/pntest/venv/bin/mitmdump',
#   '-s',
#   '/Users/evan/Code/pntest/src/mitmproxy/addon.py',
#   '-p',
#   '8080',
#   '--set',
#   'confdir=./include',
#   '--set',
#   'client_certs=./include/mitmproxy-client.pem',
# ]

arg_parser = ArgumentParser()
arg_parser.add_argument('-p', '--port')
arg_parser.add_argument('--client-id')
arg_parser.add_argument('--zmq-server')
args, _ = arg_parser.parse_known_args(argv)

class ProxyHttpFlow(MitmHTTPFlow):
    intercept_response: bool

class ProxyEventsAddon:
    settings: Optional[dict[str,Any]]
    client_id: int
    recording_enabled: bool
    intercept_enabled: bool
    intercepted_flows: list[ProxyHttpFlow]
    socket: zmq.Socket

    def __init__(self, client_id: int):
        current_path = pathlib.Path(__file__).parent.resolve()

        self.client_id = client_id
        self.intercepted_flows = []
        self.settings = None
        self.intercept_enabled = False
        self.recording_enabled = True

    def zmq_connect(self):
        print("[ZMQClient] starting...")

        # Connect to the ZMQ server
        context = zmq.Context()
        self.socket = context.socket(zmq.DEALER)
        self.socket.setsockopt_string(zmq.IDENTITY, str(self.client_id))
        self.socket.connect("tcp://%s" % args.zmq_server)

        # Let em know we've started
        self.socket.send_string(json.dumps({'type': 'started'}))

        # Listen to messages from the server
        listen_thread = threading.Thread(target=self.zmq_listen, daemon=True)
        listen_thread.start()

        print("[ZMQClient] listening...")

    def zmq_listen(self):
        poll = zmq.Poller()
        poll.register(self.socket, zmq.POLLIN)
        last_poll_at = int(time.time())

        while True:
            sockets = dict(poll.poll(1000))

            diff = int(time.time()) - last_poll_at
            # If the src/__main__.py process has stopped, then the proxy should kill itself
            if diff >= TIMEOUT_AFTER_SECONDS_NO_POLL:
                print(f'[Proxy] last poll was {diff} secs ago! Shutting down..')
                os.kill(os.getpid(), signal.SIGTERM)

            if sockets:
                message_raw = self.socket.recv()

                # ZMQ typing is not accurate here
                if type(message_raw) is not bytes:
                    raise Exception(f'[Proxy] Could not parse message of type {type(message_raw)}')

                message_raw_str = message_raw.decode('utf-8')
                message = json.loads(message_raw)

                if message['type'] in ['forward', 'forward_and_intercept']:
                    self.forward_flow(message)

                elif message['type'] == 'drop':
                    self.drop_flow(message)

                elif message['type'] == 'forward_all':
                    self.forward_all()

                elif message['type'] == 'enable_intercept':
                    self.set_intercept_enabled(message['value'])

                elif message['type'] == 'enable_recording':
                    self.set_recording_enabled(message['value'])

                elif message['type'] == 'set_settings':
                    self.set_settings(message['value'])

                elif message['type'] == 'poll':
                    last_poll_at = int(time.time())

    # ---------------------------------------------------------------------------
    # Actions:
    # ---------------------------------------------------------------------------
    def forward_flow(self, message):
        type = message['type']
        modified_flow = message['flow']

        print(f'[Proxy] {type} flow {modified_flow["uuid"]}')
        flow = self.intercepted_flows.pop(0)

        # Overwrite the MitmProxy flow with the values of the Pntest Flow:
        if flow.websocket:
            flow.websocket.messages[-1].content = modified_flow['websocket_messages'][-1]['content'].encode()

        elif flow.response:
            flow.response.status_code = modified_flow['response']['status_code']
            flow.response.headers = self.__convert_headers_for_mitm(modified_flow['response']['headers'])
            flow.response.content = modified_flow['response']['content'].encode()

        else:
            flow.request.path = modified_flow['request']['path']
            flow.request.method = modified_flow['request']['method']
            flow.request.host = modified_flow['request']['host']
            flow.request.port = modified_flow['request']['port']
            flow.request.headers = self.__convert_headers_for_mitm(modified_flow['request']['headers'])
            flow.request.content = modified_flow['request']['content'].encode()
            flow.intercept_response = (message['type'] == 'forward_and_intercept')

        flow.resume()

    def forward_all(self):
        for flow in self.intercepted_flows:
            print(f'[Proxy] forwarding flow {flow.id}')
            flow.intercept_response = False
            flow.resume()

    def drop_flow(self, message):
        type = message['type']
        modified_flow = message['flow']

        print(f'[Proxy] {type} flow {modified_flow["uuid"]}')
        flow = self.intercepted_flows.pop(0)
        flow.kill()

    def intercept_flow(self, flow):
        flow.intercept()
        self.intercepted_flows.append(flow)

    def set_intercept_enabled(self, enabled: bool):
        self.intercept_enabled = enabled

    def set_recording_enabled(self, enabled: bool):
        self.recording_enabled = enabled

    def set_settings(self, settings):
        self.settings = settings

    # ---------------------------------------------------------------------------
    # MitmProxy Events:
    # ---------------------------------------------------------------------------
    def request(self, flow: MitmHTTPFlow):
        print('[Proxy] HTTP request')
        # The default homepage at http://pntest
        if flow.request.host == 'pntest':
            flow.response = MitmResponse.make(200, self.__proxy_home_page_html(), {"content-type": "text/html"})

        if not self.should_request_be_captured(flow):
            return

        request_state = cast(ProxyRequest, convert_dict_bytes_to_strings(flow.request.get_state()))
        request_state['flow_uuid'] = flow.id
        request_state['type'] = 'request'
        request_state['client_id'] = self.client_id
        request_state['intercepted'] = self.__should_intercept_request(flow)
        request_state['content'] = flow.request.text or ''
        # NOTE: WE have to re-write this when in docker_proxy because iptables intercepts this after the host
        # has been replaced with an ip address
        if flow.request.host_header:
            request_state['host'] = flow.request.host_header
        self.__send_message(request_state)

        if request_state['intercepted']:
            self.intercept_flow(flow)

    def response(self, flow: MitmHTTPFlow):
        print('[Proxy] HTTP response')
        intercept_response = getattr(flow, 'intercept_response', False)

        if flow is None:
            return

        if flow.response is None or not self.should_request_be_captured(flow):
            return

        response_state = cast(ProxyResponse, convert_dict_bytes_to_strings(flow.response.get_state()))
        response_state['flow_uuid'] = flow.id
        response_state['type'] = 'response'
        response_state['intercepted'] = intercept_response

        # NOTE: Even though it breaks the type-checking, we convert this to a string so it can be sent
        # over zmq, then in ProxyZmqServer we decode base64 and decode back to raw bytes
        # TODO: Have a seperate ProxyResponseZmq type with content: str
        if flow.response.content is None:
            response_state['content'] = '' # type:ignore
        else:
            response_state['content'] = flow.response.content.hex() # type:ignore

        self.__send_message(response_state)

        if intercept_response:
            self.intercept_flow(flow)

    def websocket_message(self, flow: MitmHTTPFlow):
        print('[Proxy] websocket message')
        if flow.websocket is None or self.recording_enabled == False:
            return

        message = flow.websocket.messages[-1]
        direction = 'outgoing' if message.from_client else 'incoming'

        message_state: ProxyWebsocketMessage = {
            'type': 'websocket_message',
            'flow_uuid': flow.id,
            'direction': direction,
            'content': message.content.decode(),
            'intercepted': self.__should_intercept_request(flow)
        }
        self.__send_message(message_state)

        if message_state['intercepted']:
            flow.intercepted_message = message  # type: ignore
            self.intercept_flow(flow)

    # ---------------------------------------------------------------------------
    # Private methods:
    # ---------------------------------------------------------------------------
    def __should_intercept_request(self, flow):
        return self.intercept_enabled

    def __convert_headers_for_mitm(self, headers):
        headers_obj = json.loads(headers)
        headers_list = []

        for key, value in headers_obj.items():
            header = (key.encode(), value.encode())
            headers_list.append(header)

        return Headers(headers_list)

    def __send_message(self, message):
        try:
            self.socket.send_string(json.dumps(message))
        except UnicodeDecodeError:
            print(f'ERROR => Could not send message for flow {message["flow_uuid"]}')
            return

    def should_request_be_captured(self, flow: MitmHTTPFlow) -> bool:
        if self.recording_enabled == False:
            return False

        if self.settings is None:
            return True

        # Check host
        host_list = self.settings['capture_filters']['host_list']
        has_hosts = len(host_list) > 0
        host_setting = self.settings['capture_filters']['host_setting']

        if flow.request.host in host_list and host_setting == 'exclude' and has_hosts:
            return False

        if flow.request.host not in host_list and host_setting == 'include' and has_hosts:
            return False

        return True

    def __proxy_home_page_html(self) -> str:
        html = HOME_PAGE_HTML

        html = html.replace('{{client_id}}', str(self.client_id))
        html = html.replace('{{proxy_port}}', args.port)
        return html

def convert_dict_bytes_to_strings(d):
    new_d = {}
    for key, value in d.items():
        if type(value) is bytes and key != 'content':
            new_d[key] = value.decode()
        elif key == 'headers':
            new_d[key] = convert_headers_bytes_to_strings(value)
        else:
            new_d[key] = value

    return new_d

def convert_headers_bytes_to_strings(headers):
    new_headers = []
    for header in headers:
        new_header = (header[0].decode(), header[1].decode())
        new_headers.append(new_header)

    return new_headers

proxy_events = ProxyEventsAddon(int(args.client_id))
proxy_events.zmq_connect()

addons = [proxy_events]


# TODO:
# we should have a function that makes the activity maps & separate that from the functions that do curve fits or whatever. Since the activity maps are the most time consuming part of the process.
#

import os
import sys

import pickle
import numpy as np

import utils.model_handling as model_funcs
import utils.dataset_generation as iso_data
import utils.histogram_analysis as hist_funcs

current_path = os.path.dirname(os.path.abspath(__file__))
parent_path = os.path.dirname(current_path)
if parent_path not in sys.path: sys.path.append(parent_path)
santi_path = parent_path+'/santi_iso_response'
santi_etc_path = os.path.join(santi_path, 'etc')
from santi_iso_response.iso_response import utils as santi_utils

# NOTE:This script uses about 19GB of memory with batch_size=100; num_images=900; num_targets=166; num_comparisons=166

with open(os.path.join(santi_etc_path, 'meis.pkl'), 'rb') as g:
    mei = pickle.load(g)
num_mei = len(mei['images'])
neuron_mei = [mei['images'][idx] for idx in range(num_mei)]
target_neuron_ids = [idx for idx in range(num_mei) if mei['performance'][idx] > 0.01]
total_num_neurons = len(neuron_mei)

experiment_params = dict()
experiment_params['batch_size'] = 100
experiment_params['target_neuron_ids'] = target_neuron_ids
experiment_params['num_comparisons'] = len(target_neuron_ids)
experiment_params['min_angle'] = 15
experiment_params['x_range'] = (-2.0, 2.0)
experiment_params['y_range'] = (-2.0, 2.0)
experiment_params['num_images'] = int(30**2)
experiment_params['image_scale'] = 12
experiment_params['target_activity'] = 0.5
experiment_params['iso_window_bounds'] = ((-1, 1), (-1, 1))
experiment_params['comp_method'] = 'closest'
experiment_params['output_directory'] = parent_path+'/iso_analysis/'
experiment_params['save_prefix'] = 'santi_windowed_'

def get_curvatures_from_target_comparison_vectors(model, target_vectors, comparison_vectors, act_func, kwargs):
    contour_dataset, _ = iso_data.get_contour_dataset(
        target_vectors,
        comparison_vectors,
        kwargs['x_range'],
        kwargs['y_range'],
        kwargs['num_images'],
        kwargs['image_scale'],
        return_datapoints=False
    )
    num_neurons = len(target_vectors)
    num_planes = len(comparison_vectors)
    num_edge_images = int(np.sqrt(kwargs['num_images']))
    response_images = np.zeros((num_neurons, num_planes, num_edge_images, num_edge_images))
    all_iso_curvatures = []
    all_attn_curvatures = []
    all_mei_lengths = []
    for target_index, target_proj_matrix in enumerate(contour_dataset['proj_matrix']):
        sub_mei_lengths = []
        sub_iso_curvatures = []
        sub_attn_curvatures = []
        for plane_index, proj_matrix in enumerate(target_proj_matrix): # 1 plane at a time
            datapoints = iso_data.inject_data(proj_matrix, contour_dataset['proj_datapoints'],
                kwargs['image_scale'])
            activations = model_funcs.get_normalized_activations(
                model,
                [kwargs['target_neuron_ids'][target_index]],
                [[datapoints]],
                act_func
            )
            response_images[target_index, plane_index, ...] = np.squeeze(activations).copy()

            iso_curvatures, attn_curvatures = hist_funcs.compute_curvature_poly_fits(
                activations,
                contour_dataset,
                kwargs['target_activity'],
                bounds=kwargs['iso_window_bounds']
                #measure_loc='right'
                #measure_upper_right=False
            )
            ## Add code to measure mei length
            #mei = kwargs['mei']
            # mei_lengths = get_mei_lengths(mei, proj_matrix)
            #     -- normalize mei
            #     -- project mei using proj_matrix
            mei_lengths = None
            sub_mei_lengths.append(mei_lengths)
            sub_iso_curvatures.append(iso_curvatures[0][0])
            sub_attn_curvatures.append(attn_curvatures[0][0])
        all_mei_lengths.append(sub_mei_lengths)
        all_iso_curvatures.append(sub_iso_curvatures)
        all_attn_curvatures.append(sub_attn_curvatures)
    out_dict = {
        'mei_lengths':all_mei_lengths,
        'contour_dataset':contour_dataset,
        'iso_curvatures':all_iso_curvatures,
        'response_images': response_images,
        'attn_curvatures':all_attn_curvatures
    }
    return out_dict

def get_curvatures_from_exciting_vectors(model, exciting_vectors, act_func, kwargs):
    iso_vectors = iso_data.compute_comp_vectors(
        exciting_vectors,
        kwargs['target_neuron_ids'],
        kwargs['min_angle'],
        kwargs['num_comparisons'],
        comp_method=kwargs['comp_method']
    )
    comparison_vector_ids = iso_vectors[0]
    target_vectors = iso_vectors[1]
    comparison_vectors = iso_vectors[2]
    kwargs['mei'] = exciting_vectors
    out_dict = get_curvatures_from_target_comparison_vectors(
        model,
        target_vectors,
        comparison_vectors,
        act_func,
        kwargs
    )
    out_dict['comparison_vector_ids'] = comparison_vector_ids
    out_dict['target_vectors'] = target_vectors
    out_dict['comparison_vectors'] = comparison_vectors
    return out_dict

if not os.path.exists(experiment_params['output_directory']):
    os.makedirs(experiment_params['output_directory'])
np.savez(experiment_params['output_directory']+experiment_params['save_prefix']+'meis_params.npz',
    data=experiment_params)

model = santi_utils.load_model()

"""
Maximum Exciting Images
"""
comp_results = get_curvatures_from_exciting_vectors(
    model,
    neuron_mei,
    santi_utils.get_activations_cell,
    experiment_params
)
np.savez(experiment_params['output_directory']+experiment_params['save_prefix']+'meis.npz',
    data=comp_results)
print('Maximum exciting images experiment complete.')

#"""
#Random planes from MEIs
#"""
#iso_vectors = iso_data.compute_rand_vectors(
#    [neuron_mei[idx] for idx in experiment_params['target_neuron_ids']],
#    experiment_params['num_comparisons']
#)
#target_vectors = iso_vectors[0]
#orth_vectors = iso_vectors[1]
#comp_results = get_curvatures_from_target_comparison_vectors(
#    model,
#    target_vectors,
#    orth_vectors,
#    santi_utils.get_activations_cell,
#    experiment_params
#)
#comp_results['target_vectors'] = target_vectors
#comp_results['comparison_vectors'] = orth_vectors
#np.savez(experiment_params['output_directory']+experiment_params['save_prefix']+'rand.npz',
#    data=comp_results)
#print('Random plane experiment complete.')

#"""
#Maximum Exciting Stimuli
#"""
#iid_stim_images = np.squeeze(model.data.train()[0])
#num_stim = iid_stim_images.shape[0]
#stim_activity = santi_utils.get_activations(model, iid_stim_images)
#stim_argsort = np.argsort(stim_activity, axis=0)
#neuron_mes = [] # most exciting stimulus
#for neuron_idx in range(total_num_neurons):
#    most_exciting_stim = iid_stim_images[stim_argsort[0, neuron_idx], ...]
#    most_exciting_stim = iso_data.normalize_vector(most_exciting_stim)
#    neuron_mes.append(most_exciting_stim)
#del iid_stim_images
#"""
#### PROBLEM - Some neurons have the same exciting stim.
#If this is the case we should assign the second most exciting stim.
#But then we have an assignment problem...
#Not sure what to do here.
#"""
#comp_results = get_curvatures_from_exciting_vectors(
#    model,
#    neuron_mes,
#    santi_utils.get_activations_cell,
#    experiment_params
#)
#np.savez(experiment_params['output_directory']+experiment_params['save_prefix']+'stim.npz',
#    data=comp_results)
#print('Maximum exciting stimuli experiment complete.')


import os
from urllib import request
from zipfile import ZipFile
import sqlite3

DATABASE_DIR = '../database/'
DATA_FILES_DIR = DATABASE_DIR + 'data_files/'
DATABASE_NAME = 'num2word.db'
LETTER_DICT_RU = {'н': '0', 'м': '0', 'г': '1', 'ж': '1', 'д': '2', 'т': '2', 'к': '3', 'х': '3', 'ч': '4', 'щ': '4',
      'п': '5', 'б': '5', 'ш': '6', 'л': '6', 'с': '7', 'з': '7', 'в': '8', 'ф': '8', 'р': '9', 'ц': '9'}

def calc_word2num(word):
    num = ''
    for l in word:
        num += LETTER_DICT_RU.get(l, '')
    return num

def load_dictionary_files():
    try:
        f = open('../conf/dictionary_RU.txt')
        for url in f:
            if url[0] == '#' or url == '':
                continue
            print(url)
            if not os.path.exists(DATA_FILES_DIR):
                os.mkdir(DATA_FILES_DIR)
            dest = DATA_FILES_DIR + url.split('/')[-1]
            request.urlretrieve(url, dest)
            zip = ZipFile(dest)
            zip.extractall(DATA_FILES_DIR)
            zip.close()
            os.remove(zip.filename)
    finally:
        f.close()

def init_database_schema():
    con = sqlite3.connect(DATABASE_DIR + DATABASE_NAME)
    cur = con.cursor()
    cur.execute("DROP TABLE IF EXISTS words")
    cur.execute('''
            CREATE TABLE IF NOT EXISTS words (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                value_text TEXT NOT NULL,
                value_num TEXT NOT NULL,
                UNIQUE(value_text)
            )

            ''')

def load_dict_file_to_database(filename):
    if not os.path.exists(filename):
        exit()
    con = sqlite3.connect(DATABASE_DIR + DATABASE_NAME)
    cur = con.cursor()
    data = []
    for row in open(filename, 'r'):
        data.append([row, calc_word2num(row)])
    cur.executemany("INSERT OR IGNORE INTO words (value_text, value_num) VALUES (?, ?);", data)
    con.commit()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('main', '0003_game'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='board',
            name='owner',
        ),
        migrations.AlterField(
            model_name='game',
            name='ai_board',
            field=models.CharField(default='0', max_length=320),
        ),
        migrations.AlterField(
            model_name='game',
            name='player_board',
            field=models.CharField(default='0', max_length=320),
        ),
        migrations.DeleteModel(
            name='Board',
        ),
    ]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri May 15 12:18:17 2020

Preparing dataset -> from audio files to mel spectrograms using librosa package

@author: piotrek
"""

import librosa
import os
import numpy as np
import librosa.display
import matplotlib
import time

import warnings
warnings.filterwarnings('ignore')

matplotlib.use('Agg')
import matplotlib.pyplot as plt

PATH_FILES = './fma_small/'

OUTPUT_PATH = 'spectrograms'

os.chdir('Documents/studia/mgr/master-thesis/')

def plot_spectrogram(waveform, sampling_rate):

    S = librosa.feature.melspectrogram(y=waveform, sr=sampling_rate)
    librosa.display.specshow(librosa.power_to_db(S, ref=np.max))

def save_img(dir, filename, failIfNotExists = False):

    if not os.path.exists(dir):
        if failIfNotExists:
            raise OSError('Provided directory does not exist ', dir)

        os.makedirs(dir)

    save_path = os.path.join(dir, filename.split('.')[0])
    plt.savefig(save_path + '.png')

def count_folders(dir):
    folders = [dirnames for _, dirnames, filenames in os.walk(dir)]

    # only top level folders
    return len(folders[0])

import shutil

def move_files(top_level_path):
    for folder in os.listdir(top_level_path):
        for file in os.listdir(os.path.join(top_level_path, folder)):
            print(file)
            shutil.move(os.path.join(top_level_path, folder, file), os.path.join(top_level_path, file))

move_files('spectrograms/')

start = time.process_time()

processed = 0
folders_count = count_folders(PATH_FILES)
for folder in os.listdir(PATH_FILES):
    processed += 1
    if os.path.exists(os.path.join(OUTPUT_PATH, folder)): # in ['018', '044', '066', '072', '076', '090']:
        print('skipped folder ', folder)
        continue
    print('processing ', folder)
    for file in os.listdir(os.path.join(PATH_FILES, folder)):

        try:
            y, sr  = librosa.load(os.path.join(PATH_FILES, folder, file))
        except Exception:
            print(f'problem with {file} occured')
            continue

        fig, ax = plt.subplots(1)
        fig.subplots_adjust(left=0,right=1,bottom=0,top=1)
        ax.axis('off')

        plot_spectrogram(waveform=y, sampling_rate=sr)
        save_img(os.path.join(OUTPUT_PATH, folder), file)

        plt.close(fig)

    print('done printing ', folder)
    print(f'processed {processed}/{folders_count}')

print('done')

end = time.process_time()
print('it took ', end - start)


from google.cloud import storage
import os
import re

if not os.path.exists("DATA"):
    os.mkdir("DATA")

storage_client = storage.Client.from_service_account_json('API Project-4675d7ee66b1.json')

ls_itr = storage_client.list_buckets(prefix="scanvid--images--")
for b in ls_itr:
    if "scanvid--images--" not in b.name:
        continue
    results = re.search("scanvid--images--(\d+)", b.name)
    if not results:
        continue
    label = results.group(1)
    bucket = storage_client.get_bucket(b.name)
    blobs = bucket.list_blobs()
    idx = 0
    for blob in blobs:
        if not os.path.exists(os.path.join("DATA", label)):
            os.mkdir(os.path.join("DATA", label))
        dir = os.path.join("DATA", label)
        file = os.path.join(dir, "{}.png".format(str(idx).zfill(10)))
        data = bucket.blob(blob.name)
        data.download_to_filename(file)
        idx += 1


import pygame
from gameScreen import GameScreen

pygame.init()
SCREEN_WIDTH = 1200
SCREEN_HEIGHT = 800
screen = pygame.display.set_mode([SCREEN_WIDTH, SCREEN_HEIGHT])
screen.fill('black')
timer = pygame.time.Clock()
fps = 30
pygame.display.set_caption("Container Pre-Marshalling Game")

def run_game():
    scene = GameScreen(screen)
    running = True
    while running:
        timer.tick(fps)
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                running = False
                break
            scene.handle_event(event)
            scene.draw(screen)

        pygame.display.flip()

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    run_game()

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


from telegram.ext import BaseHandler, CommandHandler, ConversationHandler, MessageHandler, filters

from pdf_bot.consts import TEXT_FILTER
from pdf_bot.telegram_handler import AbstractTelegramHandler
from pdf_bot.telegram_internal import TelegramService

from .merge_service import MergeService

class MergeHandler(AbstractTelegramHandler):
    _MERGE_COMMAND = "merge"

    def __init__(self, merge_service: MergeService, telegram_service: TelegramService) -> None:
        self.merge_service = merge_service
        self.telegram_service = telegram_service

    @property
    def handlers(self) -> list[BaseHandler]:
        return [
            ConversationHandler(
                entry_points=[
                    CommandHandler(self._MERGE_COMMAND, self.merge_service.ask_first_pdf)
                ],
                states={
                    MergeService.WAIT_MERGE_PDF: [
                        MessageHandler(filters.Document.PDF, self.merge_service.check_pdf),
                        MessageHandler(TEXT_FILTER, self.merge_service.check_text),
                    ],
                },
                fallbacks=[CommandHandler("cancel", self.telegram_service.cancel_conversation)],
                allow_reentry=True,
            )
        ]


import os

from core.errors import PredictException, ModelLoadException
from core.config import MODEL_NAME, MODEL_PATH
from loguru import logger

class MachineLearningModelHandlerScore(object):
    model = None

    # Similarity = (A.B) / (||A||.||B||)

    @classmethod
    def predict(cls, input, load_wrapper=None, method="predict"):
        clf = cls.get_model(load_wrapper)

        def cosine(vec_a, vec_b):
            dot = sum(a*b for a, b in zip(vec_a, vec_b))
            norm_a = sum(a*a for a in vec_a) ** 0.5
            norm_b = sum(b*b for b in vec_b) ** 0.5

            # Cosine similarity
            cos_sim = dot / (norm_a * norm_b)
            return cos_sim

        result = []
        input = input.flatten()
        for dct in clf:
            result.append({list(dct.keys())[0]: cosine(input, list(dct.values())[0])})
        ids = []
        sorted_res = sorted(result, key=lambda item: list(item.values())[0])[:10]
        return sorted_res

    @classmethod
    def get_model(cls, load_wrapper):
        if cls.model is None and load_wrapper:
            cls.model = cls.load(load_wrapper)
        return cls.model

    @staticmethod
    def load(load_wrapper):
        model = None
        if MODEL_PATH.endswith("/"):
            path = f"{MODEL_PATH}{MODEL_NAME}"
        else:
            path = f"{MODEL_PATH}/{MODEL_NAME}"
        if not os.path.exists(path):
            message = f"Machine learning model at {path} not exists!"
            logger.error(message)
            raise FileNotFoundError(message)
        with open(path, 'rb') as handle:
            model = load_wrapper(handle)
        if not model:
            message = f"Model {model} could not load!"
            logger.error(message)
            raise ModelLoadException(message)
        return model


from typing import Union

import discord
from unidecode import unidecode  # pip install unidecode

class Misc():

    def display_time(
        seconds: int,
        granularity: int = 2
    ) -> str:
        intervals = (
            ('weeks', 604800),  # 7 days * 24 hours * 60 minutes * 60 seconds
            ('days', 86400),  # 24 hours * 60 minutes * 60 seconds
            ('hours', 3600),  # 60 minutes * 60 seconds
            ('minutes', 60),  # 60 seconds
            ('seconds', 1),
        )

        result = []
        for name, count in intervals:
            value = seconds // count
            if value:
                seconds -= value * count
                if value == 1:
                    name = name.rstrip("s")
                result.append(f"{value} {name}")
        time = ", ".join(result[:granularity])
        return time

    def alpha_unidecode_lower(
        string: str
    ) -> str:
        # Process the string:
        # 1. Remove non-alphanumeric characters.
        # 2. Convert from Unicode to ASCII equivalent.
        # 3. Convert from uppercase to lowercase.
        return unidecode("".join(filter(str.isalpha, string))).lower()

    def send_log(
        context: Union[discord.ApplicationContext, discord.Interaction],
        event: str
    ) -> None:
        try:
            guild_name = unidecode(context.guild.name)
            guild_id = context.guild_id
            channel_name = unidecode(context.channel.name)
            channel_id = context.channel_id
        except AttributeError:
            channel_info = f"private messages with {context.user}"
        else:
            channel_info = f"{guild_name} ({guild_id}) #{channel_name} ({channel_id})"

        log = f"{event[:-1]} in {channel_info}."
        print(log)

    def decapitalize(sting: str):
        return sting[:1].lower() + sting[1:]


import os, sys
import json

WANDB_API_KEY = os.getenv("WANDB_API_KEY")
with open("../configs/wandb.json") as fin:
    wandb_config = json.load(fin)
    if WANDB_API_KEY == None:
        WANDB_API_KEY = wandb_config["api_key"]
# print(WANDB_API_KEY)

logf = sys.argv[1]
outf = open(sys.argv[2], "w")
start = int(sys.argv[3])
end = int(sys.argv[4])

dataset_name = sys.argv[5]
model_name = sys.argv[6]
nums = sys.argv[7].split(",")
# print(f"{dataset_name}_{model_name}")
if len(sys.argv) == 8:
    project_name = "kt_toolkits"
else:
    project_name = sys.argv[8]

cmdpre = f"WANDB_API_KEY={WANDB_API_KEY} nohup "
endcmdpre =f"WANDB_API_KEY={WANDB_API_KEY} "

idx = 0
with open(logf, "r") as fin:
    i = 0
    lines = fin.readlines()
    l = []
    num = 0
    while i < len(lines):
        if lines[i].strip().startswith("wandb: Creating sweep from: "):
            fname = lines[i].strip().split(": ")[-1].split("/")[-1]
        else:
            print("error!")
        if lines[i+3].strip().startswith("wandb: Run sweep agent with: "):
            sweepid = lines[i+3].strip().split(": ")[-1]
        else:
            print("error!")
        fname = fname.split(".")[0]
        # print(f"fname is {fname}")
        if not fname.startswith(dataset_name) or fname.find("_" + model_name + "_") == -1:
            i += 4
            continue
        # print(f"dataset_name: {dataset_name}, model_name: {model_name}, fname: {fname}")
        if idx >= start and idx < end:
            cmd = "CUDA_VISIBLE_DEVICES=" + str(nums[num]) +" " + cmdpre + sweepid + " &"
            print(cmd)
            outf.write(cmd + "\n")
            num += 1
        idx += 1
        i += 4


import numpy as np
import matplotlib.pyplot as plt

SKETCHES = 'sketches'
PRED_SKETCHES = 'pred_sketches'
SKETCH_IDS = 'sketch_ids'

def strokes_from_sketch(sketch):
    indices = np.where(sketch[:, 2] == 1)[0] + 1
    strokes = []
    prev_idx = 0
    stroke = stroke_to_xy(sketch)
    for idx in indices:
        strk = stroke[prev_idx:idx, :]
        if len(strk) > 0:
            strokes.append(strk)
        prev_idx = idx
    return strokes

def sketch_to_xy(sketch):
    xy = sketch[:, :2][::-1].cumsum(axis=0)
    xy[:, 0] *= -1

    # MinMax Normalize
    xmin, xmax = xy[:, 0].min(), xy[:, 0].max()
    ymin, ymax = xy[:, 1].min(), xy[:, 1].max()

    def normalise(a, amin, amax):
        l = amax - amin
        a -= amin
        return (a - amin) / l if l > 0 else a

    # Reposition
    xy[:, 0] = normalise(xy[:, 0], xmin, xmax)
    xy[:, 1] = normalise(xy[:, 1], ymin, ymax)
    xy -= xy[0,:] # Start at origin

    return xy

def save_sketch_as_png(sketch, filename='sample.png'):
    plt.figure(1)
    strokes = strokes_from_sketch(sketch)
    for xy in strokes:
        plt.plot(xy[:, 0], xy[:, 1], 'b')
#         plt.plot(xy[0, 0], xy[0, 1], 'ro')
#         plt.plot(xy[-1, 0], xy[-1, 1], 'bo')
    plt.savefig(filename, bbox_inches='tight')
    plt.close(1)

def plot_sketch(sketch):
    plt.figure()
    strokes = strokes_from_sketch(sketch)
    for xy in strokes:
        plt.plot(xy[:, 0], xy[:, 1], 'b')
#         plt.plot(xy[0, 0], xy[0, 1], 'ro')
#         plt.plot(xy[-1, 0], xy[-1, 1], 'bo')
    plt.show()

def stroke_to_xy(stroke):
    return sketch_to_xy(stroke)

def processed_data(data):
    all_data = {}
    for d in data:
        cat = d[0].split('%')[0]        # category
        img_id = int(d[1].split('%')[1])   # image_name
        sketch_id = int(d[0].split('%')[1])

        key = (cat, img_id)
        if key not in all_data:
            all_data[key] = {SKETCHES: [], PRED_SKETCHES: [], SKETCH_IDS: []}
        all_data[key][SKETCHES].append(d[2])
        all_data[key][PRED_SKETCHES].append(d[3])
        all_data[key][SKETCH_IDS].append(sketch_id)
    return all_data

def get_sketches(data, cat, img_id, exclude_id=None):
    sketches = data[(cat, img_id)][SKETCHES]
    if exclude_id != None:
        ids = data[(cat, img_id)][SKETCH_IDS]
        sketches = [sketches[i] for i in range(len(sketches)) if ids[i] != exclude_id]
    return sketches

def get_sketch_ids(data, cat, img_id):
    return data[(cat, img_id)][SKETCH_IDS]

def get_pred_sketches(data, cat, img_id):
    return data[(cat, img_id)][PRED_SKETCHES]


# from sqlite3 import connect
# from wsgiref.util import request_uri
import requests
from bs4 import BeautifulSoup
import pandas
import argparse
import connect

parser=argparse.ArgumentParser()
parser.add_argument("--page_num_max",help="enter the number of pages to parse",type=int)
parser.add_argument("--dbname",help="enter the number of pages to parse",type=int)
# parser.add_argument("--dbname",help="enter the name of db",type=str())
args=parser.parse_args()

oyo_url="https://www.oyorooms.com/search?location=Nagpur%2C%20Maharashtra%2C%20India&city=Nagpur&searchType=city&coupon=&checkin=03%2F02%2F2022&checkout=04%2F02%2F2022&roomConfig%5B%5D=2&showSearchElements=false&country=india&guests=2&rooms=1&filters%5Bcity_id%5D=43"

page_num_MAX=args.page_num_max
scrapped_info_list=[]
connect.connect(args.dbname)

for page_num in range(1,page_num_MAX):
    # url=oyo_url+str(page_num)
    # print("GET request for:"+url)
    req=requests.get(oyo_url+str(page_num))
    content=req.content

# print(content)
    soup=BeautifulSoup(content,"html.parser")

    all_hotels=soup.find_all("div",{"class":"hotelCardListting",})
    # scrapped_info_list={}

for hotel in all_hotels:
    hotel_dict={}
    hotel_dict["name"]= hotel_name=hotel.find("h3",{"calss":"ListingHotelDescription_hotelName"}).text
    hotel_dict["address"]= hotel_address=hotel.find("span",{"itemprop":"streetAddress"}).text
    hotel_dict["price"]= hotel_price=hotel.find("span",{"class":"ListingPrice_finalPrice"}).text

    try:
         hotel_dict["rating"] =hotel_rating=hotel.find("span",{"class":"hotelRating_ratingSummary"}).text
    except AttributeError:
        # hotel_dict["rating"]=None
        pass

    parent_amenities_element=hotel.find("div",{"c;ass":"amenityWrapper"})

    amenities_list=[]

    for  amenity in parent_amenities_element.finda_ll("div",{"c;ass":"amenityWrapper_amenity"}):
        amenities_list.append(amenity.find("span",{"class":"d-body-sm"}).text.strip())

    hotel_dict["amenities"]=', '.join(amenities_list[:-1])

    scrapped_info_list.append(hotel_dict)
    connect.insert_list_table(args.dbname, tuple(hotel_dict.values()))

    # print(hotel_name,hotel_address, hotel_price, hotel_rating,amenities_list)

dataFrame=pandas.DataFrame(scrapped_info_list)
print("creating csv file..")
# dataFrame.to_csv("oyo.csv")
connect.get_hotel_info(args.dbname)


#!/usr/bin/env python
# coding: utf-8

# In[163]:

from flask import Flask,request, url_for, redirect, render_template
import pickle
import numpy as np

# In[168]:

app = Flask(__name__,template_folder=r"C:\Users\SANYA\Desktop\work\Hackathon BVIP")

model=pickle.load(open('model.pkl','rb'))

# In[169]:

@app.route('/')
def hello_world():
    return render_template('input.html.html')

# In[170]:

@app.route('/predict',methods=['POST','GET'])
def predict():
    int_features=[x for x in request.form.values()]
    f=np.array(int_features)
    prediction=model.predict([f])
    output='{0:.{1}f}'.format(prediction[0][1], 2)
    return render_template('input.html.html')

# In[ ]:

if __name__ == "__main__":
   from werkzeug.serving import run_simple
   run_simple('localhost', 9000, app)

# In[ ]:

# In[ ]: