import numpy as np
import pandas as pd
import pickle
from tqdm import tqdm
import os
import time

class DataLoader():
    def __init__(self, dataset, min_num_buy, train_negative_samples):
        self.dataset_path = './data/' + dataset + '/'
        self.min_num_buy = min_num_buy
        self.train_negative_samples = train_negative_samples

    def _is_data_exist(self):
        return os.path.exists(self.dataset_path + 'data_dict.pkl')

    def _get_statistics_file(self):
        df = pd.read_csv(self.dataset_path + 'UserBehavior.csv', header=None)
        users = list(set(df.iloc[:, 0].tolist()))

        users_behaviors = []
        for user in tqdm(users):
            behaviors_statistics = {'pv': 0, 'cart': 0, 'fav': 0, 'buy': 0}
            user_behaviors = df[df.iloc[:, 0] == user]
            user_behaviors_num = user_behaviors.iloc[:, 3].value_counts()
            for each in ['pv', 'cart', 'fav', 'buy']:
                try:
                    each_num = user_behaviors_num[each]
                    behaviors_statistics[each] = each_num
                except:
                    break
            user_behavior = list(behaviors_statistics.values())
            user_behavior.append(user)
            users_behaviors.append(user_behavior)

        with open(self.dataset_path + 'statistics.pkl', 'wb') as f:
            pickle.dump(users_behaviors, f)

    def _load_statistics_file(self):
        print('... Loading Data Statistics ...')

        with open(self.dataset_path+'statistics.pkl', 'rb') as f:
            self.statistics = np.array(pickle.load(f))

    def _data_filtering(self):
        print('... Data Filtering ...')

        filtered_statistics = self.statistics[self.statistics[:, 3] >= self.min_num_buy, :]
        filtered_statistics = filtered_statistics[filtered_statistics[:, 0] <= 800, :]
        self.filtered_users = filtered_statistics[:, 4]

        raw_df = pd.read_csv(self.dataset_path + 'UserBehavior.csv', header=None)
        self.filtered_df = raw_df[raw_df.iloc[:, 0].isin(self.filtered_users)]

    def _id_mapping(self, id_name):
        if id_name == 'user_id':
            print('... Generating User Index Mapping ...')
            id_list = np.array(self.filtered_df.iloc[:, 0])
        elif id_name == 'item_id':
            print('... Generating Item Index Mapping ...')
            id_list = np.array(self.filtered_df.iloc[:, 1])
        elif id_name == 'cat_id':
            print('... Generating Category Index Mapping ...')
            id_list = np.array(self.filtered_df.iloc[:, 2])

        id_list = np.unique(id_list)
        id_list.sort()
        id_list = id_list.tolist()
        id_list_len = len(id_list)
        new_id_list = [i for i in range(1, id_list_len + 1)]
        id_mapping_dict = dict(zip(id_list, new_id_list))

        return id_mapping_dict

    def _data_unification(self):
        print('... Data Unifying ...')

        self.filtered_df.iloc[:, 1] = self.filtered_df.iloc[:, 1].map(lambda x: self._item_id_mapping_dict[x])
        self.filtered_df.iloc[:, 2] = self.filtered_df.iloc[:, 2].map(lambda x: self._cat_id_mapping_dict[x])
        self.filtered_df.iloc[:, 3] = self.filtered_df.iloc[:, 3].map(lambda x: self._behavior_mapping_dict[x])

        self.data_dict = dict()
        for user in tqdm(self.filtered_users):
            user_interactions = self.filtered_df[self.filtered_df.iloc[:, 0] == user]
            user_interactions = np.array(user_interactions)
            items = user_interactions[:, 1].reshape(-1)
            cats = user_interactions[:, 2].reshape(-1)
            behaviors = user_interactions[:, 3].reshape(-1)
            timestamps = user_interactions[:, 4].reshape(-1)
            user_dict = {'behaviors': behaviors, 'timestamps': timestamps, 'items': items, 'cats': cats}
            self.data_dict[self._user_id_mapping_dict[user]] = user_dict

    def load(self):
        if self._is_data_exist():
            print('... Data Dict Exists ...')
            with open(self.dataset_path + 'data_dict.pkl', 'rb') as f:
                self.data_dict = pickle.load(f)
            with open(self.dataset_path + 'user_item_cat_num.pkl', 'rb') as f:
                self.user_num, self.item_num, self.cat_num = pickle.load(f)

            # user_num = len(self.data_dict)
            # behaviors = []
            # items = []
            # for user in self.data_dict.keys():
            #     behaviors.extend(self.data_dict[user]['behaviors'])
            #     items.extend(self.data_dict[user]['items'])
            # behaviors = np.array(behaviors)
            # items_num = len(set(items))

            # print(user_num)
            # print(items_num)
            # print(np.count_nonzero(behaviors == 1))
            # print(np.count_nonzero(behaviors == 2))
            # print(np.count_nonzero(behaviors == 3))
            # print(np.count_nonzero(behaviors == 4))

            # time.sleep(50)
        else:
            if not os.path.exists(self.dataset_path + 'statistics.pkl'):
                self._get_statistics_file()
            self._load_statistics_file()
            self._data_filtering()

            self._user_id_mapping_dict = self._id_mapping('user_id')
            self._item_id_mapping_dict = self._id_mapping('item_id')
            self._cat_id_mapping_dict = self._id_mapping('cat_id')
            self._behavior_mapping_dict = {'pv': 1, 'fav': 2, 'cart': 3, 'buy': 4}

            self.user_num = len(self._user_id_mapping_dict)
            self.item_num = len(self._item_id_mapping_dict)
            self.cat_num = len(self._cat_id_mapping_dict)

            self._data_unification()

            with open(self.dataset_path + 'data_dict.pkl', 'wb') as f:
                pickle.dump(self.data_dict, f)
            with open(self.dataset_path + 'user_item_cat_num.pkl', 'wb') as f:
                pickle.dump([self.user_num, self.item_num, self.cat_num], f)

    def train_validation_test_data_generator(self):
        if os.path.exists(self.dataset_path + 'train_data_dict.pkl'):
            print('... Training Set Exists ...')
            with open(self.dataset_path + 'train_data_dict.pkl', 'rb') as f:
                self.train_data_dict = pickle.load(f)
            with open(self.dataset_path + 'val_test_dict.pkl', 'rb') as f:
                self.val_test_dict = pickle.load(f)

        else:
            print('... Training Set Generating ...')
            self.train_data_dict = dict()
            self.val_test_dict = dict()

            for i in tqdm(range(1, len(self.data_dict) + 1)):
                self.train_data_dict[i] = dict()
                self.val_test_dict[i] = dict()

                each_data = self.data_dict[i]
                buy_index = np.where(each_data['behaviors'] == 4)[0]

                self.train_data_dict[i]['behaviors'] = np.delete(each_data['behaviors'], buy_index[-2:])
                self.train_data_dict[i]['timestamps'] = np.delete(each_data['timestamps'], buy_index[-2:])
                self.train_data_dict[i]['items'] = np.delete(each_data['items'], buy_index[-2:])
                self.train_data_dict[i]['cats'] = np.delete(each_data['cats'], buy_index[-2:])

                buy_items = each_data['items'][buy_index]
                items_list = np.array([j for j in range(1, self.item_num + 1)])
                train_negative_samples = np.random.choice(np.setdiff1d(items_list, buy_items), len(buy_items) * self.train_negative_samples, replace=False)
                test_negative_samples = np.random.choice(np.setdiff1d(np.setdiff1d(items_list, buy_items), train_negative_samples), 99, replace=False)

                self.train_data_dict[i]['negative_samples'] = train_negative_samples
                self.val_test_dict[i]['val_target'] = buy_items[-2]
                self.val_test_dict[i]['test_target'] = buy_items[-1]
                self.val_test_dict[i]['negative_samples'] = test_negative_samples

            with open(self.dataset_path + 'train_data_dict.pkl', 'wb') as f:
                pickle.dump(self.train_data_dict, f)
            with open(self.dataset_path + 'val_test_dict.pkl', 'wb') as f:
                pickle.dump(self.val_test_dict, f)


from jwcrypto.common import JWException, JWKeyNotFound
from jwcrypto.common import JWSEHeaderParameter, JWSEHeaderRegistry
from jwcrypto.common import base64url_decode, base64url_encode
from jwcrypto.common import json_decode, json_encode
from jwcrypto.jwa import JWA
from jwcrypto.jwk import JWK, JWKSet

JWSHeaderRegistry = {
    'alg': JWSEHeaderParameter('Algorithm', False, True, None),
    'jku': JWSEHeaderParameter('JWK Set URL', False, False, None),
    'jwk': JWSEHeaderParameter('JSON Web Key', False, False, None),
    'kid': JWSEHeaderParameter('Key ID', False, True, None),
    'x5u': JWSEHeaderParameter('X.509 URL', False, False, None),
    'x5c': JWSEHeaderParameter('X.509 Certificate Chain', False, False, None),
    'x5t': JWSEHeaderParameter(
        'X.509 Certificate SHA-1 Thumbprint', False, False, None),
    'x5t#S256': JWSEHeaderParameter(
        'X.509 Certificate SHA-256 Thumbprint', False, False, None),
    'typ': JWSEHeaderParameter('Type', False, True, None),
    'cty': JWSEHeaderParameter('Content Type', False, True, None),
    'crit': JWSEHeaderParameter('Critical', True, True, None),
    'b64': JWSEHeaderParameter('Base64url-Encode Payload', True, True, None)
}
"""Registry of valid header parameters"""

default_allowed_algs = [
    'HS256', 'HS384', 'HS512',
    'RS256', 'RS384', 'RS512',
    'ES256', 'ES384', 'ES512',
    'PS256', 'PS384', 'PS512',
    'EdDSA', 'ES256K']
"""Default allowed algorithms"""

class InvalidJWSSignature(JWException):
    """Invalid JWS Signature.

    This exception is raised when a signature cannot be validated.
    """

    def __init__(self, message=None, exception=None):
        msg = None
        if message:
            msg = str(message)
        else:
            msg = 'Unknown Signature Verification Failure'
        if exception:
            msg += ' {%s}' % str(exception)
        super(InvalidJWSSignature, self).__init__(msg)

class InvalidJWSObject(JWException):
    """Invalid JWS Object.

    This exception is raised when the JWS Object is invalid and/or
    improperly formatted.
    """

    def __init__(self, message=None, exception=None):
        msg = 'Invalid JWS Object'
        if message:
            msg += ' [%s]' % message
        if exception:
            msg += ' {%s}' % str(exception)
        super(InvalidJWSObject, self).__init__(msg)

class InvalidJWSOperation(JWException):
    """Invalid JWS Object.

    This exception is raised when a requested operation cannot
    be execute due to unsatisfied conditions.
    """

    def __init__(self, message=None, exception=None):
        msg = None
        if message:
            msg = message
        else:
            msg = 'Unknown Operation Failure'
        if exception:
            msg += ' {%s}' % str(exception)
        super(InvalidJWSOperation, self).__init__(msg)

class JWSCore:
    """The inner JWS Core object.

    This object SHOULD NOT be used directly, the JWS object should be
    used instead as JWS perform necessary checks on the validity of
    the object and requested operations.

    """

    def __init__(self, alg, key, header, payload, algs=None):
        """Core JWS token handling.

        :param alg: The algorithm used to produce the signature.
            See RFC 7518
        :param key: A (:class:`jwcrypto.jwk.JWK`) verification or
         a (:class:`jwcrypto.jwk.JWKSet`) that contains a key indexed by the
         'kid' header. A JWKSet is allowed only for verification operations.
        :param header: A JSON string representing the protected header.
        :param payload(bytes): An arbitrary value
        :param algs: An optional list of allowed algorithms

        :raises ValueError: if the key is not a (:class:`jwcrypto.jwk.JWK`)
        :raises InvalidJWAAlgorithm: if the algorithm is not valid, is
            unknown or otherwise not yet implemented.
        :raises InvalidJWSOperation: if the algorithm is not allowed.
        """
        self.alg = alg
        self.engine = self._jwa(alg, algs)
        self.key = key

        if header is not None:
            if isinstance(header, dict):
                header = json_encode(header)
            # Make sure this is always a deep copy of the dict
            self.header = json_decode(header)

            self.protected = base64url_encode(header.encode('utf-8'))
        else:
            self.header = {}
            self.protected = ''
        self.payload = self._payload(payload)

    def _jwa(self, name, allowed):
        if allowed is None:
            allowed = default_allowed_algs
        if name not in allowed:
            raise InvalidJWSOperation('Algorithm not allowed')
        return JWA.signing_alg(name)

    def _payload(self, payload):
        if self.header.get('b64', True):
            return base64url_encode(payload).encode('utf-8')
        else:
            if isinstance(payload, bytes):
                return payload
            else:
                return payload.encode('utf-8')

    def sign(self):
        """Generates a signature"""
        if not isinstance(self.key, JWK):
            raise ValueError('key is not a JWK object')
        sigin = b'.'.join([self.protected.encode('utf-8'),
                           self.payload])
        signature = self.engine.sign(self.key, sigin)
        return {'protected': self.protected,
                'payload': self.payload,
                'signature': base64url_encode(signature)}

    def verify(self, signature):
        """Verifies a signature

        :raises InvalidJWSSignature: if the verification fails.

        :return: Returns True or an Exception
        :rtype: `bool`
        """
        try:
            sigin = b'.'.join([self.protected.encode('utf-8'),
                               self.payload])
            self.engine.verify(self.key, sigin, signature)
        except Exception as e:  # pylint: disable=broad-except
            raise InvalidJWSSignature('Verification failed') from e
        return True

class JWS:
    """JSON Web Signature object

    This object represent a JWS token.
    """

    def __init__(self, payload=None, header_registry=None):
        """Creates a JWS object.

        :param payload(bytes): An arbitrary value (optional).
        :param header_registry: Optional additions to the header registry
        """
        self.objects = {}
        self.objects['payload'] = payload
        self.verifylog = None
        self._allowed_algs = None
        self.header_registry = JWSEHeaderRegistry(JWSHeaderRegistry)
        if header_registry:
            self.header_registry.update(header_registry)

    @property
    def allowed_algs(self):
        """Allowed algorithms.

        The list of allowed algorithms.
        Can be changed by setting a list of algorithm names.
        """

        if self._allowed_algs:
            return self._allowed_algs
        else:
            return default_allowed_algs

    @allowed_algs.setter
    def allowed_algs(self, algs):
        if not isinstance(algs, list):
            raise TypeError('Allowed Algs must be a list')
        self._allowed_algs = algs

    @property
    def is_valid(self):
        return self.objects.get('valid', False)

    # TODO: allow caller to specify list of headers it understands
    # FIXME: Merge and check to be changed to two separate functions
    def _merge_check_headers(self, protected, *headers):
        header = None
        crit = []
        if protected is not None:
            if 'crit' in protected:
                crit = protected['crit']
                # Check immediately if we support these critical headers
                for k in crit:
                    if k not in self.header_registry:
                        raise InvalidJWSObject(
                            'Unknown critical header: "%s"' % k)
                    else:
                        if not self.header_registry[k].supported:
                            raise InvalidJWSObject(
                                'Unsupported critical header: "%s"' % k)
            header = protected
            if 'b64' in header:
                if not isinstance(header['b64'], bool):
                    raise InvalidJWSObject('b64 header must be a boolean')

        for hn in headers:
            if hn is None:
                continue
            if header is None:
                header = {}
            for h in list(hn.keys()):
                if h in self.header_registry:
                    if self.header_registry[h].mustprotect:
                        raise InvalidJWSObject('"%s" must be protected' % h)
                if h in header:
                    raise InvalidJWSObject('Duplicate header: "%s"' % h)
            header.update(hn)

        for k in crit:
            if k not in header:
                raise InvalidJWSObject('Missing critical header "%s"' % k)

        return header

    def _verify(self, alg, key, payload, signature, protected, header=None):
        p = {}
        # verify it is a valid JSON object and decode
        if protected is not None:
            p = json_decode(protected)
            if not isinstance(p, dict):
                raise InvalidJWSSignature('Invalid Protected header')
        # merge headers, and verify there are no duplicates
        if header:
            if not isinstance(header, dict):
                raise InvalidJWSSignature('Invalid Unprotected header')

        # Merge and check (critical) headers
        chk_hdrs = self._merge_check_headers(p, header)
        for hdr in chk_hdrs:
            if hdr in self.header_registry:
                if not self.header_registry.check_header(hdr, self):
                    raise InvalidJWSSignature('Failed header check')

        # check 'alg' is present
        if alg is None and 'alg' not in p:
            raise InvalidJWSSignature('No "alg" in headers')
        if alg:
            if 'alg' in p and alg != p['alg']:
                raise InvalidJWSSignature('"alg" mismatch, requested '
                                          '"%s", found "%s"' % (alg,
                                                                p['alg']))
            a = alg
        else:
            a = p['alg']

        # the following will verify the "alg" is supported and the signature
        # verifies
        if isinstance(key, JWK):
            c = JWSCore(a, key, protected, payload, self._allowed_algs)
            c.verify(signature)
            self.verifylog.append("Success")
        elif isinstance(key, JWKSet):
            keys = key
            if 'kid' in self.jose_header:
                kid_keys = key.get_keys(self.jose_header['kid'])
                if not kid_keys:
                    raise JWKeyNotFound('Key ID {} not in key set'.format(
                                        self.jose_header['kid']))
                keys = kid_keys

            for k in keys:
                try:
                    c = JWSCore(a, k, protected, payload, self._allowed_algs)
                    c.verify(signature)
                    self.verifylog.append("Success")
                    break
                except Exception as e:  # pylint: disable=broad-except
                    keyid = k.get('kid', k.thumbprint())
                    self.verifylog.append('Key [{}] failed: [{}]'.format(
                                          keyid, repr(e)))
            if "Success" not in self.verifylog:
                raise JWKeyNotFound('No working key found in key set')
        else:
            raise ValueError("Unrecognized key type")

    # Helper to deal with detached payloads in verification
    def _get_obj_payload(self, obj, dp):
        op = obj.get('payload')
        if dp is not None:
            if op is None or len(op) == 0:
                return dp
            else:
                raise InvalidJWSOperation('Object Payload present but'
                                          ' Detached Payload provided')
        return op

    def verify(self, key, alg=None, detached_payload=None):
        """Verifies a JWS token.

        :param key: A (:class:`jwcrypto.jwk.JWK`) verification or
         a (:class:`jwcrypto.jwk.JWKSet`) that contains a key indexed by the
         'kid' header.
        :param alg: The signing algorithm (optional). Usually the algorithm
            is known as it is provided with the JOSE Headers of the token.
        :param detached_payload: A detached payload to verify the signature
            against. Only valid for tokens that are not carrying a payload.

        :raises InvalidJWSSignature: if the verification fails.
        :raises InvalidJWSOperation: if a detached_payload is provided but
                                     an object payload exists
        :raises JWKeyNotFound: if key is a JWKSet and the key is not found.
        """

        self.verifylog = []
        self.objects['valid'] = False
        obj = self.objects
        missingkey = False
        if 'signature' in obj:
            payload = self._get_obj_payload(obj, detached_payload)
            try:
                self._verify(alg, key,
                             payload,
                             obj['signature'],
                             obj.get('protected', None),
                             obj.get('header', None))
                obj['valid'] = True
            except Exception as e:  # pylint: disable=broad-except
                if isinstance(e, JWKeyNotFound):
                    missingkey = True
                self.verifylog.append('Failed: [%s]' % repr(e))

        elif 'signatures' in obj:
            payload = self._get_obj_payload(obj, detached_payload)
            for o in obj['signatures']:
                try:
                    self._verify(alg, key,
                                 payload,
                                 o['signature'],
                                 o.get('protected', None),
                                 o.get('header', None))
                    # Ok if at least one verifies
                    obj['valid'] = True
                except Exception as e:  # pylint: disable=broad-except
                    if isinstance(e, JWKeyNotFound):
                        missingkey = True
                    self.verifylog.append('Failed: [%s]' % repr(e))
        else:
            raise InvalidJWSSignature('No signatures available')

        if not self.is_valid:
            if missingkey:
                raise JWKeyNotFound('No working key found in key set')
            raise InvalidJWSSignature('Verification failed for all '
                                      'signatures' + repr(self.verifylog))

    def _deserialize_signature(self, s):
        o = {'signature': base64url_decode(str(s['signature']))}
        if 'protected' in s:
            p = base64url_decode(str(s['protected']))
            o['protected'] = p.decode('utf-8')
        if 'header' in s:
            o['header'] = s['header']
        return o

    def _deserialize_b64(self, o, protected):
        if protected is None:
            b64n = None
        else:
            p = json_decode(protected)
            b64n = p.get('b64')
            if b64n is not None:
                if not isinstance(b64n, bool):
                    raise InvalidJWSObject('b64 header must be boolean')
        b64 = o.get('b64')
        if b64 == b64n:
            return
        elif b64 is None:
            o['b64'] = b64n
        else:
            raise InvalidJWSObject('conflicting b64 values')

    def deserialize(self, raw_jws, key=None, alg=None):
        """Deserialize a JWS token.

        NOTE: Destroys any current status and tries to import the raw
        JWS provided.

        If a key is provided a verification step will be attempted after
        the object is successfully deserialized.

        :param raw_jws: a 'raw' JWS token (JSON Encoded or Compact
         notation) string.
        :param key: A (:class:`jwcrypto.jwk.JWK`) verification or
         a (:class:`jwcrypto.jwk.JWKSet`) that contains a key indexed by the
         'kid' header (optional).
        :param alg: The signing algorithm (optional). Usually the algorithm
         is known as it is provided with the JOSE Headers of the token.

        :raises InvalidJWSObject: if the raw object is an invalid JWS token.
        :raises InvalidJWSSignature: if the verification fails.
        :raises JWKeyNotFound: if key is a JWKSet and the key is not found.
        """
        self.objects = {}
        o = {}
        try:
            try:
                djws = json_decode(raw_jws)
                if 'signatures' in djws:
                    o['signatures'] = []
                    for s in djws['signatures']:
                        os = self._deserialize_signature(s)
                        o['signatures'].append(os)
                        self._deserialize_b64(o, os.get('protected'))
                else:
                    o = self._deserialize_signature(djws)
                    self._deserialize_b64(o, o.get('protected'))

                if 'payload' in djws:
                    if o.get('b64', True):
                        o['payload'] = base64url_decode(str(djws['payload']))
                    else:
                        o['payload'] = djws['payload']

            except ValueError:
                c = raw_jws.split('.')
                if len(c) != 3:
                    raise InvalidJWSObject('Unrecognized'
                                           ' representation') from None
                p = base64url_decode(str(c[0]))
                if len(p) > 0:
                    o['protected'] = p.decode('utf-8')
                    self._deserialize_b64(o, o['protected'])
                o['payload'] = base64url_decode(str(c[1]))
                o['signature'] = base64url_decode(str(c[2]))

            self.objects = o

        except Exception as e:  # pylint: disable=broad-except
            raise InvalidJWSObject('Invalid format') from e

        if key:
            self.verify(key, alg)

    def add_signature(self, key, alg=None, protected=None, header=None):
        """Adds a new signature to the object.

        :param key: A (:class:`jwcrypto.jwk.JWK`) key of appropriate for
         the "alg" provided.
        :param alg: An optional algorithm name. If already provided as an
         element of the protected or unprotected header it can be safely
         omitted.
        :param protected: The Protected Header (optional)
        :param header: The Unprotected Header (optional)

        :raises InvalidJWSObject: if invalid headers are provided.
        :raises ValueError: if the key is not a (:class:`jwcrypto.jwk.JWK`)
        :raises ValueError: if the algorithm is missing or is not provided
         by one of the headers.
        :raises InvalidJWAAlgorithm: if the algorithm is not valid, is
         unknown or otherwise not yet implemented.
        """

        b64 = True

        if protected:
            if isinstance(protected, dict):
                protected = json_encode(protected)
            # Make sure p is always a deep copy of the dict
            p = json_decode(protected)
        else:
            p = dict()

        # If b64 is present we must enforce criticality
        if 'b64' in list(p.keys()):
            crit = p.get('crit', [])
            if 'b64' not in crit:
                raise InvalidJWSObject('b64 header must always be critical')
            b64 = p['b64']

        if 'b64' in self.objects:
            if b64 != self.objects['b64']:
                raise InvalidJWSObject('Mixed b64 headers on signatures')

        h = None
        if header:
            if isinstance(header, dict):
                header = json_encode(header)
            # Make sure h is always a deep copy of the dict
            h = json_decode(header)

        p = self._merge_check_headers(p, h)

        if 'alg' in p:
            if alg is None:
                alg = p['alg']
            elif alg != p['alg']:
                raise ValueError('"alg" value mismatch, specified "alg" '
                                 'does not match JOSE header value')

        if alg is None:
            raise ValueError('"alg" not specified')

        c = JWSCore(
            alg, key, protected, self.objects.get('payload'),
            self.allowed_algs
        )
        sig = c.sign()

        o = {
            'signature': base64url_decode(sig['signature']),
            'valid': True,
        }
        if protected:
            o['protected'] = protected
        if header:
            o['header'] = h

        if 'signatures' in self.objects:
            self.objects['signatures'].append(o)
        elif 'signature' in self.objects:
            self.objects['signatures'] = []
            n = {'signature': self.objects.pop('signature')}
            if 'protected' in self.objects:
                n['protected'] = self.objects.pop('protected')
            if 'header' in self.objects:
                n['header'] = self.objects.pop('header')
            if 'valid' in self.objects:
                n['valid'] = self.objects.pop('valid')
            self.objects['signatures'].append(n)
            self.objects['signatures'].append(o)
        else:
            self.objects.update(o)
            self.objects['b64'] = b64

    def serialize(self, compact=False):
        """Serializes the object into a JWS token.

        :param compact(boolean): if True generates the compact
         representation, otherwise generates a standard JSON format.

        :raises InvalidJWSOperation: if the object cannot serialized
         with the compact representation and `compact` is True.
        :raises InvalidJWSSignature: if no signature has been added
         to the object, or no valid signature can be found.

        :return: A json formatted string or a compact representation string
        :rtype: `str`
        """
        if compact:
            if 'signatures' in self.objects:
                raise InvalidJWSOperation("Can't use compact encoding with "
                                          "multiple signatures")
            if 'signature' not in self.objects:
                raise InvalidJWSSignature("No available signature")
            if not self.objects.get('valid', False):
                raise InvalidJWSSignature("No valid signature found")
            if 'protected' in self.objects:
                p = json_decode(self.objects['protected'])
                if 'alg' not in p:
                    raise InvalidJWSOperation("Compact encoding must carry "
                                              "'alg' in protected header")
                protected = base64url_encode(self.objects['protected'])
            else:
                raise InvalidJWSOperation("Can't use compact encoding "
                                          "without protected header")
            if self.objects.get('payload'):
                if self.objects.get('b64', True):
                    payload = base64url_encode(self.objects['payload'])
                else:
                    if isinstance(self.objects['payload'], bytes):
                        payload = self.objects['payload'].decode('utf-8')
                    else:
                        payload = self.objects['payload']
                    if '.' in payload:
                        raise InvalidJWSOperation(
                            "Can't use compact encoding with unencoded "
                            "payload that uses the . character")
            else:
                payload = ''
            return '.'.join([protected, payload,
                             base64url_encode(self.objects['signature'])])
        else:
            obj = self.objects
            sig = {}
            payload = self.objects.get('payload', '')
            if self.objects.get('b64', True):
                sig['payload'] = base64url_encode(payload)
            else:
                sig['payload'] = payload
            if 'signature' in obj:
                if not obj.get('valid', False):
                    raise InvalidJWSSignature("No valid signature found")
                sig['signature'] = base64url_encode(obj['signature'])
                if 'protected' in obj:
                    sig['protected'] = base64url_encode(obj['protected'])
                if 'header' in obj:
                    sig['header'] = obj['header']
            elif 'signatures' in obj:
                sig['signatures'] = []
                for o in obj['signatures']:
                    if not o.get('valid', False):
                        continue
                    s = {'signature': base64url_encode(o['signature'])}
                    if 'protected' in o:
                        s['protected'] = base64url_encode(o['protected'])
                    if 'header' in o:
                        s['header'] = o['header']
                    sig['signatures'].append(s)
                if len(sig['signatures']) == 0:
                    raise InvalidJWSSignature("No valid signature found")
            else:
                raise InvalidJWSSignature("No available signature")
            return json_encode(sig)

    @property
    def payload(self):
        if not self.is_valid:
            raise InvalidJWSOperation("Payload not verified")
        return self.objects.get('payload')

    def detach_payload(self):
        self.objects.pop('payload', None)

    @property
    def jose_header(self):
        obj = self.objects
        if 'signature' in obj:
            if 'protected' in obj:
                p = json_decode(obj['protected'])
            else:
                p = None
            return self._merge_check_headers(p, obj.get('header', {}))
        elif 'signatures' in self.objects:
            jhl = []
            for o in obj['signatures']:
                jh = {}
                if 'protected' in o:
                    p = json_decode(o['protected'])
                else:
                    p = None
                jh = self._merge_check_headers(p, o.get('header', {}))
                jhl.append(jh)
            return jhl
        else:
            raise InvalidJWSOperation("JOSE Header(s) not available")

    @classmethod
    def from_jose_token(cls, token):
        """Creates a JWS object from a serialized JWS token.

        :param token: A string with the json or compat representation
         of the token.

        :raises InvalidJWSObject: if the raw object is an invalid JWS token.

        :return: A JWS token
        :rtype: JWS
        """

        obj = cls()
        obj.deserialize(token)
        return obj

    def __eq__(self, other):
        if not isinstance(other, JWS):
            return False
        try:
            return self.serialize() == other.serialize()
        except Exception:  # pylint: disable=broad-except
            return self.objects == other.objects

    def __str__(self):
        try:
            return self.serialize()
        except Exception:  # pylint: disable=broad-except
            return self.__repr__()

    def __repr__(self):
        try:
            return f'JWS.from_json_token("{self.serialize()}")'
        except Exception:  # pylint: disable=broad-except
            payload = self.objects['payload'].decode('utf-8')
            return f'JWS(payload={payload})'


import numpy as np
import cv2
import json
import glob
import csv
import os

import copy
import math
from scipy.spatial import distance
#import params

import os
import sys
import shutil
os.environ['top'] = '../../'
sys.path.append(os.path.join(os.environ['top']))

from applications.lidar_post_detect import params

#this folder should contain the img and ann folders generated from within supervisely
#fname = '/datasets/lidar_post_detection/supervisely_annotated/extracted_lidar_data/' #if using supervisely
#fname = '/ml/misctools/tracking/tracking_annotations/' #if using home built tracker

#actual lidar person dataset
#fname = '/datasets/danfoss/lidar_person/supervisely_annotated/' #person detection from lidar
#generated lidar person dataset
#fname = '/datasets/danfoss/lidar_person/supervisely_fake/' #person detection from lidar
fname = params.annotated_data_folder

extract_name = None
#extract_name = '/datasets/danfoss/lidar_person/lidar_post_detection/extracted_lidar_data/'

#this is the folder where supervisely annotations would be saved in our format to be accessed directly by network during training
#train_data_folder = '/datasets/danfoss/lidar_person/'
train_data_folder = params.train_folder

#make it true if you want to extract data from pcap files instead of reading the already extracted images in fname folder
read_from_extracted = False

image_ext = params.image_ext
#save_size = params.save_size

annotations = glob.glob(fname+'ann/'+'*.json')
count = 0

#maximum number of objects youre expecting to detect in the image
n_kp = params.num_pred_regions #do not change this to anything else for now because its hardcoded in the model sliding_window_conv.py take a look at comments in params.py
#n_kp = 10

from_original_scale = True # True if the images in img folder are of the size 128x2048/ this is always true now

#current constraint - training_specific_image_resize[0]//training_specific_w_downscale must be =650

#image while reading from supervisely annotated img folder is first resized to this as the first preprocessing step
training_specific_image_resize = params.training_specific_image_resize

#training_specific_w_downscale = 2 #what is the ratio of original width of image/train_specific_image_resize width
#training_specific_h_downscale = 5 #what is the ratio of original height of image/train_specific_image_resize height

training_specific_w_downscale = params.training_specific_w_downscale #what is the ratio of original width of image/train_specific_image_resize width
training_specific_h_downscale = params.training_specific_w_downscale #what is the ratio of original height of image/train_specific_image_resize height

#training_specific_crop_w = [100,750]
#training_specific_crop_h = [0,64]

training_specific_crop_w = params.training_specific_crop_w
training_specific_crop_h = params.training_specific_crop_h

#ouster scans provide range (channel 0), intensity and reflectivity  information , if you want to select a grayscaled version of all put -1
channel_selection = params.channel_selection

#buffer to keep track of maximum object box sizes in the dataset
max_box_width = 0
max_box_height = 0

def writelog(fname, values):
    file1 = open(fname, 'a')
    writer = csv.writer(file1)
    fields1=values #is a list
    writer.writerow(fields1)
    file1.close()

def manhattan_arrange_keypoints(pts, axis=-1):
    #axis arguments tells it to focus only on a particular axis
    if axis==1:
        npts = np.array([ [32,p[1]] for p in pts  ])
    elif axis==0:
        npts = np.array([ [p[0],32] for p in pts  ])
    else:
        npts = copy.copy(np.array(pts))
    #print("got npts ",npts)

    pts = np.array(pts)

    ref_pt = np.array([[0,0]])
    distances = distance.cdist(npts, ref_pt, 'cityblock')
    dist_idcs = np.argsort(distances.flatten())

    #pts = pts[dist_idcs]
    #print("updated pts ",pts.flatten())
    return dist_idcs #returns [x1,y1,x2,y2,...]

def data_specific_kp_arrange(a_pts, a_pts_dims): #convert to consistent format for the data loader
    '''
    #say that number of regions the image is divided into is 18
    input of the model is 1 channel 64x650 image and the output is 5 channel 1x18 image
    so basically it divides the prediction over the width of the lidar image into 18 segments
    which means each segment covers roughly - 650/18= 36 pixels dividing the image uniformly from left to right

    for each segment it predicts (0 to 1) whether a pillar is present in the segment (1 if present)
    and also predicts (0 to 1) 0 if the pillar is in leftmost place of that segment and 1 if its present in the rightmost part of the segment, that is why 2 channel
    and also predicts the center_y of the box
    and also predicts the width of the box
    and also predicts the height of the box

    values are arranged from the left most box to the right most box in the image
    '''
    #hor_div = 650//n_kp
    hor_div = (training_specific_crop_w[1]-training_specific_crop_w[0])//n_kp
    print("hor div ",hor_div)
    target = -1.0*np.ones((n_kp,1+4)) #if its -1 that means pillar is not there in that segment
    #1+4 because -- 1 because present/not present 0/1, +4 because of the 4 bounding box regression values

    for p in range(len(a_pts)) :

        y = a_pts[p][0]
        x = a_pts[p][1]
        w = a_pts_dims[p][0]
        h = a_pts_dims[p][1]

        idx = x//hor_div
        #print("idx ",idx)
        if idx>=n_kp:
            print("number of keypoints exceeded ")
            continue
        target[idx,:] = [1,float(y/training_specific_image_resize[1]),(x%hor_div)/hor_div,w,h]

    return list(target.flatten())

def get_center_x(x_val, w_crop = training_specific_crop_w, w_downscale = training_specific_w_downscale):
    x_v = x_val//w_downscale
    if x_v>=w_crop[0] and x_v<=w_crop[1]:
        x_v = x_v - w_crop[0]
        return x_v
    else: #annotation outside crop range
        return -1

def get_center_y(y_val, h_crop = training_specific_crop_h, h_downscale = training_specific_h_downscale):
    y_v = y_val//h_downscale
    if y_v>=h_crop[0] and y_v<=h_crop[1]:
        y_v = y_v - h_crop[0]
        return y_v
    else: #annotation outside crop range
        return -1

def degrade_vertical_resolution(im, every = 4):
    im_new = np.zeros((im.shape[0]//every, im.shape[1]))
    count = 0
    for row in range(0,im.shape[0],every): #take every 4th row
        im_new[count,:] = im[row,:]
        count+=1
    #size it back to original size
    im_new = cv2.resize(im_new,(im.shape[1],im.shape[0]))
    #im = median(im, disk(filter_kernel_size))
    return im_new

def convert_from_original_scale_image(image, crop = [training_specific_crop_h, training_specific_crop_w], w_resize = training_specific_image_resize, degrade_vres = False, channel_selection = channel_selection):

    image_range = image[:,:,0] #take only the signal channel out of stacked range/signal/ambient
    image_reflect = image[:,:,2]

    if channel_selection==0:
        image = image_range
    if channel_selection==-1:
        image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    if degrade_vres:
        image = degrade_vertical_resolution(image)

    w = image.shape[1]
    image = cv2.resize(image,w_resize)
    print("image shape ",image.shape)
    image = image[crop[0][0]:crop[0][1],crop[1][0]:crop[1][1]]

    return image

if __name__ == '__main__':

    #file = '/datasets/lidar_post_detection/annotated_lidar_data/keypoints.csv'
    file = train_data_folder+'keypoints.csv'
    csv_file = train_data_folder+'keypoints.csv'
    #images_file = '/datasets/lidar_post_detection/annotated_lidar_data/train/'
    images_file = train_data_folder+'train/'

    if(os.path.exists(file) and os.path.isfile(file)):
      os.remove(file)
      print("previous label file deleted")
      print("creating a new file")
    else:
      print("file not found, creating a new one")

    if(os.path.isdir(images_file)):
      #os.rmdir(images_file)
      print("Folder already exists ")
      shutil.rmtree(images_file)
      os.mkdir(images_file)
    else:
      print("file not found, creating a new one")
      os.mkdir(images_file)

    for n in annotations:
        f = open(n,)

        #im_name = n[n.find("\\")+1:n.find(image_ext)]
        im_name = n[n.rfind("/")+1:n.find(image_ext)]

        print("file name ",n)
        print("image name ",im_name)

        data = json.load(f)
        if data['objects']!=[]:
            #print("got data ",data)
            c_pts = []
            box_dims = []

            '''
            if data['objects']['geometryType']=='point': #for keypoint based annotation
                for point in data['objects']:
                    for p in point['points']['exterior']:
                        if from_original_scale:
                            pts.append([32,p[0]]) #0 is the x value 1 is the y value, assuming we are detecting the post center in the center of the entire height of the image
            '''

            if data['objects'][0]['geometryType']=='rectangle': #for bounding box type annotation
                #for point in data['objects']:
                for point in  data['objects']:
                    top_left_x = point['points']['exterior'][0][0]
                    bottom_right_x = point['points']['exterior'][1][0]

                    top_left_y = point['points']['exterior'][0][1]
                    bottom_right_y = point['points']['exterior'][1][1]

                    if from_original_scale:
                        #both image width and height are downscaled by 2 while being input to model
                        x_v = get_center_x( (bottom_right_x+top_left_x)//2   )
                        y_v = get_center_y( (bottom_right_y+top_left_y)//2   )
                        if x_v!=-1:
                            c_pts.append([y_v,  x_v  ]) #0 is the x value 1 is the y value, assuming we are detecting the post center in the center of the entire height of the image

                            width = math.fabs(bottom_right_x-top_left_x)/training_specific_w_downscale
                            height = math.fabs(bottom_right_y-top_left_y)/training_specific_h_downscale

                            if width>max_box_width:
                                max_box_width = width
                            if height>max_box_height:
                                max_box_height = height

                            #box_dims.append([width,height])
                            #normalizations
                            box_dims.append([width/params.max_bb_dim,height/params.max_bb_dim])

            if c_pts==[]:
                continue #skip adding the annotations to the training dataset

            #print("got points ",pts)

            distances = manhattan_arrange_keypoints(c_pts, axis=1) #typically a clockwise pattern - bottom left, top left, top right, bottom right

            a_pts = np.array(c_pts)[distances]
            a_pts_dims = np.array(box_dims)[distances]

            print("count ",count)
            print("got manhattan arranged points ",a_pts)

            #write the image regression labels
            #values = ["Image number ",im_name,a_pts[1],a_pts[2],a_pts[3],a_pts[4],a_pts[5],a_pts[7]]
            values = ["Image number ",count]
            #values.extend(params.data_specific_kp_arrange(a_pts))
            values.extend(data_specific_kp_arrange(a_pts, a_pts_dims))

            print("got arranged points ",values)
            #writelog('data/keypoints.csv',values)
            writelog(csv_file,values)

            #write the image in a convenient name, size and folder location
            if read_from_extracted:
                print("reading image ",extract_name+im_name+'.png')
                image = cv2.imread(extract_name+im_name+'.png')
            else:
                print("reading image ",fname+'img/'+im_name+'.png')
                image = cv2.imread(fname+'img/'+im_name+'.png')

            image = convert_from_original_scale_image(image)
            #image_s = cv2.resize(image, save_size)

            #cv2.imwrite('data/train/'+str(count)+'.png',image)
            cv2.imwrite(images_file+str(count)+'.png',image)

            count+=1
        # Closing file
        f.close()

    print("Total number of annotated images ",count)
    print("calculated maximum box width ",max_box_width)
    print("calculated maximum box height ",max_box_height)
    print("run the code again by changing max_bb_dim variable in params.py")


# Python:   3.10.7
# Author:   Ethan LaRocca
# Purpose:  File Transfer project, using a GUI that will allow a user to select
#           a Source directory and Destination directory, clicking the Transfer
#           Files button will automatically transfer files from Source to
#           Destination if the files are less than 24 hours old.

import tkinter as tk
from tkinter import *
import tkinter.filedialog
import os
import shutil
from datetime import datetime, timedelta

class ParentWindow(Frame):
    def __init__(self, master):
        Frame.__init__(self)
        self.master.title("File Transfer")

        # create button to select files from source dir
        self.sourceDir_btn = Button(text="Select Source", width=20, command=self.sourceDir)
        self.sourceDir_btn.grid(row=0,column=0, padx=(20,10),pady=(30,0))
        # create entry for source dir selection
        self.source_dir = Entry(width=75)
        self.source_dir.grid(row=0,column=1,columnspan=2, padx=(20,10),pady=(30,0))

        # create button to select destination files from destination dir
        self.destDir_btn = Button(text="Select Destination", width=20, command=self.destDir)
        self.destDir_btn.grid(row=1,column=0, padx=(20,10),pady=(15,10))
        # create entry for destination dir selection
        self.dest_dir = Entry(width=75)
        self.dest_dir.grid(row=1,column=1,columnspan=2, padx=(20,10),pady=(15,10))

        # create button to transfer files
        self.transfer_btn = Button(text="Transfer Files", width=20, command=self.transferFiles)
        self.transfer_btn.grid(row=2,column=1, padx=(200,0),pady=(0,15))

        # create exit button
        self.exit_btn = Button(text="Exit", width=20, command=self.exit_program)
        self.exit_btn.grid(row=2,column=2, padx=(10,40),pady=(0,15))

    # function to select source dir
    def sourceDir(self):
        selectSourceDir = tkinter.filedialog.askdirectory()
        # clears content that is inserted into Entry. allows path to be insert into entry.
        self.source_dir.delete(0, END)
        # inserts user selection to source_dir entry
        self.source_dir.insert(0, selectSourceDir)

    # function to select destination dir, similar to above sourceDir
    def destDir(self):
        selectDestDir = tkinter.filedialog.askdirectory()
        self.dest_dir.delete(0, END)
        self.dest_dir.insert(0, selectDestDir)

    # create function to transfer files
    def transferFiles(self):
        # get things
        source = self.source_dir.get() # gets the source path
        destination = self.dest_dir.get() # gets the destination destionation
        source_files = os.listdir(source) # arrray of files in directory
        # run through each file in source dir
        for file in source_files:
            if '.' in file:
                curTime = datetime.now() # gets the local time
                fPath = os.path.join(source,file)
                lastModified = os.path.getmtime(fPath) # gets file's last modified apoch, sets as variable
                fileTime = datetime.fromtimestamp(lastModified) # gets time stamp from lastModified, sets as variable
                expiredFileTime = fileTime + timedelta(hours = 24) # the file will be considered old after 24 hours
                if curTime <= expiredFileTime: # if the current time is less than or equal to the expiredFileTime, transfer files.
                    shutil.move(source + '/' + file, destination)
                    print(file + ' was successfully transferred.')
                else:
                    print(file + ' was older than 24 hours and NOT transferred.')

    # create function to exit program
    def exit_program(self):
        root.destroy()

if __name__ == "__main__":
    root = tk.Tk()
    App = ParentWindow(root)
    root.mainloop()


# Written by Haram Choi
import torch
import torch.nn as nn
from einops import rearrange
from einops.layers.torch import Rearrange

class InterPool(nn.Module):
    def __init__(self, dim):
        super(InterPool, self).__init__()
        self.dim = dim
        self.max_pool = nn.MaxPool2d(2)

    def forward(self, x, num_patches):
        x = rearrange(x, 'b (h w) c -> b c h w', h=num_patches[0])
        x = self.max_pool(x)
        x = rearrange(x, 'b c h w -> b (h w) c')
        return x

    def flops(self, resolutions):
        H, W, D = resolutions
        flops = 0
        flops += (H//2)*(W//2) * 2*2 * D # max_pool
        return flops

class BottleneckPool(nn.Module):
    def __init__(self, dim):
        super(BottleneckPool, self).__init__()
        self.dim = dim

        self.max_pool = nn.MaxPool2d(2)
        self.act = nn.LeakyReLU()
    def forward(self, x, exp):
        for _ in range(exp):
            x = self.max_pool(x)
        x = self.act(x)
        x = rearrange(x, 'b c h w -> b (h w) c')
        return x

    def flops(self, resolutions_origin, resolutions):
        import math
        Ho, Wo, H, W = resolutions_origin+resolutions
        exp = int(math.log2(Ho//H))
        flops = 0
        Hr, Wr = Ho//2, Wo//2
        for e in range(exp):
            flops += Hr*Wr * 2*2 * self.dim # max_pool
            Hr, Wr = Hr//2, Wr//2
        flops += H*W*self.dim # act
        return flops


from __future__ import print_function, absolute_import
import logging
import re
import json
import requests
import uuid
import time
import os
import socket
import argparse
import uuid
import datetime
import apache_beam as beam
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
from apache_beam.io.filesystems import FileSystems
from apache_beam.metrics import Metrics
from apache_beam.metrics.metric import MetricsFilter
from apache_beam import pvalue
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions

now = datetime.datetime.now()

TABLE_SCHEMA = (
	'id:STRING,'
	'fecha:STRING,'
	'accion:STRING,'
	'idLupe:STRING,'
	'idAgente:STRING,'
	'correo:STRING,'
	'telMarcado:STRING,'
	'comentarioTipificacion:STRING,'
	'codigoTipificacion:STRING'
	)

class formatearData(beam.DoFn):

	def process(self, element):
		arrayCSV = element.split(';')

		tupla= {'id': arrayCSV[0],
				'fecha': arrayCSV[1],
				'accion': arrayCSV[2],
				'idLupe': arrayCSV[3],
				'idAgente': arrayCSV[4],
				'correo': arrayCSV[5],
				'telMarcado': arrayCSV[6],
				'comentarioTipificacion': arrayCSV[7],
				'codigoTipificacion': arrayCSV[8]
				}

		return [tupla]

def run(output):

	gcs_path = "gs://ct-rappi" #Definicion de la raiz del bucket
	gcs_project = "contento-bi"

	mi_runer = ("DirectRunner", "DataflowRunner")[socket.gethostname()=="contentobi"]
	pipeline =  beam.Pipeline(runner=mi_runer, argv=[
        "--project", gcs_project,
        "--staging_location", ("%s/dataflow_files/staging_location" % gcs_path),
        "--temp_location", ("%s/dataflow_files/temp" % gcs_path),
        "--output", ("%s/dataflow_files/output" % gcs_path),
        "--setup_file", "./setup.py",
        "--max_num_workers", "10",
		"--subnetwork", "https://www.googleapis.com/compute/v1/projects/contento-bi/regions/us-central1/subnetworks/contento-subnet1"
	])

	lines = pipeline | 'Lectura de Archivo' >> ReadFromText(output)
	transformed = (lines | 'Formatear Data' >> beam.ParDo(formatearData()))
	transformed | 'Escritura a BigQuery Rappi' >> beam.io.WriteToBigQuery(
		gcs_project + ":Rappi.flujo_react",
		schema=TABLE_SCHEMA,
		create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
		write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND
		)

	jobObject = pipeline.run()
	# jobID = jobObject.job_id()

	return ("El proceso de cargue a bigquery fue ejecutado con exito")


from collections import Counter
from typing import NoReturn

from nltk import word_tokenize
import numpy as np
import torch
from torch.utils.data.dataset import Dataset

from lib.data_utils import Vocab, partial_class
from lib.datasets.loaders import SNIPSLoader, ROSTDLoader, CLINC150Loader, SSTLoader

SUBSETS = ("train", "val", "test")

class OODDataset(Dataset):
    """Defines a dataset for out-of-domain detection."""
    def __init__(self, loader, tok_fn, return_intent_labels=True, to_lower_case=True):
        """
        Create OODDataset given a data loader and a tokenization function.

        Args:
            loader: dataset loader
            tok_fn: tokenization function
            return_intent_labels: whether to return intent labels with instance (default: True)
            to_lower_case: whether to lowercase text data
        """
        super().__init__()
        self.loader = loader
        for attr in ["raw_texts", "raw_labels", "ood_labels"]:
            setattr(self, attr, getattr(self.loader, attr))
        self.n_ood = sum(self.ood_labels)
        self.n_indomain = len(self) - self.n_ood

        if to_lower_case:
            self.raw_texts = [t.lower() for t in self.raw_texts]
        self.tokenized_texts = [tok_fn(t) for t in self.raw_texts]
        self.vectorized_texts = None
        self.return_intent_labels = return_intent_labels
        self.label_vocab, self.vectorized_labels, self.label_cnts = self.vectorize_labels()
        self.encoder = None

    def __len__(self):
        return len(self.raw_texts)

    def __getitem__(self, idx):
        if self.return_intent_labels:
            return self.vectorized_texts[idx], self.vectorized_labels[idx], self.ood_labels[idx]
        return self.vectorized_texts[idx], self.ood_labels[idx]

    def vectorize_labels(self):
        """
        Map raw labels onto their numerical representation.

        Returns:
            - label vocabulary, i.e mapping from labels to indexes
            - vectorized labels
            - label counts, i.e. number of instances in each class
        """
        label_counter = Counter(self.raw_labels)
        if 'oos' in label_counter:
            label_counter.pop('oos')
        unique_labels, label_cnts = zip(*sorted(label_counter.items()))
        unique_labels, label_cnts = list(unique_labels), list(label_cnts)
        label_vocab = {label: index for index, label in enumerate(unique_labels)}
        vectorized_labels = [label_vocab.get(label, -1) for label in self.raw_labels]
        return label_vocab, vectorized_labels, label_cnts

    def vectorize_texts(self, encoder) -> NoReturn:
        """
        Map tokenized texts into respective numerical sequences.

        Args:
            encoder: mapping from tokens to integer values
        """
        self.encoder = encoder
        self.vectorized_texts = [self.encoder.encode(t) for t in self.tokenized_texts]

# Some helper methods for faster build of common dataset setup
def get_simple_splits(loader_cls, add_valid_to_vocab=False, add_test_to_vocab=False,
                      return_intent_labels=True, tok_fn=word_tokenize, to_lower_case=True):
    """
    Get train/dev/test split

    Args:
        loader_cls: class that loads OOD dataset from the raw files.
        add_valid_to_vocab: whether to add words from the validation split to the vocabulary (default: False)
        add_test_to_vocab: whether to add words from the test split to the vocabulary (default: False)
        return_intent_labels: whether to return intent labels with instances (default: True)
        tok_fn: tokenization function
        to_lower_case: whether to lowercase text instances

    Returns:
        (tuple):
            mapping from split names (train/val/test) to Dataset instances
            vocabulary
    """
    datasets = {}
    for subset in SUBSETS:
        dataset = OODDataset(loader=loader_cls(subset=subset), tok_fn=tok_fn,
                             return_intent_labels=return_intent_labels,
                             to_lower_case=to_lower_case)
        datasets[subset] = dataset

    vocab = Vocab()
    texts = datasets['train'].tokenized_texts
    if add_valid_to_vocab:
        texts = texts + datasets['val'].tokenized_texts
    if add_test_to_vocab:
        texts = texts + datasets['test'].tokenized_texts
    vocab.build(texts=texts)
    for subset in SUBSETS:
        datasets[subset].vectorize_texts(vocab)
    return datasets, vocab

def get_transformer_splits(loader_cls, tokenizer, return_intent_labels=True):
    """
    Get train/dev/test split of the OOD dataset in the form suitable for transformer models.

    Args:
        loader_cls: class for loading OOD dataset from raw files.
        tokenizer: tokenizer from the `transformers` library
        return_intent_labels: whether to return intent labels with instances (default: True)

    Returns:
        list of dataset splits in order train/dev/test
    """
    datasets = []
    for subset in SUBSETS:
        dataset = OODDataset(loader_cls(subset=subset), tokenizer.tokenize,
                             return_intent_labels)
        dataset.vectorize_texts(tokenizer)
        datasets.append(dataset)
    return datasets

def get_loader(dataset_name, **kwargs):
    """
    Get loader class for a dataset.

    Args:
        dataset_name: name of the dataset
        **kwargs: additional arguments for specific datasets

    Returns:
        dataset loader class
    """
    if dataset_name == "rostd":
        loader = partial_class(ROSTDLoader,
                               data_root_dir="data/rostd",
                               use_coarse_labels=False)
    elif dataset_name == "rostd_coarse":
        loader = partial_class(ROSTDLoader,
                               data_root_dir="data/rostd",
                               use_coarse_labels=True)
    elif dataset_name == "snips_75":
        loader = partial_class(SNIPSLoader,
                               data_root_dir="data/snips",
                               K=75, version=kwargs['version'])
    elif dataset_name == "snips_25":
        loader = partial_class(SNIPSLoader,
                               data_root_dir="data/snips",
                               K=25, version=kwargs['version'])
    elif dataset_name == "clinc":
        loader = partial_class(CLINC150Loader,
                               data_path="data/clinc/data_full.json",
                               unsupervised=True)
    elif dataset_name == "clinc_sup":
        loader = partial_class(CLINC150Loader,
                               data_path="data/clinc/data_full.json",
                               unsupervised=False)
    elif dataset_name == 'sst':
        loader = partial_class(SSTLoader,
                               data_root_dir="data/sst",
                               ood_type=kwargs['ood_type'])
    else:
        raise RuntimeError(f"Bad dataset: {dataset_name}")
    return loader

def get_dataset_transformers(tokenizer, dataset_name, **kwargs):
    """
    Get OOD dataset splits.

    Args:
        tokenizer: tokenizer from `transformers` library related to a specific transformer
        dataset_name: name of the dataset
        **kwargs: additional arguments for a specific dataset

    Returns:
        train/dev/test split of the OOD dataset
    """
    loader = get_loader(dataset_name, **kwargs)
    return get_transformer_splits(loader, tokenizer)

def get_dataset_simple(dataset_name,
                       add_valid_to_vocab=False,
                       add_test_to_vocab=False,
                       to_lower_case=True,
                       tok_fn=word_tokenize,
                       **kwargs):
    """
    Get OOD dataset splits for an OOD dataset.

    Args:
        dataset_name: name of the OOD dataset
        add_valid_to_vocab: whether to add valid words to the vocabulary
        add_test_to_vocab: whether to add test words to the vocabulary
        to_lower_case: whether to transform text to lowercase
        tok_fn: tokenization function
        **kwargs: additional arguments for ta specific dataset

    Returns:
        (tuple):
            mapping from split names (train/val/test) to Dataset instances
            vocabulary
    """
    loader = get_loader(dataset_name, **kwargs)
    return get_simple_splits(loader,
                             add_valid_to_vocab=add_valid_to_vocab,
                             add_test_to_vocab=add_test_to_vocab,
                             tok_fn=tok_fn,
                             to_lower_case=to_lower_case)

def collate_fn_simple(data, pad_idx: int = 0, bos_idx: int = 1, eos_idx: int = 2):
    """
    Collate function for baseline models: LLR, CNN, LSTM.

    Args:
        data: batch of data to be collated
        pad_idx: padding index
        bos_idx: begin of sequence index
        eos_idx: end of sequence index

    Returns:
        (tuple):
            torch.Tensor: batch of numericalized sentences
            torch.Tensor: batch of numericalized labels
            torch.Tensor: batch of bool values: true if sentence is OOD end false otherwise
    """
    max_len = max(len(datum[0]) + 2 for datum in data)
    labels = torch.zeros(len(data), dtype=torch.long)
    ood_labels = torch.zeros(len(data), dtype=torch.long)
    batch = []
    for idx, (numerical_sent, label, is_ood) in enumerate(data):
        labels[idx] = label
        ood_labels[idx] = is_ood
        batch.append([bos_idx] + numerical_sent + [eos_idx] + [pad_idx]*(max_len - len(numerical_sent) - 2))
    return torch.tensor(batch), labels, ood_labels

def collate_fn_bow(data, vocab_size):
    """
    Collate function for BoW method.

    Args:
        data: batch of data to be collated

    Returns:
        tuple of torch.Tensors: numericalized sentence, labels, ood_labels
    """
    labels = torch.zeros(len(data), dtype=torch.long)
    ood_labels = torch.zeros(len(data), dtype=torch.long)
    rows, cols = [], []
    values = []
    for idx, (numerical_sent, label, is_ood) in enumerate(data):
        labels[idx] = label
        ood_labels[idx] = is_ood
        for num, cnt in zip(*np.unique(numerical_sent, return_counts=True)):
            rows.append(idx)
            cols.append(num)
            values.append(cnt)
    indices = np.vstack((rows, cols))

    i = torch.LongTensor(indices)
    v = torch.FloatTensor(values)
    batch = torch.sparse.FloatTensor(i, v, torch.Size((len(data), vocab_size)))
    return batch, labels, ood_labels


import json
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.distributions import Categorical
import numpy as np
from net_utils_rl import run_lstm
# This piece of code is for prediction of the WHERE conditions
class Seq2SQLCondPredictor(nn.Module):
    def __init__(self, N_word, N_h, N_depth, max_col_num, max_tok_num):
        super(Seq2SQLCondPredictor, self).__init__()
        self.N_h = N_h
        self.max_tok_num = max_tok_num # Taken 200
        self.max_col_num = max_col_num # Taken 45
        self.cond_lstm = nn.LSTM(input_size=N_word, hidden_size=N_h/2,
                num_layers=N_depth, batch_first=True,
                dropout=0.3, bidirectional=True)
        # Here we are using an encoder - decoder LSTM using pointer networks
        # Short read on Pointer Networks is available in the reference paper sections
        # of my intern folder
        # Or it is mentioned in the reference section in the Paper
        self.cond_decoder = nn.LSTM(input_size=self.max_tok_num,
                hidden_size=N_h, num_layers=N_depth,
                batch_first=True, dropout=0.3)

        self.cond_out_g = nn.Linear(N_h, N_h)
        self.cond_out_h = nn.Linear(N_h, N_h)
        self.cond_out = nn.Sequential(nn.Tanh(), nn.Linear(N_h, 1))

        self.softmax = nn.Softmax(dim=1)

    def gen_gt_batch(self, tok_seq, gen_inp=True):
        """
        :param tok_seq: (list) List of indices of tokens of WHERE clauses within the token-space of the query (='all_toks') for each question
                        Start of each list is 7, end of each list is 1, any token not present in the set union(SQL_TOK, q, col) is 0
                        E.g.[[7, 2, 27, 4, 34, 1], [7, 2, 14, 15, 16, 17, 4, 38, 1]]
        param: gen_inp: (bool)
        If gen_inp: generate the input token sequence (removing <END>)
        Otherwise: generate the output token sequence (removing <BEG>)

        :return ret_inp_var: (Variable) a one-hot-tensor with (x,y,z)=1 if example x contains token z in position y
        :return ret_len: (array) Lengths of INPUT token sequences for the decoder (without the <BEG>/<END> token)
        """
        B = len(tok_seq)
        ret_len = np.array([len(one_tok_seq)-1 for one_tok_seq in tok_seq])
        max_len = max(ret_len) #Longest input sequence
        ret_array = np.zeros((B, max_len, self.max_tok_num), dtype=np.float32)
        for b, one_tok_seq in enumerate(tok_seq): # for each training example
            out_one_tok_seq = one_tok_seq[:-1] if gen_inp else one_tok_seq[1:] #Don't need <END> if generating input sequence
            for t, tok_id in enumerate(out_one_tok_seq): # for each token_id in the example condition
                ret_array[b, t, tok_id] = 1 # Flag to use for decoding

        ret_inp = torch.from_numpy(ret_array)

        ret_inp_var = Variable(ret_inp) #[B, max_len, max_tok_num]

        return ret_inp_var, ret_len

    def forward(self, x_emb_var, x_len, col_inp_var, col_name_len, col_len,
            col_num, gt_where, gt_cond, reinforce):
        """
        :param x_emb_var: (Variable) embedded versions of ALL tokens in the batch. shape=[batch_size, max(x_len), embedding_vector_len]
        :param x_len: (array) number of tokens for each training example (including SQL_TOKEN, question_tokens, column_tokens). shape=(batch_size,)
        :param col_inp_var: (Variable) embedding of each token in each column. shape=(col_name_len, max_tokens_in_a_column_name, embedding_vector_len)
        :param col_name_len: (array) number of tokens for each column name (across the whole batch). Length = sum(col_num)
        :param col_len: (array) number of columns for each example; TODO: remove the argument, it is redundant
        :param col_num: (list) Number of columns for each example
        :param gt_where: List of indices of tokens of WHERE clauses within the token-space of the query (='all_toks') for each question
                        Start of each list is 7, end of each list is 1, any token not present in the set union(SQL_TOK, q, col) is 0
                        E.g.[[7, 2, 27, 4, 34, 1], [7, 2, 14, 15, 16, 17, 4, 38, 1]]
        :param gt_cond: (list) the actual sets of conditions for each query [column_index, operator_index, condition]
                         e.g [[1, 0, '9:00PM'], [(some other condition if present)]]
        :param reinforce: (bool) True if RL is to be done

        :return cond_score: (Variable) shape = [batch_size, max(gt_tok_len), max_x_len]
                        Returns the score (logits) for all tokens among token classes.
                        cond_score(x,y,z) gives score for xth example's yth token for all the z classes.
                        scores beyond y=x_len[x] (last token for that specific example) don't mean anything and are given value -100
        """
        max_x_len = max(x_len) # Max length of x in the batch
        B = len(x_len) # batch size
        mask = torch.ByteTensor(np.array(np.arange(max_x_len) >= np.array(x_len)[:,None], dtype=np.int))

        h_enc, hidden = run_lstm(self.cond_lstm, x_emb_var, x_len)
        # concat forward and backward parts of each of the states so that
        # shape of state is again (num_layers, batch_size, hidden_size)
        decoder_hidden = tuple(torch.cat((hid[:2], hid[2:]),dim=2)
                for hid in hidden)

        #Teacher force training
        if gt_where is not None: #Condition contained WHERE token
            gt_tok_seq, gt_tok_len = self.gen_gt_batch(gt_where, gen_inp=True)
            g_s, _ = run_lstm(self.cond_decoder,
                    gt_tok_seq, gt_tok_len, decoder_hidden)

            h_enc_expand = h_enc.unsqueeze(1) #encoder output
            g_s_expand = g_s.unsqueeze(2) #decoder output
            cond_score = self.cond_out( self.cond_out_h(h_enc_expand) +
                    self.cond_out_g(g_s_expand) ).squeeze() #Equation 2 in paper

            # for each training example...
            #is training example padded on the outside?
            # if yes, set the score for padded entries to -100

            for idx, num in enumerate(x_len):
                if num < max_x_len:
                    cond_score[idx, :, num:] = -100
        else:
            h_enc_expand = h_enc.unsqueeze(1)
            scores = []
            choices = []
            done_set = set()
            mask = torch.ByteTensor(np.array(np.arange(max_x_len) >= np.array(x_len)[:,None], dtype=np.int))

            t = 0

            cur_inp = Variable(torch.zeros(B, 1, self.max_tok_num))
            cur_inp.data[:,0,7] = 1  #Set the <BEG> token
            cur_h = decoder_hidden
            while len(done_set) < B and t < 50:
                g_s, cur_h = self.cond_decoder(cur_inp, cur_h)
                g_s_expand = g_s.unsqueeze(2)

                cur_cond_score = self.cond_out(self.cond_out_h(h_enc_expand) +
                        self.cond_out_g(g_s_expand)).squeeze() #shape = (batch_size, max_x_len)

                cur_cond_score[mask] = -100
                scores.append(cur_cond_score)

                if reinforce:
                    ans_tok_var = self.softmax(cur_cond_score).multinomial() #Convert to a prob distribution for each row and then sample one value per training example
                    choices.append(ans_tok_var)
                else:
                    _, ans_tok_var = cur_cond_score.max(1).unsqueeze(1) #get highest score value and corresponding index for each row

                # Get the next sets of actions
                cur_inp = Variable(torch.zeros(B, self.max_tok_num).scatter_(1, ans_tok_var.data, 1)) # one hot
                cur_inp = cur_inp.unsqueeze(1)

                #Find the <END> (=1) token and add to done_set if found
                [done_set.add(a.numpy()[0]) for a in torch.nonzero(ans_tok_var.data == 1)]
                t += 1

            cond_score = torch.stack(scores, 1)

        if reinforce:
        	return cond_score, choices # choices are the prob action values
            # utilized during reinforce backward - In policy Gradient method
        else:
            return cond_score


import os
import pathlib

def analyze_data(file: str):
    if not os.path.exists(file):
        # if not os.path.isfile(file):
        # if not os.path.isdir(file):

        print(f"Does not exist: {file}")
        return

    with open(file) as fp:
        ...

def analyze_data(file: str):
    try:
        with open(file) as fp:
            data = fp.read()
    except FileNotFoundError:
        print(f"Does not exist: {file}")

def analyze_data(file: str):
    try:
        data = pathlib.Path(file).read_text()
    except FileNotFoundError:
        print(f"Does not exist: {file}")
    except OSError:
        print(f"Failed to open: {file}")
    else:
        ...

def analyze_data(path: pathlib.Path):
    try:
        data = path.read_text()
    except FileNotFoundError:
        print(f"Does not exist: {path}")
    except OSError:
        print(f"Failed to open: {path}")
    else:
        ...

def initialize_app():
    config_dir = "."
    if is_user_config_dir_present(config_dir):
        ...  # load config files
    else:
        ...  # load defaults

def your_options():
    # file = "/path/to/some/file.txt"
    # path = pathlib.Path(file)
    path = pathlib.Path("path", "to", "some", "file.txt")
    if path.is_dir():
        print("Found")
    else:
        print("Not found")


import os
from collections import OrderedDict

from conans.errors import ConanException
from conans.util.conan_v2_mode import conan_v2_behavior

DEFAULT_INCLUDE = "include"
DEFAULT_LIB = "lib"
DEFAULT_BIN = "bin"
DEFAULT_RES = "res"
DEFAULT_SHARE = "share"
DEFAULT_BUILD = ""
DEFAULT_FRAMEWORK = "Frameworks"

class DefaultOrderedDict(OrderedDict):

    def __init__(self, factory):
        self.factory = factory
        super(DefaultOrderedDict, self).__init__()

    def __getitem__(self, key):
        if key not in self.keys():
            super(DefaultOrderedDict, self).__setitem__(key, self.factory())
        return super(DefaultOrderedDict, self).__getitem__(key)

class _CppInfo(object):
    """ Object that stores all the necessary information to build in C/C++.
    It is intended to be system independent, translation to
    specific systems will be produced from this info
    """
    def __init__(self):
        self.name = None
        self.names = {}
        self.system_libs = []  # Ordered list of system libraries
        self.includedirs = []  # Ordered list of include paths
        self.srcdirs = []  # Ordered list of source paths
        self.libdirs = []  # Directories to find libraries
        self.resdirs = []  # Directories to find resources, data, etc
        self.bindirs = []  # Directories to find executables and shared libs
        self.builddirs = []
        self.frameworks = []  # Macos .framework
        self.frameworkdirs = []
        self.rootpaths = []
        self.libs = []  # The libs to link against
        self.defines = []  # preprocessor definitions
        self.cflags = []  # pure C flags
        self.cxxflags = []  # C++ compilation flags
        self.sharedlinkflags = []  # linker flags
        self.exelinkflags = []  # linker flags
        self.build_modules = []
        self.rootpath = ""
        self.sysroot = ""
        self._build_modules_paths = None
        self._include_paths = None
        self._lib_paths = None
        self._bin_paths = None
        self._build_paths = None
        self._res_paths = None
        self._src_paths = None
        self._framework_paths = None
        self.version = None  # Version of the conan package
        self.description = None  # Description of the conan package
        # When package is editable, filter_empty=False, so empty dirs are maintained
        self.filter_empty = True

    def _filter_paths(self, paths):
        abs_paths = [os.path.join(self.rootpath, p)
                     if not os.path.isabs(p) else p for p in paths]
        if self.filter_empty:
            return [p for p in abs_paths if os.path.isdir(p)]
        else:
            return abs_paths

    @property
    def build_modules_paths(self):
        if self._build_modules_paths is None:
            self._build_modules_paths = [os.path.join(self.rootpath, p) if not os.path.isabs(p)
                                         else p for p in self.build_modules]
        return self._build_modules_paths

    @property
    def include_paths(self):
        if self._include_paths is None:
            self._include_paths = self._filter_paths(self.includedirs)
        return self._include_paths

    @property
    def lib_paths(self):
        if self._lib_paths is None:
            self._lib_paths = self._filter_paths(self.libdirs)
        return self._lib_paths

    @property
    def src_paths(self):
        if self._src_paths is None:
            self._src_paths = self._filter_paths(self.srcdirs)
        return self._src_paths

    @property
    def bin_paths(self):
        if self._bin_paths is None:
            self._bin_paths = self._filter_paths(self.bindirs)
        return self._bin_paths

    @property
    def build_paths(self):
        if self._build_paths is None:
            self._build_paths = self._filter_paths(self.builddirs)
        return self._build_paths

    @property
    def res_paths(self):
        if self._res_paths is None:
            self._res_paths = self._filter_paths(self.resdirs)
        return self._res_paths

    @property
    def framework_paths(self):
        if self._framework_paths is None:
            self._framework_paths = self._filter_paths(self.frameworkdirs)
        return self._framework_paths

    def get_name(self, generator):
        return self.names.get(generator, self.name)

    # Compatibility for 'cppflags' (old style property to allow decoration)
    def get_cppflags(self):
        conan_v2_behavior("'cpp_info.cppflags' is deprecated, use 'cxxflags' instead")
        return self.cxxflags

    def set_cppflags(self, value):
        conan_v2_behavior("'cpp_info.cppflags' is deprecated, use 'cxxflags' instead")
        self.cxxflags = value

    cppflags = property(get_cppflags, set_cppflags)

class Component(_CppInfo):

    def __init__(self, rootpath):
        super(Component, self).__init__()
        self.rootpath = rootpath
        self.includedirs.append(DEFAULT_INCLUDE)
        self.libdirs.append(DEFAULT_LIB)
        self.bindirs.append(DEFAULT_BIN)
        self.resdirs.append(DEFAULT_RES)
        self.builddirs.append(DEFAULT_BUILD)
        self.frameworkdirs.append(DEFAULT_FRAMEWORK)

class CppInfo(_CppInfo):
    """ Build Information declared to be used by the CONSUMERS of a
    conans. That means that consumers must use this flags and configs i order
    to build properly.
    Defined in user CONANFILE, directories are relative at user definition time
    """
    def __init__(self, root_folder):
        super(CppInfo, self).__init__()
        self.rootpath = root_folder  # the full path of the package in which the conans is found
        self.includedirs.append(DEFAULT_INCLUDE)
        self.libdirs.append(DEFAULT_LIB)
        self.bindirs.append(DEFAULT_BIN)
        self.resdirs.append(DEFAULT_RES)
        self.builddirs.append(DEFAULT_BUILD)
        self.frameworkdirs.append(DEFAULT_FRAMEWORK)
        self.components = DefaultOrderedDict(lambda: Component(self.rootpath))
        # public_deps is needed to accumulate list of deps for cmake targets
        self.public_deps = []
        self.configs = {}

    def __getattr__(self, config):
        def _get_cpp_info():
            result = _CppInfo()
            result.rootpath = self.rootpath
            result.sysroot = self.sysroot
            result.includedirs.append(DEFAULT_INCLUDE)
            result.libdirs.append(DEFAULT_LIB)
            result.bindirs.append(DEFAULT_BIN)
            result.resdirs.append(DEFAULT_RES)
            result.builddirs.append(DEFAULT_BUILD)
            result.frameworkdirs.append(DEFAULT_FRAMEWORK)
            return result

        return self.configs.setdefault(config, _get_cpp_info())

    def _raise_if_mixing_components(self):
        if (self.includedirs != [DEFAULT_INCLUDE] or
                self.libdirs != [DEFAULT_LIB] or
                self.bindirs != [DEFAULT_BIN] or
                self.resdirs != [DEFAULT_RES] or
                self.frameworkdirs != [DEFAULT_FRAMEWORK] or
                self.libs or
                self.system_libs or
                self.frameworks or
                self.defines or
                self.cflags or
                self.cxxflags or
                self.sharedlinkflags or
                self.exelinkflags or
                self.build_modules) and self.components:
            raise ConanException("self.cpp_info.components cannot be used with self.cpp_info "
                                 "global values at the same time")
        if self.configs and self.components:
            raise ConanException("self.cpp_info.components cannot be used with self.cpp_info configs"
                                 " (release/debug/...) at the same time")

class _BaseDepsCppInfo(_CppInfo):
    def __init__(self):
        super(_BaseDepsCppInfo, self).__init__()

    def update(self, dep_cpp_info):

        def merge_lists(seq1, seq2):
            return [s for s in seq1 if s not in seq2] + seq2

        self.system_libs = merge_lists(self.system_libs, dep_cpp_info.system_libs)
        self.includedirs = merge_lists(self.includedirs, dep_cpp_info.include_paths)
        self.srcdirs = merge_lists(self.srcdirs, dep_cpp_info.src_paths)
        self.libdirs = merge_lists(self.libdirs, dep_cpp_info.lib_paths)
        self.bindirs = merge_lists(self.bindirs, dep_cpp_info.bin_paths)
        self.resdirs = merge_lists(self.resdirs, dep_cpp_info.res_paths)
        self.builddirs = merge_lists(self.builddirs, dep_cpp_info.build_paths)
        self.frameworkdirs = merge_lists(self.frameworkdirs, dep_cpp_info.framework_paths)
        self.libs = merge_lists(self.libs, dep_cpp_info.libs)
        self.frameworks = merge_lists(self.frameworks, dep_cpp_info.frameworks)
        self.build_modules = merge_lists(self.build_modules, dep_cpp_info.build_modules_paths)
        self.rootpaths.append(dep_cpp_info.rootpath)

        # Note these are in reverse order
        self.defines = merge_lists(dep_cpp_info.defines, self.defines)
        self.cxxflags = merge_lists(dep_cpp_info.cxxflags, self.cxxflags)
        self.cflags = merge_lists(dep_cpp_info.cflags, self.cflags)
        self.sharedlinkflags = merge_lists(dep_cpp_info.sharedlinkflags, self.sharedlinkflags)
        self.exelinkflags = merge_lists(dep_cpp_info.exelinkflags, self.exelinkflags)

        if not self.sysroot:
            self.sysroot = dep_cpp_info.sysroot

    @property
    def build_modules_paths(self):
        return self.build_modules

    @property
    def include_paths(self):
        return self.includedirs

    @property
    def lib_paths(self):
        return self.libdirs

    @property
    def src_paths(self):
        return self.srcdirs

    @property
    def bin_paths(self):
        return self.bindirs

    @property
    def build_paths(self):
        return self.builddirs

    @property
    def res_paths(self):
        return self.resdirs

    @property
    def framework_paths(self):
        return self.frameworkdirs

class DepCppInfo(object):

    def __init__(self, cpp_info):
        self._cpp_info = cpp_info
        self._libs = None
        self._system_libs = None
        self._frameworks = None
        self._defines = None
        self._cxxflags = None
        self._cflags = None
        self._sharedlinkflags = None
        self._exelinkflags = None

        self._include_paths = None
        self._lib_paths = None
        self._bin_paths = None
        self._build_paths = None
        self._res_paths = None
        self._src_paths = None
        self._framework_paths = None
        self._build_module_paths = None

    def __getattr__(self, item):
        try:
            attr = self._cpp_info.__getattribute__(item)
        except AttributeError:  # item is not defined, get config (CppInfo)
            attr = self._cpp_info.__getattr__(item)
        return attr

    @staticmethod
    def _merge_lists(seq1, seq2):
        return seq1 + [s for s in seq2 if s not in seq1]

    def _aggregated_values(self, item):
        values = getattr(self, "_%s" % item)
        if values is not None:
            return values
        values = getattr(self._cpp_info, item)
        if self._cpp_info.components:
            for component in self._cpp_info.components.values():
                values = self._merge_lists(values, getattr(component, item))
        setattr(self, "_%s" % item, values)
        return values

    def _aggregated_paths(self, item):
        paths = getattr(self, "_%s_paths" % item)
        if paths is not None:
            return paths
        paths = getattr(self._cpp_info, "%s_paths" % item)
        if self._cpp_info.components:
            for component in self._cpp_info.components.values():
                paths = self._merge_lists(paths, getattr(component, "%s_paths" % item))
        setattr(self, "_%s_paths" % item, paths)
        return paths

    @property
    def build_modules_paths(self):
        return self._aggregated_paths("build_modules")

    @property
    def include_paths(self):
        return self._aggregated_paths("include")

    @property
    def lib_paths(self):
        return self._aggregated_paths("lib")

    @property
    def src_paths(self):
        return self._aggregated_paths("src")

    @property
    def bin_paths(self):
        return self._aggregated_paths("bin")

    @property
    def build_paths(self):
        return self._aggregated_paths("build")

    @property
    def res_paths(self):
        return self._aggregated_paths("res")

    @property
    def framework_paths(self):
        return self._aggregated_paths("framework")

    @property
    def libs(self):
        return self._aggregated_values("libs")

    @property
    def system_libs(self):
        return self._aggregated_values("system_libs")

    @property
    def frameworks(self):
        return self._aggregated_values("frameworks")

    @property
    def defines(self):
        return self._aggregated_values("defines")

    @property
    def cxxflags(self):
        return self._aggregated_values("cxxflags")

    @property
    def cflags(self):
        return self._aggregated_values("cflags")

    @property
    def sharedlinkflags(self):
        return self._aggregated_values("sharedlinkflags")

    @property
    def exelinkflags(self):
        return self._aggregated_values("exelinkflags")

class DepsCppInfo(_BaseDepsCppInfo):
    """ Build Information necessary to build a given conans. It contains the
    flags, directories and options if its dependencies. The conans CONANFILE
    should use these flags to pass them to the underlaying build system (Cmake, make),
    so deps info is managed
    """

    def __init__(self):
        super(DepsCppInfo, self).__init__()
        self._dependencies = OrderedDict()
        self.configs = {}

    def __getattr__(self, config):
        return self.configs.setdefault(config, _BaseDepsCppInfo())

    @property
    def dependencies(self):
        return self._dependencies.items()

    @property
    def deps(self):
        return self._dependencies.keys()

    def __getitem__(self, item):
        return self._dependencies[item]

    def update(self, cpp_info, pkg_name):
        assert isinstance(cpp_info, (CppInfo, DepCppInfo))
        self._dependencies[pkg_name] = cpp_info
        super(DepsCppInfo, self).update(cpp_info)
        for config, cpp_info in cpp_info.configs.items():
            self.configs.setdefault(config, _BaseDepsCppInfo()).update(cpp_info)


#PureGymFunctions
import datetime
import sqlite3

def time2num(t):
#Takes a (date)time and converts it to a number of minutes index
    hour = t.hour
    minute = t.minute
    num = (hour * 60) + minute
    return num, hour, minute

def num2time(n):
#takes a number of minutes index and converts it to a (date)time
    minute = int(n%60)
    hour = int((n-minute)/60)
    t = datetime.datetime(2020,1,1,hour,minute,0,0)
    return t, hour, minute

if __name__ == '__main__':
    databases = ['PG2020_SQL.db','PG2020_SQL_CLEAN.db']
    for db in databases:
        print(db)
        conn = sqlite3.connect(db)
        c = conn.cursor()
        c.execute("SELECT name FROM sqlite_master WHERE type='table';")
        print(c.fetchall())

        # strvar = 'Day TEXT PRIMARY KEY, '
        # for i in range(1439):
        #     strvar = strvar + 't'+str(i+1)+' INTEGER, '
        # strvar = strvar + 't1440 INTEGER'
        #print("CREATE TABLE test_table ({})".format(strvar))

        #c.execute("CREATE TABLE test_table ({})".format(strvar))
        #conn.commit()

        ###-----CREATE&INSERT NULL ROWS--------------###
        numdays = 365
        start_date = datetime.datetime(2021,10,1,0,0,0,0)
        date_list = [start_date + datetime.timedelta(days=x) for x in range(numdays)]
        date_list1 = ['{}/{}/{}'.format(d.strftime('%d'),d.strftime('%m'),d.strftime('%y')) for d in date_list]
        #print(date_list1)

        for j in range(numdays):
            #strrow = "'01/11/20', "
            strrow = "'{}', ".format(date_list1[j])
            for i in range(1440):
                strrow = strrow + 'NULL, '
            strrow = strrow + 'NULL, 0' #COMMENT: NULL    ML:0
            #print(strrow)
            c.execute("INSERT INTO puregym_table VALUES ({})".format(strrow))
        conn.commit()

        ###---ADD COMMENTS COLUMN---###
        #c.execute("ALTER TABLE puregym_table ADD Comments TEXT")
        #conn.commit()

        conn.close()


import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
import matplotlib.pyplot as plt

rawfile = r"E:\\Programming languages\\Python\\Machine_Learning\\07_heart_attack\\heart.csv"
dataset = pd.read_csv(rawfile)
print(dataset)

x_train = dataset.iloc[:,:-1].values
y_train = dataset.iloc[:,-1].values

x_test = dataset.iloc[:,:-1]

scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)

x_train = pd.DataFrame(x_train)
y_train = pd.DataFrame(y_train)

x_test = pd.DataFrame(x_test)

params = {'C':[2,8,12],'max_iter':[150,250]}
lg_model = LogisticRegression(C=50,max_iter=85)
lg_model_grid = GridSearchCV(lg_model,params,cv=5)
lg_model_grid.fit(x_train,y_train)
y_predicted = lg_model_grid.predict(x_test)

confusion_mat = confusion_matrix(y_train,y_predicted)
accuracy_scr = accuracy_score(y_train,y_predicted)
classification_rpt = classification_report(y_train,y_predicted)

list_pre = []

y_predicted = list(y_predicted)
list_pre.insert(0,y_predicted.count(0))
list_pre.insert(1,y_predicted.count(1))

color = ['green','red']
explode = [0.2,0.0]
label = ["Heart Attack Not-Declared","Heart Attack Declared"]
plt.pie(list_pre,labels=label,shadow=True,autopct="%1.1f%%",explode=explode,colors=color,wedgeprops={'edgecolor':'black'})
plt.title("Heart Attack prediction")
plt.tight_layout()
plt.show()