import re
import six
import time
import inspect
import importlib
from .dataType import *
import spannerorm.base_model
from datetime import date
from .relation import Relation
from google.api_core.datetime_helpers import DatetimeWithNanoseconds

class Helper(object):
    @classmethod
    def is_property(cls, v):
        """
        Check is property

        :type: object
        :param v:

        :return:
        """
        return isinstance(v, property)

    @classmethod
    def is_attr(cls, v):
        """
        Check is model attr

        :type: object
        :param v:
        """
        return isinstance(v, StringField) | isinstance(v, IntegerField) | isinstance(v, BoolField) \
               | isinstance(v, IntegerField) | isinstance(v, FloatField) | isinstance(v, BytesField) \
               | isinstance(v, DateField) | isinstance(v, TimeStampField) | isinstance(v, EnumField)

    @classmethod
    def is_relational_attr(cls, v):
        """
        Check is model relational attr

        :type: object
        :param v:
        """
        return isinstance(v, Relation)

    @classmethod
    def get_model_prop_by_name(cls, model_cls, prop_name):
        """
        Return model prop by name

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls:

        :type prop_name: str
        :param prop_name:

        :rtype: property | None
        :return:
        """
        for key, prop in inspect.getmembers(model_cls, Helper.is_property):
            if key == prop_name:
                return prop

        return None

    @classmethod
    def get_model_props_value_by_key(cls, model_obj, prop_name):
        """
        Return model props key-value

        :type model_obj: spannerorm.base_model.BaseModel
        :param model_obj: model

        :type prop_name: str
        :param prop_name: property name

        :rtype: dict
        :return:
        """
        for key, value in inspect.getmembers(model_obj.__class__, Helper.is_property):
            if key == prop_name:
                return model_obj.__getattribute__(key)

        return None

    @classmethod
    def get_model_props(cls, model_cls):
        """
        Return model props

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: model

        :rtype: dict
        :return:
        """
        model_props = {}
        for key, value in inspect.getmembers(model_cls, Helper.is_property):
            model_props[key] = value

        return model_props

    @classmethod
    def get_model_attrs(cls, model_cls):
        """
        Return model attr

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: Model class

        :rtype: dict
        :return: model attributes in key value pairs
        """
        attrs = {}
        for key, value in inspect.getmembers(model_cls, Helper.is_attr):
            attrs[key] = value

        return attrs

    @classmethod
    def get_model_relations_attrs(cls, model_cls):
        """
        Return model relation attrs

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: Model class

        :rtype: dict
        :return: model relational attributes in key value pairs
        """
        attrs = {}
        for key, value in inspect.getmembers(model_cls, Helper.is_relational_attr):
            attrs[key] = value

        return attrs

    @classmethod
    def get_model_props_key_value(cls, model_obj):
        """
        Return model props key-value

        :type model_obj: spannerorm.base_model.BaseModel
        :param model_obj: model

        :rtype: dict
        :return:
        """
        model_props = {}
        for key, value in inspect.getmembers(model_obj.__class__, Helper.is_property):
            model_props[key] = model_obj.__getattribute__(key)

        return model_props

    @classmethod
    def model_attr_by_prop(cls, model_cls, prop):
        """
        Return model attribute by property

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: Model class

        :type prop: property
        :param prop: Model class property

        :rtype: DataType
        :return: Model attribute
        """
        if isinstance(prop, property) is False:
            raise TypeError('Invalid object property')

        return Helper.model_attr_by_prop_name(model_cls, prop.fget.__name__)

    @classmethod
    def model_attr_by_prop_name(cls, model_cls, prop_name):
        """
        Return model attribute by property name

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: Model class

        :type prop_name: str
        :param prop_name: Model class property name

        :rtype: DataType
        :return: Model attribute
        """
        model_attr_name = '_' + prop_name
        model_attrs = Helper.get_model_attrs(model_cls)
        if model_attr_name not in model_attrs:
            raise TypeError('Criteria model property {} not exist'.format(model_attr_name))

        return model_attrs.get(model_attr_name)

    @classmethod
    def model_relational_attr_by_prop(cls, model_cls, prop):
        """
        Return model relational attribute by property

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: Model class

        :type prop: property
        :param prop: Model class property

        :rtype: Relation
        :return: Model relational attribute
        """
        if isinstance(prop, property) is False:
            raise TypeError('Invalid object property')

        return Helper.model_relational_attr_by_prop_name(model_cls, prop.fget.__name__)

    @classmethod
    def model_relational_attr_by_prop_name(cls, model_cls, prop_name):
        """
        Return model relational attribute by property name

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: Model class

        :type prop_name: str
        :param prop_name: Model class property name

        :rtype: Relation
        :return: Model relational attribute
        """
        model_attr_name = '_' + prop_name
        model_attrs = Helper.get_model_relations_attrs(model_cls)
        if model_attr_name not in model_attrs:
            raise TypeError('Criteria model property {} not exist'.format(model_attr_name))

        return model_attrs.get(model_attr_name)

    @classmethod
    def get_db_columns(cls, model_cls):
        model_attrs = Helper.get_model_attrs(model_cls)

        columns = []
        for attr_name in model_attrs:
            attr = model_attrs.get(attr_name)
            columns.append(attr.db_column)

        return columns

    @classmethod
    def validate_model_prop(cls, model_cls, prop):
        """
        Validate model attr

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls: Model class

        :type prop: property
        :param prop:

        :rtype: dict
        :return: {'is_valid':bool, 'error_msg':str}
        """
        if isinstance(prop, property) is False:
            raise TypeError('Invalid object property')

        model_attr_name = '_' + prop.fget.__name__
        model_attrs = Helper.get_model_attrs(model_cls)

        if model_attr_name in model_attrs:
            attr = model_attrs.get(model_attr_name)
            if isinstance(attr, IntegerField) or isinstance(attr, FloatField):
                return Helper.validate_number_field(attr.value, max_value=attr.max_value, min_value=attr.min_value,
                                                    null=attr.null)
            elif isinstance(attr, StringField):
                return Helper.validate_string_field(attr.value, max_length=attr.max_length, reg_exr=attr.reg_exr,
                                                    null=attr.null)
            elif isinstance(attr, BoolField):
                return Helper.validate_bool_field(attr.value, null=attr.null)
            elif isinstance(attr, TimeStampField):
                return Helper.validate_timestamp_field(attr.value, null=attr.null)
            elif isinstance(attr, DateField):
                return Helper.validate_date_field(attr.value, null=attr.null)
            elif isinstance(attr, EnumField):
                return Helper.validate_enum_field(attr.value, enum_list=attr.enum_list, null=attr.null)

        return {
            'is_valid': True,
            'error_msg': None
        }

    @classmethod
    def validate_number_field(cls, value, max_value=None, min_value=None, null=True):
        """
        Validate number field value

        :type value: int | float
        :param value: integer value

        :type max_value: int
        :param max_value: max allow number value

        :type min_value: int
        :param min_value: min allow integer value

        :type null: bool
        :param null: is allow None value

        :rtype: dict
        :return: {'is_valid':bool, 'error_msg':str}
        """

        is_valid = True
        error_msg = None
        if null is False and value is None:
            is_valid = False
            error_msg = 'Property value should not be None'

        if value is not None:
            if isinstance(value, int) is False and isinstance(value, float) is False:
                is_valid = False
                error_msg = 'Data type should be <int>'

            if max_value is not None and value > max_value:
                is_valid = False
                error_msg = 'Max allow value: {}'.format(max_value)

            if min_value is not None and value < min_value:
                is_valid = False
                error_msg = 'Min allow value: {}'.format(min_value)

        return {
            'is_valid': is_valid,
            'error_msg': error_msg
        }

    @classmethod
    def validate_string_field(cls, value, max_length=None, reg_exr=None, null=True):
        """
        Validate string field value

        :type max_length: int
        :param max_length: max allow string lenght

        :type reg_exr: str
        :param reg_exr: regex pattern

        :type null: bool
        :param null: is allow None value

        :rtype: dict
        :return: {'is_valid':bool, 'error_msg':str}
        """
        is_valid = True
        error_msg = None
        if null is False and (value is None or (value is not None and str(value).strip() == '')):
            is_valid = False
            error_msg = 'Data should not be None or empty'

        if value is not None:
            if isinstance(value, six.string_types) is False:
                is_valid = False
                error_msg = 'Data type should be <str>'

            if max_length is not None and len(value) > max_length:
                is_valid = False
                error_msg = 'Max allow string length: {}'.format(max_length)

            if reg_exr is not None:
                pattern = re.compile(reg_exr)
                if pattern.match(value) is None:
                    is_valid = False
                    error_msg = 'String should match regex pattern: {}'.format(reg_exr)

        return {
            'is_valid': is_valid,
            'error_msg': error_msg
        }

    @classmethod
    def validate_bool_field(cls, value, null=True):
        """
        Validate bool field value

        :type value: bool
        :param value:

        :type null: bool
        :param null: is allow None value

        :rtype: dict
        :return: {'is_valid':bool, 'error_msg':str}
        """
        is_valid = True
        error_msg = None
        if null is False and value is None:
            is_valid = False
            error_msg = 'Data should not be None'

        if value is not None and isinstance(value, bool) is False:
            is_valid = False
            error_msg = 'Data type should be <bool>'

        return {
            'is_valid': is_valid,
            'error_msg': error_msg
        }

    @classmethod
    def validate_timestamp_field(cls, value, null=True):
        """
        Validate timestamp field value

        :type value: int | float
        :param value:

        :type null: bool
        :param null: is allow None value

        :rtype: dict
        :return: {'is_valid':bool, 'error_msg':str}
        """
        is_valid = True
        error_msg = None
        if null is False and value is None:
            is_valid = False
            error_msg = 'Data should not be None'

        if value is not None and isinstance(value, int) is False and isinstance(value, float) is False:
            is_valid = False
            error_msg = 'Data type should be <float> or <int> timestamp'

        return {
            'is_valid': is_valid,
            'error_msg': error_msg
        }

    @classmethod
    def validate_date_field(cls, value, null=True):
        """
        Validate enum field value

        :type value: date
        :param value:

        :type null: bool
        :param null: is allow None value

        :rtype: dict
        :return: {'is_valid':bool, 'error_msg':str}
        """
        is_valid = True
        error_msg = None
        if null is False and value is None:
            is_valid = False
            error_msg = 'Data should not be None'

        if value is not None and isinstance(value, date) is False:
            is_valid = False
            error_msg = 'Data type should be <datetime.date>'

        return {
            'is_valid': is_valid,
            'error_msg': error_msg
        }

    @classmethod
    def validate_enum_field(cls, value, enum_list, null=True):
        """
        Validate enum field value

        :type value: object
        :param value:

        :type enum_list: list
        :param enum_list: list of allow value

        :type null: bool
        :param null: is allow None value

        :rtype: dict
        :return: {'is_valid':bool, 'error_msg':str}
        """
        is_valid = True
        error_msg = None
        if null is False and value is None:
            is_valid = False
            error_msg = 'Data should not be None'

        if value is not None:
            if value in enum_list is False:
                is_valid = False
                error_msg = 'Data value should be from list: {}'.format(enum_list)

        return {
            'is_valid': is_valid,
            'error_msg': error_msg
        }

    @classmethod
    def get_model_props_details(cls, model_cls):
        """
        Return model props details

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls:

        :rtype: dict
        :return:
        """
        model_props = Helper.get_model_props(model_cls)
        model_attrs = Helper.get_model_attrs(model_cls)

        props_details = {}
        for prop_name in model_props:
            model_attr_name = '_' + prop_name
            if model_attr_name in model_attrs:
                attr = model_attrs.get(model_attr_name)
                props_details.update({
                    prop_name: Helper.get_prop_details(attr)
                })

        return props_details

    @classmethod
    def get_relation_props_details(cls, model_cls):
        """
        Return model relation props details

        :type model_cls: spannerorm.base_model.BaseModel
        :param model_cls:

        :rtype: dict
        :return:
        """
        model_props = Helper.get_model_props(model_cls)
        model_relation_attrs = Helper.get_model_relations_attrs(model_cls)

        props_details = {}
        for prop_name in model_props:
            model_attr_name = '_' + prop_name
            if model_attr_name in model_relation_attrs:
                attr = model_relation_attrs.get(model_attr_name)
                props_details.update({
                    prop_name: Helper.get_relation_pop_detail(attr)
                })

        return props_details

    @classmethod
    def get_prop_details(cls, attr):
        """
        Return model attr field details

        :type attr: DataType
        :param attr:

        :rtype: dict
        :return:
        """
        details = {
            'db_column': attr.db_column,
            'null': attr.null,
            'default_value': attr.default
        }

        if isinstance(attr, IntegerField):
            details.update({
                'data_type': 'IntegerField',
                'max_value': attr.max_value,
                'min_value': attr.min_value
            })
        if isinstance(attr, FloatField):
            details.update({
                'data_type': 'FloatField',
                'max_value': attr.max_value,
                'min_value': attr.min_value,
                'decimal_places': attr.decimal_places
            })
        if isinstance(attr, StringField):
            details.update({
                'data_type': 'StringField',
                'max_length': attr.max_length,
                'reg_exr': attr.reg_exr
            })
        if isinstance(attr, EnumField):
            details.update({
                'data_type': 'EnumField',
                'enum_list': attr.enum_list
            })
        if isinstance(attr, DateField):
            details.update({
                'data_type': 'DateField'
            })
        if isinstance(attr, TimeStampField):
            details.update({
                'data_type': 'TimeStampField'
            })
        if isinstance(attr, BoolField):
            details.update({
                'data_type': 'BoolField'
            })
        if isinstance(attr, BytesField):
            details.update({
                'data_type': 'BytesField'
            })

        return details

    @classmethod
    def get_relation_pop_detail(cls, attr):
        """
        Return model relation attr field details

        :type attr: Relation
        :param attr:

        :rtype: dict
        :return:
        """
        return {
            'relation_type': attr.relation_type,
            'relation_name': attr.relation_name,
            'join_on': attr.join_on,
            'refer_to': attr.refer_to
        }

    @classmethod
    def init_model_with_default(cls, model_class):
        """
        Init model object with default values

        :type model_class: spannerorm.base_model.BaseModel
        :param model_class:

        :rtype: spannerorm.base_model.BaseModel
        :return: model object
        """
        model_object = model_class()
        model_attrs = Helper.get_model_attrs(model_object)
        for attr_name in model_attrs:
            attr = model_attrs.get(attr_name)
            if attr.default is not None:
                attr.value = attr.default

        return model_object

    @classmethod
    def model_cls_by_module_name(cls, prop_module_name):
        """
        import module by name & return model

        :type prop_module_name: str
        :param prop_module_name:

        :rtype: spannerorm.base_model.BaseModel
        :return:
        """
        prop_module = importlib.import_module(prop_module_name)
        for name, model in inspect.getmembers(prop_module):
            if inspect.isclass(model) and prop_module_name == model.__module__:
                return model

        return None

    @classmethod
    def process_result_set(cls, results):
        data = []
        for row in results:
            row_data = {}

            for col in row:
                index = 0

                for field in results.fields:
                    if isinstance(row[index], six.string_types):
                        value = str(row[index])
                    elif isinstance(row[index], DatetimeWithNanoseconds):
                        value = time.mktime(row[index].timetuple())
                    else:
                        value = row[index]

                    field_name = str(field.name)
                    row_data[field_name] = value
                    index += 1
            data.append(row_data)

        return data


import torch
import torch.nn as nn
import torch.utils.data as Data
from torch.autograd import Variable
import torch.nn.functional as F
import time
import numpy as np
assert F
import csv,random
from imblearn.over_sampling import SMOTE

class Datamanager():
    def __init__(self):
        self.dataset = {}
        self.threshold = 1e-10              ## The threshold for classifying the label of the sample after simoid function
        self.normalize = {}                 ## Record the mean and standard deviation for testing set normalization

        self.over_sample = False            ## Determine whether copy the positive (attribute #4 = 2) samples
        self.over_sample_rate = 150         ## Number of copies

        self.down_sample = False            ## Determine whether delete the negative (attribute #4 = 1) samples
        self.down_sample_rate = 2           ## Make number of negative samples = down_sample_rate * Number of positive samples

        self.smote = True                   ## Determine whether use SMOTE to generate minor class samples    source: https://imbalanced-learn.org/en/stable/generated/imblearn.over_sampling.SMOTE.html

        self.weighted_loss = True           ## Determine whether adjust the weight of BCE loss function
        self.weighted_loss_rate = 0.1       ## Weight of negative samples in loss function ( 1 - weighted_loss_rate for positive samples)

    def get_data(self,name,file_name,b_size,args,shuf=True):
        with open(file_name,newline='') as csvfile:
            rows = csv.reader(csvfile)                                                          ## Read file
            data = []                                                                           ## Store the data from file
            for row in rows:
                data.append(row)
            data = data[2:]

            data = np.array(data)
            if name == 'train' :
                data = np.delete(data,2,0)                                                      ## Missing data => remove
                data = np.delete(data,0,0)                                                      ## Positive(attribute #4 = 2) outlier => remove
                data = np.delete(data,4,0)
                data = np.delete(data,5,0)
                data = np.delete(data,6,0)
                data = np.delete(data,11,0)
                data = np.delete(data,2542,0)
            if name == 'test' :
                data = np.delete(data,1103,0)
                data = np.delete(data,1102,0)
                data = np.delete(data,699,0)
                data = np.delete(data,5,0)

            data = np.delete(data,176,1)                                                        ## These columns have std = 0 => remove
            data = np.delete(data,167,1)
            data = np.delete(data,166,1)
            data = np.delete(data,165,1)
            data = np.delete(data,5,1)
            data = np.delete(data,4,1)
            data = np.delete(data,166,1)
            data = np.delete(data,165,1)
            data = np.delete(data,164,1)

            for i in range(data.shape[1]):
                if i == 3 :
                    for j in range(data.shape[0]):                                              ## Transform label of attribute #4 '2' to 1(positive), '1' to 0(negative)
                        if data[j][i] == '1':
                            data[j][i] = 0
                        elif data[j][i] == '2':
                            data[j][i] = 1
                        else:
                            print('error target')
                elif data[0][i] == 'TRUE' or data[0][i] == 'FALSE': #                           ## Transform label 'TRUE' to 1, 'Negative' to 0
                    for j in range(data.shape[0]):
                        if data[j][i] == 'TRUE':
                            data[j][i] = 1.0
                        elif data[j][i] == 'FALSE':
                            data[j][i] = 0.0
                        else:
                            print(j,i,data[j][i])
                            print('other type')
                    mean = data[:,i].astype(np.double).mean()                                   ## Normalization. Record mean and standard deviation
                    std = data[:,i].astype(np.double).std()
                    if(std == 0):
                        print(i)
                    data[:,i] = (data[:,i].astype(np.double) - mean) / std
                    self.normalize[i] = [mean,std]

                else:
                    if name == 'train':                                                         ## Normalization. Record mean and standard deviation
                        mean = data[:,i].astype(np.double).mean()
                        std = data[:,i].astype(np.double).std()
                        if(std == 0):
                            print(i)
                        data[:,i] = (data[:,i].astype(np.double) - mean) / std
                        self.normalize[i] = [mean,std]
                    else:
                        data[:,i] = (data[:,i].astype(np.double) - self.normalize[i][0]) / self.normalize[i][1]

            if name == 'train' :
                np.random.shuffle(data)
                Y = data[:int(data.shape[0]*0.9),3].reshape(-1,1).astype(np.double)             ## Split training and validation set, and extract attribute #4 as targets
                Y_val = data[int(data.shape[0]*0.9):,3].reshape(-1,1).astype(np.double)
                X = np.delete(data,3,1).astype(np.double)[:int(data.shape[0]*0.9),:]
                X_val = np.delete(data,3,1).astype(np.double)[int(data.shape[0]*0.9):,:]

                if self.over_sample or self.down_sample or self.smote:
                    count_0 = 0
                    count_1 = 0
                    count_1_list = []
                    for i in range(Y.shape[0]):
                        if Y[i][0] == 0:
                            count_0 = count_0 + 1
                        else:
                            count_1 = count_1 + 1
                            count_1_list.append(i)
                    print('count_0:',count_0)
                    print('count_1:',count_1)
                    if self.over_sample:                                                        ## Copy the positive (attribute #4 = 2) samples
                        ori_one_X , ori_one_Y = X[count_1_list] , Y[count_1_list]
                        for i in range(self.over_sample_rate):
                            noise = np.random.normal(0, 0.3, ori_one_X.shape)
                            add_one_X = ori_one_X + noise
                            X = np.concatenate((X,add_one_X),axis = 0)
                            Y = np.concatenate((Y,ori_one_Y),axis = 0)

                    if self.down_sample:                                                        ## Delete the negative (attribute #4 = 1) samples
                        number = int(count_0 - count_1 * (self.over_sample_rate + 1) * self.down_sample_rate)
                        while(number > 0):
                            for i in range(Y.shape[0]):
                                if Y[i][0] == 0:
                                    X = np.delete(X,i,0)
                                    Y = np.delete(Y,i,0)
                                    number = number - 1
                                    break
                    if self.smote:                                                              ## Use SMOTE to generate minor class(positive) samples
                        sm = SMOTE(sampling_strategy = 1)
                        X, Y = sm.fit_resample(X, Y)
                        Y = Y.reshape(-1,1)
                        count_0 = 0
                        count_1 = 0
                        for i in range(Y.shape[0]):
                            if Y[i][0] == 0:
                                count_0 = count_0 + 1
                            else:
                                count_1 = count_1 + 1
                        #print('count_0:',count_0)
                        #print('count_1:',count_1)

                #print(X.shape)
                #print(Y.shape)

                X,Y = torch.from_numpy(X).cuda(),torch.from_numpy(Y).cuda()                                         ## Convert numpy array to tensor for Pytorch
                train_dataset = Data.TensorDataset(data_tensor=X[:], target_tensor=Y[:])                            ## Wrap up the input/target tensor into TensorDataset   source: https://pytorch.org/docs/stable/data.html
                self.dataset['train'] = Data.DataLoader(dataset=train_dataset, batch_size=b_size, shuffle=shuf)     ## Put the TensorDataset in Dataloader (stored in a dictionary), shuffling the samples    source: https://pytorch.org/docs/stable/data.html

                X_val,Y_val = torch.from_numpy(X_val).cuda(),torch.from_numpy(Y_val).cuda()
                val_dataset = Data.TensorDataset(data_tensor=X_val[:], target_tensor=Y_val[:])
                self.dataset['val'] = Data.DataLoader(dataset=val_dataset, batch_size=b_size, shuffle=shuf)

            elif name == 'test':                                                                                ## Process the testing set
                Y = data[:,3].reshape(-1,1).astype(np.double)
                X = np.delete(data,3,1).astype(np.double)

                X,Y = torch.from_numpy(X).cuda(),torch.from_numpy(Y).cuda()
                train_dataset = Data.TensorDataset(data_tensor=X[:], target_tensor=Y[:])
                self.dataset['test'] = Data.DataLoader(dataset=train_dataset, batch_size=b_size, shuffle=shuf)      ## Put the TensorDataset in Dataloader (stored in a dictionary), not shuffling the samples    source: https://pytorch.org/docs/stable/data.html

    def train(self,model,trainloader,epoch):                                            ## Train the model
        model.train()                                                                   ## Set to training mode
        optimizer = torch.optim.Adam(model.parameters())                                ## Use Adam optimizer to optimize all DNN parameters    source: https://pytorch.org/docs/stable/optim.html
        loss_func = nn.BCELoss()                                                        ## Use binary cross entropoy for model evaluation       source: https://pytorch.org/docs/stable/nn.html
        total_loss = 0                                                                  ## Calculate total loss in a epoch
        t1_p1 = 0                                                                       ## Confusion matrix initialization
        t1_p0 = 0
        t0_p1 = 0
        t0_p0 = 0

        for batch_index, (x, y) in enumerate(trainloader):                              ## Process a batch of data in each timestep
            x, y= Variable(x).cuda(), Variable(y).cuda()
            output = model(x)                                                           ## Use present model to forecast the the result
            if self.weighted_loss:                                                      ## Adjust the weight of BCE loss functional             source: https://pytorch.org/docs/stable/nn.html
                weight = np.empty([len(x)])
                for i in range(len(x)):
                    weight[i] = self.weighted_loss_rate
                arr = np.where(y.data == 1)
                weight[arr[0].tolist()] = 1 - self.weighted_loss_rate
                weight = torch.from_numpy(weight).cuda().double().view(len(x),1)
                loss_func = nn.BCELoss(weight = weight)

            loss = loss_func(output,y)
            optimizer.zero_grad()                                                       ## Set the gradient in the previous time step to zero
            loss.backward()                                                             ## Back propagate    source: https://pytorch.org/docs/stable/optim.html
            optimizer.step()                                                            ## Gradient descent    source: https://pytorch.org/docs/stable/autograd.html
            if batch_index % 4 == 0:                                                    ## Print model status    source: https://pytorch.org/docs/stable/optim.html
                print('\rTrain Epoch: {} | [{}/{} ({:.0f}%)]\t '.format(
                        epoch, batch_index * len(x), len(trainloader.dataset),
                        100. * batch_index / len(trainloader)),end='')

            total_loss+= loss.data[0]*len(x)                                            ## Sum up batch loss

            pred = np.empty([len(x),1])                                                 ## Calculate confusion matrix
            output = output.cpu().data.numpy()

            for i in range(len(x)):
                if(output[i] > self.threshold):
                    pred[i,0] = 1
                else:
                    pred[i,0] = 0

            y = y.cpu().data.numpy()

            for i in range(pred.shape[0]):
                if y[i] == 1 and pred[i] == 1:
                    t1_p1 = t1_p1 + 1
                elif y[i] == 1 and pred[i] == 0:
                    t1_p0 = t1_p0 + 1
                elif y[i] == 0 and pred[i] == 1:
                    t0_p1 = t0_p1 + 1
                elif y[i] == 0 and pred[i] == 0:
                    t0_p0 = t0_p0 + 1

        total_loss/= len(trainloader.dataset)
        print('Total loss: {:.4f}'.format(total_loss))                                  ## Print model status
        print('t1_p1: ',t1_p1 , 't0_p1: ',t0_p1 )
        print('t1_p0: ',t1_p0 , 't0_p0: ',t0_p0 )
        return total_loss

    def val(self,model,name,valloader):                                                 ## Test the model
        model.eval()                                                                    ## Set to evaluation mode
        val_loss = 0                                                                    ## Calculate total loss
        t1_p1 = 0                                                                       ## Confusion matrix initialization
        t1_p0 = 0
        t0_p1 = 0
        t0_p0 = 0
        for x, y in valloader:
            x, y = Variable(x, volatile=True).cuda(), Variable(y,volatile=True).cuda()
            output = model(x)                                                           ## Use present model to forecast the the result
            val_loss += F.binary_cross_entropy(output, y, size_average=False).data[0]   ## Sum up batch loss

            pred = np.empty([len(x),1])                                                 ## Calculate confusion matrix
            output = output.cpu().data.numpy()
            for i in range(len(x)):
                if(output[i] > self.threshold):
                    pred[i,0] = 1
                else:
                    pred[i,0] = 0
            y = y.cpu().data.numpy()

            for i in range(pred.shape[0]):
                if y[i] == 1 and pred[i] == 1:
                    t1_p1 = t1_p1 + 1
                elif y[i] == 1 and pred[i] == 0:
                    t1_p0 = t1_p0 + 1
                elif y[i] == 0 and pred[i] == 1:
                    t0_p1 = t0_p1 + 1
                elif y[i] == 0 and pred[i] == 0:
                    t0_p0 = t0_p0 + 1

        val_loss /= len(valloader.dataset)
        print(name , ' set: Average loss: {:.4f}'.format(val_loss))                     ## Print model status
        print('t1_p1: ',t1_p1 , 't0_p1: ',t0_p1 )
        print('t1_p0: ',t1_p0 , 't0_p0: ',t0_p0 )
        return val_loss

class DNN(nn.Module):                                                                   ## Set up DNN
    def __init__(self,args):
        super(DNN, self).__init__()
        print(args.unit)
        self.den=nn.ModuleList()
        for i in range(1,len(args.unit)-1):                                             ## Set up hidden layers
            self.den.append( nn.Sequential(
                nn.Linear(args.unit[i-1], args.unit[i]),                                ## Source: https://pytorch.org/docs/stable/nn.html
                nn.ReLU(),
                nn.Dropout(0.2)
            ))
        self.den.append( nn.Sequential(
            nn.Linear(args.unit[-2], args.unit[-1]),
            nn.Dropout(0.2),
            nn.Sigmoid(),
        ))

    def forward(self, x):                                                               ## Connect layers and activation function
        for i in self.den:
            x = i(x)
        return x


#! /usr/bin/env python3
"""

--- besspin.py is the main BESSPIN-Tool-Suite program. All documentation and features are based on
    solely executing this file. Please do not execute any other file.
--- usage: besspin.py [-h] [-c CONFIGFILE | -cjson CONFIGFILESERIALIZED]
               [-w WORKINGDIRECTORY] [-l LOGFILE] [-d]
               [-ep {devHost,ciOnPrem,ciAWS,awsProd,awsDev}] [-job JOBID]

BESSPIN (Balancing Evaluation of System Security Properties with Industrial Needs)

optional arguments:
  -h, --help            show this help message and exit
  -c CONFIGFILE, --configFile CONFIGFILE
                        Overwrites the default config file: ./config.ini
  -cjson CONFIGFILESERIALIZED, --configFileSerialized CONFIGFILESERIALIZED
                        Overwrites and augments the default production
                        settings
  -w WORKINGDIRECTORY, --workingDirectory WORKINGDIRECTORY
                        Overwrites the default working directory: ./workDir/
  -l LOGFILE, --logFile LOGFILE
                        Overwrites the default logFile: ./${workDir}/besspin.log
  -d, --debug           Enable debugging mode.
  -ep {devHost,ciOnPrem,ciAWS,awsProd,awsDev}, --entrypoint {devHost,ciOnPrem,ciAWS,awsProd,awsDev}
                        The entrypoint
  -job JOBID, --jobId JOBID
                        The job ID in production mode.

--- Defaults:
        workingDirectory = ./workDir
        configFile = ./config.ini
        logFile = ./${workDir}/besspin.log
        logging level = INFO
        entrypoint = devHost
"""

try:
    from besspin.base.utils.misc import *
    from besspin.base.config import loadConfiguration, genProdConfig
    from besspin.target.launch import startBesspin, endBesspin, resetTarget
    from besspin.cyberPhys.launch import startCyberPhys, endCyberPhys
    from besspin.base.utils import aws
    from besspin.base.threadControl import createBesspinLocks
    import logging, argparse, os, shutil, atexit, signal
except Exception as exc:
    try:
        #check we're in nix
        import sys
        if (sys.executable.split('/')[1] != 'nix'):
            print (f"(Error)~  Please run within a nix shell. [Run <nix-shell> in besspin directory].")
        else:
            raise
    except:
        print(f"(Error)~  <{exc.__class__.__name__}>: {exc}")
    print(f"(Info)~  End of BESSPIN! [Exit code -1:Fatal]")
    exit(-1)

def main (xArgs):
    # Create working Directory
    repoDir = os.path.abspath(os.path.dirname(__file__))
    if (xArgs.workingDirectory):
        workDir = os.path.abspath(xArgs.workingDirectory)
    else:
        workDir = os.path.join(repoDir,'workDir')
    if (os.path.isdir(workDir)): # already exists, delete
        try:
            shutil.rmtree(workDir)
        except Exception as exc:
            print(f"(Error)~  Failed to delete <{workDir}>.\n{formatExc(exc)}.")
            exitBesspin(EXIT.Configuration, preSetup=True)
    try:
        os.mkdir(workDir)
    except Exception as exc:
        print(f"(Error)~  Failed to create the working directory <{workDir}>.\n{formatExc(exc)}.")
        exitBesspin(EXIT.Files_and_paths, preSetup=True)

    # Check log file
    if (xArgs.logFile):
        logFile = os.path.abspath(xArgs.logFile)
    else:
        logFile = os.path.join(workDir,'besspin.log')
    try:
        fLog = open(logFile,'w')
        fLog.close()
    except Exception as exc:
        print(f"(Error)~  Failed to create the log file <{logFile}>.\n{formatExc(exc)}.")
        exitBesspin(EXIT.Files_and_paths, preSetup=True)

    # Entrypoint
    if(xArgs.entrypoint is None):
        xArgs.entrypoint = 'devHost'

    # Check jobId is valid, if provided
    if(xArgs.jobId):
        if(not(re.match("^[A-Za-z0-9-_+.]+$", xArgs.jobId))):
            print("(Error)~  Provided jobId contained invalid character(s). It must match regex '[A-Za-z0-9-_+.]'")
            exitBesspin(EXIT.Files_and_paths, preSetup=True)

    # setup the logging
    logLevel = logging.DEBUG if (xArgs.debug) else logging.INFO
    logging.basicConfig(filename=logFile,filemode='w',format='%(asctime)s: (%(levelname)s)~  %(message)s',datefmt='%I:%M:%S %p',level=logLevel)
    printAndLog(f"Welcome to BESSPIN!")

    #Prepare the peaceful exit
    setSetting('trash',trashCanObj())
    atexit.register(exitPeacefully,getSetting('trash'))

    # Store critical settings
    setSetting('repoDir', repoDir)
    setSetting ('workDir', workDir)
    setSetting('logFile', logFile)
    setSetting('debugMode', xArgs.debug)
    setSetting('besspinEntrypoint',xArgs.entrypoint)
    setSetting('prodJobId', xArgs.jobId)

    # Load all configuration and setup settings
    setupEnvFile = os.path.join(repoDir,'besspin','base','utils','setupEnv.json')
    setSetting('setupEnvFile', setupEnvFile)
    if (xArgs.configFile):
        configFile = os.path.abspath(xArgs.configFile)
    elif (xArgs.configFileSerialized):
        configFile = os.path.join(workDir,'production.ini')
        genProdConfig (xArgs.configFileSerialized, configFile)
        printAndLog(f"Configuration deserialized successfully to <{configFile}>.")
    else:
        configFile = os.path.join(repoDir,'config.ini')
        printAndLog(f"Using the default configuration in <{configFile}>.")
    setSetting('configFile', configFile)

    #Load the config file(s)
    loadConfiguration(configFile)

    #Create the global semaphores/Locks
    createBesspinLocks()

    #launch the tool
    if (isEqSetting('mode','cyberPhys')):
        startCyberPhys()
        endCyberPhys()
    else:
        xTarget = startBesspin()

        instruction = None
        if (isEqSetting('mode','production')):
            def sendSuccessMsgToPortal (nodeSuffix, reasonSuffix):
                aws.sendSQS(getSetting(f'{getSetting("besspinEntrypoint")}SqsQueueTX'), logAndExit, 'success',
                        getSetting('prodJobId'), f"{getSetting('prodJobId')}-{nodeSuffix}",
                        reason=f'besspin-production-{reasonSuffix}',
                        hostIp=aws.getInstanceIp(logAndExit),
                        fpgaIp=getSetting('productionTargetIp')
                        )
                printAndLog(f"Sent {reasonSuffix} message to the SQS queue.")

            # Notify portal that we have deployed successfully
            sendSuccessMsgToPortal('DEPLOY','deployment')

            # Wait for portal to instruct us to do something
            while (instruction != 'termination'):
                instruction = aws.pollPortalIndefinitely (getSetting(f'{getSetting("besspinEntrypoint")}S3Bucket'), xTarget.process, logAndExit)
                if (instruction == 'deadProcess'):
                    warnAndLog ("The main process is dead. Will exit without a notice from Portal.")
                    break
                printAndLog(f"Received {instruction} notice from Portal.")

                if (instruction == 'reset'):
                    # execute reset flow
                    xTarget = resetTarget(xTarget)
                    # Notify portal that we have reset successfully
                    sendSuccessMsgToPortal('RESET','reset')

        endBesspin(xTarget,(instruction=='deadProcess'))

    exitBesspin(EXIT.Success)

if __name__ == '__main__':
    # Reading the bash arguments
    xArgParser = argparse.ArgumentParser (description='BESSPIN (Balancing Evaluation of System Security Properties with Industrial Needs)')
    xGroupConfig = xArgParser.add_mutually_exclusive_group(required=False)
    xGroupConfig.add_argument ('-c', '--configFile', help='Overwrites the default config file: ./config.ini')
    xGroupConfig.add_argument ('-cjson', '--configFileSerialized', help='Overwrites and augments the default production settings')
    xArgParser.add_argument ('-w', '--workingDirectory', help='Overwrites the default working directory: ./workDir/')
    xArgParser.add_argument ('-l', '--logFile', help='Overwrites the default logFile: ./${workDir}/besspin.log')
    xArgParser.add_argument ('-d', '--debug', help='Enable debugging mode.', action='store_true')
    xArgParser.add_argument ('-ep', '--entrypoint', choices=['devHost','ciOnPrem','ciAWS','awsProd','awsDev'], help='The entrypoint')
    xArgParser.add_argument ('-job', '--jobId', help='The job ID in production mode.')
    xArgs = xArgParser.parse_args()

    #Trapping the signals
    signalsToCatch = [signal.SIGINT, signal.SIGTERM]
    for xSignal in signalsToCatch:
        signal.signal(xSignal,exitOnInterrupt)

    main(xArgs)


import os.path

from django.http import HttpResponse

from cmis_storage.storage import CMISStorage

def get_file(request, path):
    """
    Returns a file stored in the CMIS-compatible content management system

    :param path: The full path of the file within the CMS
    """
    _, filename = os.path.split(path)

    storage = CMISStorage()
    stream = storage.open_stream(path)

    response = HttpResponse()
    response['Content-Disposition'] = 'attachment; filename=%s' % filename
    response.write(stream.read())

    return response


import os
import random

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

import torch
from  torch.utils.data import TensorDataset
import torch.nn as nn
import torch.nn.functional as F
import os
from transformers import logging
import warnings
import yaml
from pathlib import Path
logging.set_verbosity_warning()
task_path = './tasks/'
train_path = './SST2_train.tsv'
val_path = './SST2_dev.tsv'
test_path = './SST2_test.tsv'
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def load_config():
    with open(Path('/root/configs/config.yaml'), "r") as config_file:
        try:
            run_config = yaml.safe_load(config_file)
        except yaml.YAMLError as exc:
            print(exc)
            return {}

    return run_config

def transform_dataset(transformer, dataset, dataloader_flag = False):
    if transformer is None:
        dataset_new = dataset
        if dataloader_flag:
            feature_loader =torch.utils.data.DataLoader(dataset_new,
                                    batch_size = 128, shuffle = True)
            return feature_loader
        return dataset_new

    X = dataset.features
    X = X.reshape(X.shape[0], -1)
    feature = torch.tensor(transformer.fit_transform(X)).float()
    targets = dataset.targets.clone().detach()
    dataset_new = TensorDataset(feature, targets)
    dataset_new.features = feature
    dataset_new.targets = targets

    if dataloader_flag:
        feature_loader =torch.utils.data.DataLoader(dataset_new,
                                    batch_size = 512, shuffle = False)
        return feature_loader
    return dataset_new

def create_feature_loader(model, data, file_path, batch_size = 512, shuffle= False):
    if os.path.isfile(file_path):
        print("loading from dataset")
        feature_dataset = torch.load(file_path)
    else:
        print("reconstruct")
        feature_dataset = model.get_feature_dataset(data, file_path)

    feature_loader =torch.utils.data.DataLoader(feature_dataset,
                                    batch_size=batch_size, shuffle = shuffle)
    return feature_loader


'''
Created on 2017��3��23��

@author: gb
'''
import sys
import os
import jieba
import gensim,logging
import sys

def savefile(savepath,content):
     fp=open(savepath,"w")
     fp.write(content)
     fp.close()
def readfile(path):
    fp=open(path,"r")
    content=fp.read()
    fp.close()
    return content

corpus_path="train_corpus_small/"
seg_path="train_corpus_seg/"
catelist=os.listdir(corpus_path)
print(catelist)
for mydir in catelist:
    class_path=corpus_path+mydir+"/"
    seg_dir=seg_path+mydir+'/'
    if not os.path.exists(seg_dir):
        os.makedirs(seg_dir)
    file_list=os.listdir(class_path)
    for file_path in file_list:
        fullname=class_path+file_path

        content=readfile(fullname).strip()
        content=content.replace("\r\n","").strip()
        content_seg=jieba.cut(content)

        savefile(seg_dir+file_path," ".join(content_seg))

print("jieba end")


# ported from uniborg thanks to @s_n_a_p_s , @r4v4n4 ,  @spechide and @PhycoNinja13b
#:::::Credit Time::::::
# 1) Coded By: @s_n_a_p_s
# 2) Ported By: @r4v4n4 (Noodz Lober)
# 3) End Game Help By: @spechide
# 4) Better Colour Profile Pic By @PhycoNinja13b

import asyncio
import base64
import os
import random
import shutil
import time
from datetime import datetime

from PIL import Image, ImageDraw, ImageFont
from pySmartDL import SmartDL
from telethon.errors import FloodWaitError
from telethon.tl import functions

from . import AUTONAME, BOTLOG, BOTLOG_CHATID, DEFAULT_BIO
from .sql_helper.globals import addgvar, delgvar, gvarstatus

DEFAULTUSERBIO = DEFAULT_BIO or " рЌ»рЌЕрЈєрјбрЈєрЉјрЈђ рЈърЈєрЈдрЌ┤ рјбрЈєрЌ░рЌ┤  "
CHANGE_TIME = Config.CHANGE_TIME
DEFAULTUSER = AUTONAME or Config.ALIVE_NAME

FONT_FILE_TO_USE = "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf"

autopic_path = os.path.join(os.getcwd(), "userbot", "original_pic.png")
digitalpic_path = os.path.join(os.getcwd(), "userbot", "digital_pic.png")
autophoto_path = os.path.join(os.getcwd(), "userbot", "photo_pfp.png")

digitalpfp = Config.DIGITAL_PIC or "https://telegra.ph/file/aeaebe33b1f3988a0b690.jpg"

@bot.on(admin_cmd(pattern="autopic ?(.*)"))
async def autopic(event):
    if event.fwd_from:
        return
    if Config.DEFAULT_PIC is None:
        return await edit_delete(
            event,
            "**Error**\nFor functing of autopic you need to set DEFAULT_PIC var in Heroku vars",
            parse_mode=parse_pre,
        )
    downloader = SmartDL(Config.DEFAULT_PIC, autopic_path, progress_bar=False)
    downloader.start(blocking=False)
    while not downloader.isFinished():
        pass
    input_str = event.pattern_match.group(1)
    if input_str:
        try:
            input_str = int(input_str)
        except ValueError:
            input_str = 60
    else:
        if gvarstatus("autopic_counter") is None:
            addgvar("autopic_counter", 30)
    if gvarstatus("autopic") is not None and gvarstatus("autopic") == "true":
        return await edit_delete(event, f"`Autopic is already enabled`")
    addgvar("autopic", True)
    if input_str:
        addgvar("autopic_counter", input_str)
    await edit_delete(event, f"`Autopic has been started by my Master`")
    await autopicloop()

@bot.on(admin_cmd(pattern="digitalpfp$"))
async def main(event):
    if event.fwd_from:
        return
    downloader = SmartDL(digitalpfp, digitalpic_path, progress_bar=False)
    downloader.start(blocking=False)
    while not downloader.isFinished():
        pass
    if gvarstatus("digitalpic") is not None and gvarstatus("digitalpic") == "true":
        return await edit_delete(event, f"`Digitalpic is already enabled`")
    addgvar("digitalpic", True)
    await edit_delete(event, f"`digitalpfp has been started by my Master`")
    await digitalpicloop()

@bot.on(admin_cmd(pattern="bloom$"))
async def autopic(event):
    if event.fwd_from:
        return
    if Config.DEFAULT_PIC is None:
        return await edit_delete(
            event,
            "**Error**\nFor functing of bloom you need to set DEFAULT_PIC var in Heroku vars",
            parse_mode=parse_pre,
        )
    downloader = SmartDL(Config.DEFAULT_PIC, autopic_path, progress_bar=True)
    downloader.start(blocking=False)
    while not downloader.isFinished():
        pass
    if gvarstatus("bloom") is not None and gvarstatus("bloom") == "true":
        return await edit_delete(event, f"`Bloom is already enabled`")
    addgvar("bloom", True)
    await edit_delete(event, f"`Bloom has been started by my Master`")
    await bloom_pfploop()

@bot.on(admin_cmd(pattern="autoname$"))
async def _(event):
    if event.fwd_from:
        return
    if gvarstatus("autoname") is not None and gvarstatus("autoname") == "true":
        return await edit_delete(event, f"`Autoname is already enabled`")
    addgvar("autoname", True)
    await edit_delete(event, "`AutoName has been started by my Master `")
    await autoname_loop()

@bot.on(admin_cmd(pattern="autobio$"))
async def _(event):
    if event.fwd_from:
        return
    if gvarstatus("autobio") is not None and gvarstatus("autobio") == "true":
        return await edit_delete(event, f"`Autobio is already enabled`")
    addgvar("autobio", True)
    await edit_delete(event, "`Autobio has been started by my Master `")
    await autobio_loop()

@bot.on(admin_cmd(pattern="end (.*)"))
async def _(event):  # sourcery no-metrics
    if event.fwd_from:
        return
    input_str = event.pattern_match.group(1)
    if input_str == "autopic":
        if gvarstatus("autopic") is not None and gvarstatus("autopic") == "true":
            delgvar("autopic")
            if os.path.exists(autopic_path):
                file = await event.client.upload_file(autopic_path)
                try:
                    await event.client(functions.photos.UploadProfilePhotoRequest(file))
                    os.remove(autopic_path)
                except BaseException:
                    return
            return await edit_delete(event, "`Autopic has been stopped now`")
        return await edit_delete(event, "`Autopic haven't enabled`")
    if input_str == "digitalpfp":
        if gvarstatus("digitalpic") is not None and gvarstatus("digitalpic") == "true":
            delgvar("digitalpic")
            await event.client(
                functions.photos.DeletePhotosRequest(
                    await bot.get_profile_photos("me", limit=1)
                )
            )
            return await edit_delete(event, "`Digitalpfp has been stopped now`")
        return await edit_delete(event, "`Digitalpfp haven't enabled`")
    if input_str == "bloom":
        if gvarstatus("bloom") is not None and gvarstatus("bloom") == "true":
            delgvar("bloom")
            if os.path.exists(autopic_path):
                file = await event.client.upload_file(autopic_path)
                try:
                    await event.client(functions.photos.UploadProfilePhotoRequest(file))
                    os.remove(autopic_path)
                except BaseException:
                    return
            return await edit_delete(event, "`Bloom has been stopped now`")
        return await edit_delete(event, "`Bloom haven't enabled`")
    if input_str == "autoname":
        if gvarstatus("autoname") is not None and gvarstatus("autoname") == "true":
            delgvar("autoname")
            await event.client(
                functions.account.UpdateProfileRequest(first_name=DEFAULTUSER)
            )
            return await edit_delete(event, "`Autoname has been stopped now`")
        return await edit_delete(event, "`Autoname haven't enabled`")
    if input_str == "autobio":
        if gvarstatus("autobio") is not None and gvarstatus("autobio") == "true":
            delgvar("autobio")
            await event.client(
                functions.account.UpdateProfileRequest(about=DEFAULTUSERBIO)
            )
            return await edit_delete(event, "`Autobio has been stopped now`")
        return await edit_delete(event, "`Autobio haven't enabled`")

async def autopicloop():
    AUTOPICSTART = gvarstatus("autopic") == "true"
    if AUTOPICSTART and Config.DEFAULT_PIC is None:
        if BOTLOG:
            return await bot.send_message(
                BOTLOG_CHATID,
                "**Error**\n`For functing of autopic you need to set DEFAULT_PIC var in Heroku vars`",
            )
        return
    if gvarstatus("autopic") is not None:
        try:
            counter = int(gvarstatus("autopic_counter"))
        except Exception as e:
            LOGS.warn(str(e))
    while AUTOPICSTART:
        if not os.path.exists(autopic_path):
            downloader = SmartDL(Config.DEFAULT_PIC, autopic_path, progress_bar=False)
            downloader.start(blocking=False)
            while not downloader.isFinished():
                pass
        shutil.copy(autopic_path, autophoto_path)
        im = Image.open(autophoto_path)
        file_test = im.rotate(counter, expand=False).save(autophoto_path, "PNG")
        current_time = datetime.now().strftime("  Time: %H:%M \n  Date: %d.%m.%y ")
        img = Image.open(autophoto_path)
        drawn_text = ImageDraw.Draw(img)
        fnt = ImageFont.truetype(FONT_FILE_TO_USE, 30)
        drawn_text.text((150, 250), current_time, font=fnt, fill=(124, 252, 0))
        img.save(autophoto_path)
        file = await bot.upload_file(autophoto_path)
        try:
            await bot(functions.photos.UploadProfilePhotoRequest(file))
            os.remove(autophoto_path)
            counter += counter
            await asyncio.sleep(CHANGE_TIME)
        except BaseException:
            return
        AUTOPICSTART = gvarstatus("autopic") == "true"

async def digitalpicloop():
    DIGITALPICSTART = gvarstatus("digitalpic") == "true"
    i = 0
    while DIGITALPICSTART:
        if not os.path.exists(digitalpic_path):
            downloader = SmartDL(digitalpfp, digitalpic_path, progress_bar=False)
            downloader.start(blocking=False)
            while not downloader.isFinished():
                pass
        shutil.copy(digitalpic_path, autophoto_path)
        Image.open(autophoto_path)
        current_time = datetime.now().strftime("%H:%M")
        img = Image.open(autophoto_path)
        drawn_text = ImageDraw.Draw(img)
        cat = str(base64.b64decode("dXNlcmJvdC9oZWxwZXJzL3N0eWxlcy9kaWdpdGFsLnR0Zg=="))[
            2:36
        ]
        fnt = ImageFont.truetype(cat, 200)
        drawn_text.text((350, 100), current_time, font=fnt, fill=(124, 252, 0))
        img.save(autophoto_path)
        file = await bot.upload_file(autophoto_path)
        try:
            if i > 0:
                await bot(
                    functions.photos.DeletePhotosRequest(
                        await bot.get_profile_photos("me", limit=1)
                    )
                )
            i += 1
            await bot(functions.photos.UploadProfilePhotoRequest(file))
            os.remove(autophoto_path)
            await asyncio.sleep(CHANGE_TIME)
        except BaseException:
            return
        DIGITALPICSTART = gvarstatus("digitalpic") == "true"

async def bloom_pfploop():
    BLOOMSTART = gvarstatus("bloom") == "true"
    if BLOOMSTART and Config.DEFAULT_PIC is None:
        if BOTLOG:
            return await bot.send_message(
                BOTLOG_CHATID,
                "**Error**\n`For functing of bloom you need to set DEFAULT_PIC var in Heroku vars`",
            )
        return
    while BLOOMSTART:
        if not os.path.exists(autopic_path):
            downloader = SmartDL(Config.DEFAULT_PIC, autopic_path, progress_bar=False)
            downloader.start(blocking=False)
            while not downloader.isFinished():
                pass
        # RIP Danger zone Here no editing here plox
        R = random.randint(0, 256)
        B = random.randint(0, 256)
        G = random.randint(0, 256)
        FR = 256 - R
        FB = 256 - B
        FG = 256 - G
        shutil.copy(autopic_path, autophoto_path)
        image = Image.open(autophoto_path)
        image.paste((R, G, B), [0, 0, image.size[0], image.size[1]])
        image.save(autophoto_path)
        current_time = datetime.now().strftime("\n Time: %H:%M:%S \n \n Date: %d/%m/%y")
        img = Image.open(autophoto_path)
        drawn_text = ImageDraw.Draw(img)
        fnt = ImageFont.truetype(FONT_FILE_TO_USE, 60)
        ofnt = ImageFont.truetype(FONT_FILE_TO_USE, 250)
        drawn_text.text((95, 250), current_time, font=fnt, fill=(FR, FG, FB))
        drawn_text.text((95, 250), "      Ъўѕ", font=ofnt, fill=(FR, FG, FB))
        img.save(autophoto_path)
        file = await bot.upload_file(autophoto_path)
        try:
            await bot(functions.photos.UploadProfilePhotoRequest(file))
            os.remove(autophoto_path)
            await asyncio.sleep(CHANGE_TIME)
        except BaseException:
            return
        BLOOMSTART = gvarstatus("bloom") == "true"

async def autoname_loop():
    AUTONAMESTART = gvarstatus("autoname") == "true"
    while AUTONAMESTART:
        DM = time.strftime("%d-%m-%y")
        HM = time.strftime("%H:%M")
        name = f"Рїџ№ИЈ {HM}||Рђ║  {DEFAULTUSER} Рђ╣||ЪЊЁ {DM}"
        LOGS.info(name)
        try:
            await bot(functions.account.UpdateProfileRequest(first_name=name))
        except FloodWaitError as ex:
            LOGS.warning(str(ex))
            await asyncio.sleep(ex.seconds)
        await asyncio.sleep(CHANGE_TIME)
        AUTONAMESTART = gvarstatus("autoname") == "true"

async def autobio_loop():
    AUTOBIOSTART = gvarstatus("autobio") == "true"
    while AUTOBIOSTART:
        DMY = time.strftime("%d.%m.%Y")
        HM = time.strftime("%H:%M:%S")
        bio = f"ЪЊЁ {DMY} | {DEFAULTUSERBIO} | Рїџ№ИЈ {HM}"
        LOGS.info(bio)
        try:
            await bot(functions.account.UpdateProfileRequest(about=bio))
        except FloodWaitError as ex:
            LOGS.warning(str(ex))
            await asyncio.sleep(ex.seconds)
        await asyncio.sleep(CHANGE_TIME)
        AUTOBIOSTART = gvarstatus("autobio") == "true"

bot.loop.create_task(autopicloop())
bot.loop.create_task(digitalpicloop())
bot.loop.create_task(bloom_pfploop())
bot.loop.create_task(autoname_loop())
bot.loop.create_task(autobio_loop())

CMD_HELP.update(
    {
        "autoprofile": """**Plugin : **`autoprofile`

Рђб  **Syntax : **`.autopic angle`
Рђб  **Function : **__Rotating image along with the time on it with given angle if no angle is given then doesnt rotate. You need to set __`DEFAULT_PIC`__ in heroku__

Рђб  **Syntax : **`.digitalpfp`
Рђб  **Function : **__Your profile pic changes to digitaltime profile picutre__

Рђб  **Syntax : **`.bloom`
Рђб  **Function : **__Random colour profile pics will be set along with time on it. You need to set__ `DEFAULT_PIC`__ in heroku__

Рђб  **Syntax : **`.autoname`
Рђб  **Function : **__for time along with name, you must set __`AUTONAME`__ in the heroku vars first for this to work__

Рђб  **Syntax : **`.autobio`
Рђб  **Function : **__for time along with your bio, Set __`DEFAULT_BIO`__ in the heroku vars first__

Рђб  **Syntax : **`.end function`
Рђб  **Function : **__To stop the given functions like autopic ,difitalpfp , bloom , autoname and autobio__

**Рџа№ИЈDISCLAIMERРџа№ИЈ**
__USING THIS PLUGIN CAN RESULT IN ACCOUNT BAN. WE ARE NOT RESPONSIBLE FOR YOUR BAN.__
"""
    }
)


# settings.py
import os
from os.path import join, dirname
from dotenv import load_dotenv

dotenv_path = join(dirname(__file__), '.env')
load_dotenv(dotenv_path)

# Accessing variables.
NUM_REQUESTS = os.getenv('NUM_REQUESTS')
URL = os.getenv('URL')
SECRET = os.getenv('SECRET')
SLEEP_SECONDS = os.getenv('SLEEP_SECONDS')

# Using variables.
print('Num Requests: ' + NUM_REQUESTS)
print('URL: ' + URL)
print('Secret: ' + SECRET)
print('Sleep Seconds: ' + SLEEP_SECONDS)


from ctypes import *
import sys
import pytest

@pytest.fixture
def dll(sofile):
    return CDLL(str(sofile), use_errno=True)

def test_char_result(dll):
    f = dll._testfunc_i_bhilfd
    f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
    f.restype = c_char
    result = f(0, 0, 0, 0, 0, 0)
    assert result == b'\x00'

def test_boolresult(dll):
    f = dll._testfunc_i_bhilfd
    f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
    f.restype = c_bool
    false_result = f(0, 0, 0, 0, 0, 0)
    assert false_result is False
    true_result = f(1, 0, 0, 0, 0, 0)
    assert true_result is True

def test_unicode_function_name(dll):
    f = dll[u'_testfunc_i_bhilfd']
    f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
    f.restype = c_int
    result = f(1, 2, 3, 4, 5.0, 6.0)
    assert result == 21

def test_truncate_python_longs(dll):
    f = dll._testfunc_i_bhilfd
    f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double]
    f.restype = c_int
    x = sys.maxint * 2
    result = f(x, x, x, x, 0, 0)
    assert result == -8

def test_convert_pointers(dll):
    f = dll.deref_LP_c_char_p
    f.restype = c_char
    f.argtypes = [POINTER(c_char_p)]
    #
    s = c_char_p(b'hello world')
    ps = pointer(s)
    assert f(ps) == b'h'
    assert f(s) == b'h'  # automatic conversion from char** to char*

################################################################

def test_call_some_args(dll):
    f = dll.my_strchr
    f.argtypes = [c_char_p]
    f.restype = c_char_p
    result = f(b"abcd", ord("b"))
    assert result == b"bcd"

def test_variadic_sum(dll):
    f = dll.variadic_sum
    f.argtypes = [c_long]
    f.restype = c_long
    result = f(3, 13, 38, 100)
    assert result == 13 + 38 + 100
    result = f(2, 13, 38)
    assert result == 13 + 38

@pytest.mark.pypy_only
def test_keepalive_buffers(monkeypatch, dll):
    import gc
    f = dll.my_strchr
    f.argtypes = [c_char_p]
    f.restype = c_char_p
    #
    orig__call_funcptr = f._call_funcptr
    def _call_funcptr(funcptr, *newargs):
        gc.collect()
        gc.collect()
        gc.collect()
        return orig__call_funcptr(funcptr, *newargs)
    monkeypatch.setattr(f, '_call_funcptr', _call_funcptr)
    #
    result = f(b"abcd", ord("b"))
    assert result == b"bcd"

def test_caching_bug_1(dll):
    # the same test as test_call_some_args, with two extra lines
    # in the middle that trigger caching in f._ptr, which then
    # makes the last two lines fail
    f = dll.my_strchr
    f.argtypes = [c_char_p, c_int]
    f.restype = c_char_p
    result = f(b"abcd", ord("b"))
    assert result == b"bcd"
    result = f(b"abcd", ord("b"), 42)
    assert result == b"bcd"

def test_argument_conversion_and_checks(dll):
    #This test is designed to check for segfaults if the wrong type of argument is passed as parameter
    strlen = dll.my_strchr
    strlen.argtypes = [c_char_p, c_int]
    strlen.restype = c_char_p
    assert strlen(b"eggs", ord("g")) == b"ggs"

    # Should raise ArgumentError, not segfault
    with pytest.raises(ArgumentError):
        strlen(0, 0)
    with pytest.raises(ArgumentError):
        strlen(False, 0)

def test_union_as_passed_value(dll):
    class UN(Union):
        _fields_ = [("x", c_short),
                    ("y", c_long)]
    dll.ret_un_func.restype = UN
    dll.ret_un_func.argtypes = [UN]
    A = UN * 2
    a = A()
    a[1].x = 33
    u = dll.ret_un_func(a[1])
    assert u.y == 33 * 10000

@pytest.mark.pypy_only
def test_cache_funcptr(dll):
    tf_b = dll.tf_b
    tf_b.restype = c_byte
    tf_b.argtypes = (c_byte,)
    assert tf_b(-126) == -42
    ptr = tf_b._ptr
    assert ptr is not None
    assert tf_b(-126) == -42
    assert tf_b._ptr is ptr

def test_custom_from_param(dll):
    class A(c_byte):
        @classmethod
        def from_param(cls, obj):
            seen.append(obj)
            return -126
    tf_b = dll.tf_b
    tf_b.restype = c_byte
    tf_b.argtypes = (c_byte,)
    tf_b.argtypes = [A]
    seen = []
    assert tf_b("yadda") == -42
    assert seen == ["yadda"]

@pytest.mark.xfail(reason="warnings are disabled")
def test_warnings(dll):
    import warnings
    warnings.simplefilter("always")
    with warnings.catch_warnings(record=True) as w:
        dll.get_an_integer()
        assert len(w) == 1
        assert issubclass(w[0].category, RuntimeWarning)
        assert "C function without declared arguments called" in str(w[0].message)

@pytest.mark.xfail
def test_errcheck(dll):
    import warnings
    def errcheck(result, func, args):
        assert result == -42
        assert type(result) is int
        arg, = args
        assert arg == -126
        assert type(arg) is int
        return result
    #
    tf_b = dll.tf_b
    tf_b.restype = c_byte
    tf_b.argtypes = (c_byte,)
    tf_b.errcheck = errcheck
    assert tf_b(-126) == -42
    del tf_b.errcheck
    with warnings.catch_warnings(record=True) as w:
        dll.get_an_integer.argtypes = []
        dll.get_an_integer()
        assert len(w) == 1
        assert issubclass(w[0].category, RuntimeWarning)
        assert "C function without declared return type called" in str(w[0].message)

    with warnings.catch_warnings(record=True) as w:
        dll.get_an_integer.restype = None
        dll.get_an_integer()
        assert len(w) == 0

    warnings.resetwarnings()

def test_errno(dll):
    test_errno = dll.test_errno
    test_errno.restype = c_int
    set_errno(42)
    res = test_errno()
    n = get_errno()
    assert (res, n) == (42, 43)
    set_errno(0)
    assert get_errno() == 0

def test_issue1655(dll):
    def ret_list_p(icount):
        def sz_array_p(obj, func, args):
            assert ('.LP_c_int object' in repr(obj) or
                    '.LP_c_long object' in repr(obj))
            assert repr(args) in ("('testing!', c_int(4))",
                                    "('testing!', c_long(4))")
            assert args[icount].value == 4
            return [obj[i] for i in range(args[icount].value)]
        return sz_array_p

    get_data_prototype = CFUNCTYPE(POINTER(c_int),
                                    c_char_p, POINTER(c_int))
    get_data_paramflag = ((1,), (2,))
    get_data_signature = ('test_issue1655', dll)

    get_data = get_data_prototype(get_data_signature, get_data_paramflag)
    assert get_data(b'testing!') == 4

    get_data.errcheck = ret_list_p(1)
    assert get_data(b'testing!') == [-1, -2, -3, -4]

def test_issue2533(tmpdir):
    import cffi
    ffi = cffi.FFI()
    ffi.cdef("int **fetchme(void);")
    ffi.set_source("_x_cffi", """
        int **fetchme(void)
        {
            static int a = 42;
            static int *pa = &a;
            return &pa;
        }
    """)
    ffi.compile(verbose=True, tmpdir=str(tmpdir))

    import sys
    sys.path.insert(0, str(tmpdir))
    try:
        from _x_cffi import ffi, lib
    finally:
        sys.path.pop(0)
    fetchme = ffi.addressof(lib, 'fetchme')
    fetchme = int(ffi.cast("intptr_t", fetchme))

    FN = CFUNCTYPE(POINTER(POINTER(c_int)))
    ff = cast(fetchme, FN)

    g = ff()
    assert g.contents.contents.value == 42

    h = c_int(43)
    g[0] = pointer(h)     # used to crash here
    assert g.contents.contents.value == 43


from utils.rabbit_controller import RabbitMqController
from utils.tools import Tools
from utils.verifier import Verifier
from utils.watcher import Watcher

if __name__ == "__main__":
    """
        This is the SAAS Client!
        Results will be under the server http://3.73.75.114:5000/

        1. Get all md5 of /bin
        2. Watch changes in loop
        3. Every change get list of events
        4. Go over the event- find the file
        5. Check if the file changed
        6. Check if its cryptominer -> by strings/ filename.
        7. Check if the file is in a md5 database of malicious feed
        8. do ps -a
        9. Check if there is a reverse_shell
        10. do tcpdump, check if there is malicious ip / domain
        11. If one of the above is true: Write it to the screen.
        12. EVERY 1 MIN add cpu usage for a map of ps. Check if something is weird (cryptominer). After 30min delete
            last one.
    """
    verifier = Verifier()
    tools = Tools()
    watcher = Watcher()
    files_dict = tools.get_md5()
    resources = []
    alerts = []
    while True:
        controller = RabbitMqController()
        alerts.extend(verifier.verify_malware_dict(files_dict, is_send_to_rabbit=True))
        for alert in alerts:
            controller.send_alert(alert)
        alerts = []
        controller.connection.close()
        events = watcher.watch()
        alert_from_verifier, resources = verifier.verify_resources(resources, is_send_to_rabbit=True)
        alerts.extend(alert_from_verifier)
        alerts.extend(verifier.verify_filesystem_event(events, is_send_to_rabbit=True))
        alerts.extend(verifier.verify_cryptominer(events, is_send_to_rabbit=True))
        alerts.extend(verifier.verify_reverse_shell(events, is_send_to_rabbit=True))
        alerts.extend(verifier.verify_request(is_send_to_rabbit=True))
        files_dict = tools.get_md5()


"""
This script shows how pedestrian detections and robot data can be converted to a spatio-temporal grid

The output data of this script can then be used to train a CoPA-Map model
"""

import pandas as pd
from copa_map.model.Gridifier import Gridifier, GridParams
from copa_map.model.InitInducing import InducingInitializer
from copa_map.util.occ_grid import OccGrid
from copa_map.util import util as ut
from copa_map.util import fov
from os.path import join, exists
from copy import copy
import zipfile
data_folder = join(ut.abs_path(), "data")

csv_names = ["atc_10days_path_pedest_train.csv",
             "atc_10days_path_robot_train.csv",
             "atc_4days_path_pedest_test.csv",
             "atc_4days_path_robot_test.csv"]

# Extract the csv files from the zipped file
all_csv_exist = all([exists(join(data_folder, name)) for name in csv_names])
if not all_csv_exist:
    print("Extracting csv files from zip file...")
    with zipfile.ZipFile(join(data_folder, 'atc_rob_path_ped_detections.zip'), 'r') as zip_ref:
        zip_ref.extractall(data_folder)
a = 1

# Create pd dataframes from csv files
# The csv files with pedestrian data (*_pedest_*) will be converted to a dataframe of the following form:
# [pos_x, pos_y, tidx_bin, t] x [n_det]
# pos_x: First spatial dim of detected pedestrian
# pos_y: Second spatial dim of detected pedestrian
# tidx_bin: The bin where the detection falls into. The data is already pre binned to a bin size of one hour
# t: timestamp of the detection

# The csv files with robot data (*_robot_*) will be converted to a dataframe of the following form:
# [robot_x, robot_y, delta_t, tidx_bin, t] x [n_rob]
# The dataframe then contains the positions of the robot
# robot_x: First spatial dim of the robot
# robot_y: Second spatial dim of the robot
# delta_t: Dwell/rest time of the robot at the corresponding spatial location
# tidx_bin: The bin where the robot position falls into. The data is already pre binned to a bin size of one hour
# t: timestamp of the robot pose
df_data_train = pd.read_csv(join(data_folder, csv_names[0]), index_col=0)
df_rob_train = pd.read_csv(join(data_folder, csv_names[1]), index_col=0)

# Also create for test data, since we only want to test at locations that were visited during training
# The values of the robot's poses correspond to the values from the training data, but the timestamps were adjusted
df_data_test = pd.read_csv(join(data_folder, csv_names[2]), index_col=0)
df_rob_test = pd.read_csv(join(data_folder, csv_names[3]), index_col=0)

# Read the occupancy map to define the location of the grid
occ_map = OccGrid.from_ros_format(path_yaml=join(data_folder, "atc_map.yaml"))

# Overwrite default params of the grid with these params
# For all default params, see class definition of GridParams class
params_grid_train = GridParams(cell_resolution=0.5,
                               origin=occ_map.orig, rotation=occ_map.rotation, width=occ_map.width,
                               height=occ_map.height, rate_min=1e-5, bin_size=3600)
params_grid_test = copy(params_grid_train)

print("Creating grid for training data")
gf_train = Gridifier(occ_map=occ_map, fov=fov.Circle(r=3.5), params=params_grid_train)
gf_train.setup_data(df_data_train, df_rob_train)

print("Creating grid for test data")
gf_test = Gridifier(occ_map=occ_map, fov=fov.Circle(r=3.5), params=params_grid_test, create_gt=True)
gf_test.setup_data(df_data_test, df_rob_test)

# Create the inducing points by clustering
# There are two methods implemented:
# 3D-KMeans: Clustering over the complete input matrix X with targets as weights
# 2D-KMeans: Do separate clustering steps for each time bin. The number of clusters for every bin follows from
#            (number of spatial cells of the bin)/(number of all datapoints) * (number of all inducing points)
print("Clustering for initial inducing point selection...")
init_inducing = InducingInitializer(X=gf_train.get_input_points(), Y_all=gf_train.get_observations(), alpha=0.02)
init_inducing.get_init_inducing(method="2D-KMeans")

path_train = join(data_folder, "grid_atc_50cm_60min_train_xy.csv")
path_test = join(data_folder, "grid_atc_50cm_60min_test_xy.csv")
path_inducing = join(data_folder, "grid_atc_50cm_60min_train_z.csv")

print("Saving training data to " + str(path_train))
gf_train.output_to_text(path_train)

print("Saving inducing point data to " + str(path_inducing))
init_inducing.output_to_text(path_inducing)

print("Saving test data to " + str(path_test))
gf_test.output_to_text(path_test)

print("All data saved. Run 02_copa_map_atc_train.py to train the model.")

plot = True
if plot:
    print("Plotting training data")
    gf_train.plot()


import pandas as pd
import matplotlib.pyplot as plt

def gerar_grafico_tempos(caminho_entrada):
    df = pd.read_csv(caminho_entrada, delimiter=';')

    labels = ['Média', 'Mediana']
    values = [df['Média'].iloc[0], df['Mediana'].iloc[0]]

    plt.bar(labels, values, color=['blue', 'green'])

    plt.title('Média e Mediana do Tempo de Correção')
    plt.ylabel('Horas')
    for i, v in enumerate(values):
        plt.text(i, v + 5, str(v), ha='center', va='bottom', fontweight='bold')

    plt.tight_layout()
    plt.savefig('grafico_tempos_correcao.png', dpi=300)
    plt.show()

gerar_grafico_tempos("result_correction_time.csv")


# from https://github.com/iskandr/fancyimpute

import numpy as np
from sklearn.utils.extmath import randomized_svd
from sklearn.utils import check_array

import warnings

F32PREC = np.finfo(np.float32).eps

from joblib import Memory
memory = Memory('cache_dir', verbose=0)

def soft_impute_rank(X_obs, n_folds = 5, max_rank = 10):

    l_mae = []
    for ii in range(n_folds):
        obs_mask = ~np.isnan(X_obs)
        # randomly sample some test mask
        test_mask = np.array(np.random.binomial(np.ones_like(obs_mask), obs_mask * .2), dtype=bool)

        X_obs_train = X_obs.copy()
        X_obs_train[test_mask] = np.nan

        si = SoftImpute(max_rank=max_rank, verbose=False)
        X_obs_imp = si.fit_transform(X_obs_train)
        si.U
        mae_obs = si.mae_obs
        mae_test = np.mean(np.abs(X_obs[test_mask] - X_obs_imp[test_mask]))

        l_mae.append(mae_test)

    return si.U, l_mae

@memory.cache
def get_U_softimpute(X_obs, list_rank=None, boxplot=False, n_folds=3):
    """ return U_hat with SoftImput strategy

    Rank are cross selected (wrt MSE) in list_rank"""

    assert np.sum(np.isnan(X_obs)) > 0, 'X_obs do not contains any nan in "get_U_softimpute"'

    best_mae = float('inf')
    best_U = None
    best_rank = None

    list_rank = [1,2,3,4,5,6,7,8,9,10,20,30,100]
    ll_mae = []
    for max_rank in list_rank:

        U, l_mae = soft_impute_rank(X_obs, n_folds = n_folds, max_rank = max_rank)
        ll_mae.append(l_mae)

        # print(' -get_U_softimpute, rank={}, mae={} + {}'.format(max_rank, np.round(np.mean(l_mae),4), np.round(np.std(l_mae),4)))
        if np.mean(l_mae) < best_mae:
            best_mae = np.mean(l_mae)
            best_U = U
            best_rank = max_rank

    if boxplot:
        sns.swarmplot(data=np.array(ll_mae).T)
        plt.xticks(ticks = np.arange(len(list_rank)), labels=[str(x) for x in list_rank])
        plt.xlabel('SVD rank')
        plt.ylabel('MAE on test fold')
        plt.show()

    print('-get_U_softimpute, best_rank=',best_rank)
    print('-get_U_softimpute, best_mae=',best_mae)
    return best_U

def masked_mae(X_true, X_pred, mask):
    masked_diff = X_true[mask] - X_pred[mask]
    return np.mean(np.abs(masked_diff))

def generate_random_column_samples(column):
    col_mask = np.isnan(column)
    n_missing = np.sum(col_mask)
    if n_missing == len(column):
        # logging.warn("No observed values in column")
        return np.zeros_like(column)

    mean = np.nanmean(column)
    std = np.nanstd(column)

    if np.isclose(std, 0):
        return np.array([mean] * n_missing)
    else:
        return np.random.randn(n_missing) * std + mean

class Solver(object):
    def __init__(
            self,
            fill_method="zero",
            min_value=None,
            max_value=None,
            normalizer=None):
        self.fill_method = fill_method
        self.min_value = min_value
        self.max_value = max_value
        self.normalizer = normalizer

    def __repr__(self):
        return str(self)

    def __str__(self):
        field_list = []
        for (k, v) in sorted(self.__dict__.items()):
            if v is None or isinstance(v, (float, int)):
                field_list.append("%s=%s" % (k, v))
            elif isinstance(v, str):
                field_list.append("%s='%s'" % (k, v))
        return "%s(%s)" % (
            self.__class__.__name__,
            ", ".join(field_list))

    def _check_input(self, X):
        if len(X.shape) != 2:
            raise ValueError("Expected 2d matrix, got %s array" % (X.shape,))

    def _check_missing_value_mask(self, missing):
        if not missing.any():
            warnings.simplefilter("always")
            warnings.warn("Input matrix is not missing any values")
        if missing.all():
            raise ValueError("Input matrix must have some non-missing values")

    def _fill_columns_with_fn(self, X, missing_mask, col_fn):
        for col_idx in range(X.shape[1]):
            missing_col = missing_mask[:, col_idx]
            n_missing = missing_col.sum()
            if n_missing == 0:
                continue
            col_data = X[:, col_idx]
            fill_values = col_fn(col_data)
            if np.all(np.isnan(fill_values)):
                fill_values = 0
            X[missing_col, col_idx] = fill_values

    def fill(
            self,
            X,
            missing_mask,
            fill_method=None,
            inplace=False):
        """
        Parameters
        ----------
        X : np.array
            Data array containing NaN entries
        missing_mask : np.array
            Boolean array indicating where NaN entries are
        fill_method : str
            "zero": fill missing entries with zeros
            "mean": fill with column means
            "median" : fill with column medians
            "min": fill with min value per column
            "random": fill with gaussian samples according to mean/std of column
        inplace : bool
            Modify matrix or fill a copy
        """
        X = check_array(X, force_all_finite=False)

        if not inplace:
            X = X.copy()

        if not fill_method:
            fill_method = self.fill_method

        if fill_method not in ("zero", "mean", "median", "min", "random"):
            raise ValueError("Invalid fill method: '%s'" % (fill_method))
        elif fill_method == "zero":
            # replace NaN's with 0
            X[missing_mask] = 0
        elif fill_method == "mean":
            self._fill_columns_with_fn(X, missing_mask, np.nanmean)
        elif fill_method == "median":
            self._fill_columns_with_fn(X, missing_mask, np.nanmedian)
        elif fill_method == "min":
            self._fill_columns_with_fn(X, missing_mask, np.nanmin)
        elif fill_method == "random":
            self._fill_columns_with_fn(
                X,
                missing_mask,
                col_fn=generate_random_column_samples)
        return X

    def prepare_input_data(self, X):
        """
        Check to make sure that the input matrix and its mask of missing
        values are valid. Returns X and missing mask.
        """
        X = check_array(X, force_all_finite=False)
        if X.dtype != "f" and X.dtype != "d":
            X = X.astype(float)

        self._check_input(X)
        missing_mask = np.isnan(X)
        self._check_missing_value_mask(missing_mask)
        return X, missing_mask

    def clip(self, X):
        """
        Clip values to fall within any global or column-wise min/max constraints
        """
        X = np.asarray(X)
        if self.min_value is not None:
            X[X < self.min_value] = self.min_value
        if self.max_value is not None:
            X[X > self.max_value] = self.max_value
        return X

    def project_result(self, X):
        """
        First undo normalization and then clip to the user-specified min/max
        range.
        """
        X = np.asarray(X)
        if self.normalizer is not None:
            X = self.normalizer.inverse_transform(X)
        return self.clip(X)

    def solve(self, X, missing_mask):
        """
        Given an initialized matrix X and a mask of where its missing values
        had been, return a completion of X.
        """
        raise ValueError("%s.solve not yet implemented!" % (
            self.__class__.__name__,))

    def fit_transform(self, X, y=None):
        """
        Fit the imputer and then transform input `X`
        Note: all imputations should have a `fit_transform` method,
        but only some (like IterativeImputer in sklearn) also support inductive
        mode using `fit` or `fit_transform` on `X_train` and then `transform`
        on new `X_test`.
        """
        X_original, missing_mask = self.prepare_input_data(X)
        observed_mask = ~missing_mask
        X = X_original.copy()
        if self.normalizer is not None:
            X = self.normalizer.fit_transform(X)
        X_filled = self.fill(X, missing_mask, inplace=True)
        if not isinstance(X_filled, np.ndarray):
            raise TypeError(
                "Expected %s.fill() to return NumPy array but got %s" % (
                    self.__class__.__name__,
                    type(X_filled)))

        X_result = self.solve(X_filled, missing_mask)
        if not isinstance(X_result, np.ndarray):
            raise TypeError(
                "Expected %s.solve() to return NumPy array but got %s" % (
                    self.__class__.__name__,
                    type(X_result)))

        X_result = self.project_result(X=X_result)
        X_result[observed_mask] = X_original[observed_mask]
        return X_result

    def fit(self, X, y=None):
        """
        Fit the imputer on input `X`.
        Note: all imputations should have a `fit_transform` method,
        but only some (like IterativeImputer in sklearn) also support inductive
        mode using `fit` or `fit_transform` on `X_train` and then `transform`
        on new `X_test`.
        """
        raise ValueError(
            "%s.fit not implemented! This imputation algorithm likely "
            "doesn't support inductive mode. Only fit_transform is "
            "supported at this time." % (
                self.__class__.__name__,))

    def transform(self, X, y=None):
        """
        Transform input `X`.
        Note: all imputations should have a `fit_transform` method,
        but only some (like IterativeImputer in sklearn) also support inductive
        mode using `fit` or `fit_transform` on `X_train` and then `transform`
        on new `X_test`.
        """
        raise ValueError(
            "%s.transform not implemented! This imputation algorithm likely "
            "doesn't support inductive mode. Only %s.fit_transform is "
            "supported at this time." % (
                self.__class__.__name__, self.__class__.__name__))

class SoftImpute(Solver):
    """
    Implementation of the SoftImpute algorithm from:
    "Spectral Regularization Algorithms for Learning Large Incomplete Matrices"
    by Mazumder, Hastie, and Tibshirani.
    """
    def __init__(
            self,
            shrinkage_value=None,
            convergence_threshold=0.001,
            max_iters=100,
            max_rank=None,
            n_power_iterations=1,
            init_fill_method="zero",
            min_value=None,
            max_value=None,
            normalizer=None,
            verbose=True):
        """
        Parameters
        ----------
        shrinkage_value : float
            Value by which we shrink singular values on each iteration. If
            omitted then the default value will be the maximum singular
            value of the initialized matrix (zeros for missing values) divided
            by 100.

        convergence_threshold : float
            Minimum ration difference between iterations (as a fraction of
            the Frobenius norm of the current solution) before stopping.

        max_iters : int
            Maximum number of SVD iterations

        max_rank : int, optional
            Perform a truncated SVD on each iteration with this value as its
            rank.

        n_power_iterations : int
            Number of power iterations to perform with randomized SVD

        init_fill_method : str
            How to initialize missing values of data matrix, default is
            to fill them with zeros.

        min_value : float
            Smallest allowable value in the solution

        max_value : float
            Largest allowable value in the solution

        normalizer : object
            Any object (such as BiScaler) with fit() and transform() methods

        verbose : bool
            Print debugging info
        """
        Solver.__init__(
            self,
            fill_method=init_fill_method,
            min_value=min_value,
            max_value=max_value,
            normalizer=normalizer)
        self.shrinkage_value = shrinkage_value
        self.convergence_threshold = convergence_threshold
        self.max_iters = max_iters
        self.max_rank = max_rank
        self.n_power_iterations = n_power_iterations
        self.verbose = verbose

        self.U = None
        self.V = None
        self.S = None
        self.mae_obs = None

    def _converged(self, X_old, X_new, missing_mask):
        # check for convergence
        old_missing_values = X_old[missing_mask]
        new_missing_values = X_new[missing_mask]
        difference = old_missing_values - new_missing_values
        ssd = np.sum(difference ** 2)
        old_norm = np.sqrt((old_missing_values ** 2).sum())
        # edge cases
        if old_norm == 0 or (old_norm < F32PREC and np.sqrt(ssd) > F32PREC):
            return False
        else:
            return (np.sqrt(ssd) / old_norm) < self.convergence_threshold

    def _svd_step(self, X, shrinkage_value, max_rank=None):
        """
        Returns reconstructed X from low-rank thresholded SVD and
        the rank achieved.
        """
        if max_rank:
            # if we have a max rank then perform the faster randomized SVD
            (U, s, V) = randomized_svd(
                X,
                max_rank,
                n_iter=self.n_power_iterations)
        else:
            # perform a full rank SVD using ARPACK
            (U, s, V) = np.linalg.svd(
                X,
                full_matrices=False,
                compute_uv=True)
        s_thresh = np.maximum(s - shrinkage_value, 0)
        rank = (s_thresh > 0).sum()
        s_thresh = s_thresh[:rank]
        U_thresh = U[:, :rank]
        V_thresh = V[:rank, :]
        S_thresh = np.diag(s_thresh)
        X_reconstruction = np.dot(U_thresh, np.dot(S_thresh, V_thresh))

        self.U, self.S, self.V = U_thresh, S_thresh, V_thresh
        return X_reconstruction, rank

    def _max_singular_value(self, X_filled):
        # quick decomposition of X_filled into rank-1 SVD
        _, s, _ = randomized_svd(
            X_filled,
            1,
            n_iter=5)
        return s[0]

    def solve(self, X, missing_mask):
        X = check_array(X, force_all_finite=False)

        X_init = X.copy()

        X_filled = X
        observed_mask = ~missing_mask
        max_singular_value = self._max_singular_value(X_filled)
        if self.verbose:
            print("[SoftImpute] Max Singular Value of X_init = %f" % (
                max_singular_value))

        if self.shrinkage_value:
            shrinkage_value = self.shrinkage_value
        else:
            # totally hackish heuristic: keep only components
            # with at least 1/50th the max singular value
            shrinkage_value = max_singular_value / 50.0

        for i in range(self.max_iters):
            X_reconstruction, rank = self._svd_step(
                X_filled,
                shrinkage_value,
                max_rank=self.max_rank)
            X_reconstruction = self.clip(X_reconstruction)

            # print error on observed data
            if self.verbose:
                mae = masked_mae(
                    X_true=X_init,
                    X_pred=X_reconstruction,
                    mask=observed_mask)
                print(
                    "[SoftImpute] Iter %d: observed MAE=%0.6f rank=%d" % (
                        i + 1,
                        mae,
                        rank))

            converged = self._converged(
                X_old=X_filled,
                X_new=X_reconstruction,
                missing_mask=missing_mask)
            X_filled[missing_mask] = X_reconstruction[missing_mask]
            if converged:
                break
        if self.verbose:
            print("[SoftImpute] Stopped after iteration %d for lambda=%f" % (
                i + 1,
                shrinkage_value))

        self.mae_obs = masked_mae(
                    X_true=X_init,
                    X_pred=X_reconstruction,
                    mask=observed_mask)

        return X_filled

if __name__=='__main__':

    from generate_data import gen_lrmf, gen_dlvm
    from generate_data import ampute
    import matplotlib.pyplot as plt
    import seaborn as sns

    Z, X, w, y, ps = gen_lrmf(d=3)
    X_obs = ampute(X)

    print('boxplot of get_U_softimpute with gen_lrmf(d=3)')
    U = get_U_softimpute(X_obs, boxplot=True)


import pytest
import logging
import json
from work_order_tests.work_order_tests import work_order_get_result_params, \
    work_order_request_params
from automation_framework.work_order_submit.work_order_submit_utility \
    import verify_work_order_signature, decrypt_work_order_response
from automation_framework.utilities.request_args import TestStep
from automation_framework.utilities.workflow import validate_response_code

logger = logging.getLogger(__name__)

def test_work_order_both_in_out_Data_DataEncryptionKey_null_echo(setup_config):
    """ Testing work order request by passing
     null in encrypteddataencryption in indata or both
     in indata and outdata. """

    # input file name
    request = 'work_order_tests/input' \
              '/work_order_both_in_out_Data_EncryptionKey_null_echo.json'
    work_order_response, generic_params = work_order_request_params(
        setup_config, request)

    err_cd, work_order_get_result_response = work_order_get_result_params(
        work_order_response[:2], generic_params)

    assert (verify_work_order_signature(work_order_get_result_response,
                                        generic_params[0])
            is TestStep.SUCCESS.value)

    assert (decrypt_work_order_response(work_order_get_result_response,
                                        work_order_response[3],
                                        work_order_response[4])[0]
            is TestStep.SUCCESS.value)

    # WorkOrderGetResult API Response validation with key parameters
    assert (validate_response_code(work_order_get_result_response) is
            TestStep.SUCCESS.value)

def test_work_order_with_empty_indata_outdata(setup_config):
    """ Testing work order request by passing
     empty indata and outdata. """

    # input file name
    request = 'work_order_tests/input' \
              '/work_order_with_empty_indata_outdata.json'

    work_order_response, generic_params = (work_order_request_params
                                           (setup_config, request))
    err_cd, work_order_get_result_response = (work_order_get_result_params
                                              (work_order_response[:6],
                                               generic_params))

    # WorkOrderGetResult API Response validation with key parameters
    assert (validate_response_code(work_order_get_result_response) is
            TestStep.SUCCESS.value)