__author__ = 'M_Nour'

import numpy as np
from scipy import stats
from sklearn.semi_supervised import label_propagation
from sklearn.metrics import classification_report, confusion_matrix,accuracy_score, f1_score, recall_score
import dataset
from collections import Counter
import matplotlib.pyplot as plt
from os.path import join
from dataset import calc_features
from imblearn.over_sampling import SMOTE
from sklearn import preprocessing
from sklearn.dummy import DummyClassifier
from sklearn.ensemble import RandomForestClassifier

def make_model3():
    model =  label_propagation.LabelSpreading(kernel='knn', n_neighbors=15)
    sensor_data = dataset.load_data()
    X, y = sensor_data.data[:200], sensor_data.target[:200]
    model.fit(X, y)
    np.savetxt("X.csv", X, delimiter=",", fmt='%10.5f')
    np.savetxt("y_train.csv", y, delimiter=",", fmt='%10.1f')
    return model

def make_model():
    model = RandomForestClassifier(n_estimators=185)#label_propagation.LabelSpreading(gamma=0.25, max_iter=5)
    sensor_data = dataset.load_data()
    X, y = sensor_data.data[:200], sensor_data.target[:200]
    model.fit(X, y)
    np.savetxt("X.csv", X, delimiter=",", fmt='%10.5f')
    np.savetxt("y_train.csv", y, delimiter=",", fmt='%10.1f')
    return model

def make_model2():
    sensor_data = dataset.load_data()
    rng = np.random.RandomState(0)
    indices = np.arange(len(sensor_data.data))
    rng.shuffle(indices)
    print(len(sensor_data.data))
    sm = SMOTE(random_state=42)
    X, y  = sm.fit_sample(sensor_data.data[indices[:2000]], sensor_data.target[indices[:2000]])

    n_total_samples = len(y)
    print(len(y))
    n_labeled_points = 200
    max_iterations = 50
    unlabeled_indices = np.arange(n_total_samples)[n_labeled_points:]
    lp_model = label_propagation.LabelSpreading(kernel='knn', n_neighbors=15)

    for i in range(max_iterations):
        if len(unlabeled_indices) == 0:
            print("No unlabeled items left to label.")
            break
        y_train = np.copy(y)
        y_train[unlabeled_indices] = -1
        lp_model.fit(X, y_train)
        p = lp_model.predict_proba(X[unlabeled_indices])
        # predicted_labels = [1 if x > 0.57 else 0 for x in p[:, 1]]
        predicted_labels = lp_model.predict(X[unlabeled_indices])

        true_labels = y[unlabeled_indices]
        # print("#"*20 + "Iteration :: " + str(i) + "#"*20)
        # print(classification_report(true_labels, predicted_labels))

        pred_entropies = stats.distributions.entropy(
            lp_model.label_distributions_.T)
        uncertainty_index = np.argsort(pred_entropies)[::-1]
        uncertainty_index = uncertainty_index[
                                np.in1d(uncertainty_index, unlabeled_indices)][:40]
        delete_indices = np.array([])
        for index in uncertainty_index:
            delete_index, = np.where(unlabeled_indices == index)
            delete_indices = np.concatenate((delete_indices, delete_index))
        unlabeled_indices = np.delete(unlabeled_indices, delete_indices)
        n_labeled_points += len(uncertainty_index)

    np.savetxt("X.csv", X, delimiter=",", fmt='%10.5f')
    np.savetxt("y_train.csv", y_train, delimiter=",", fmt='%10.1f')
    return lp_model

def update_data(mdl , x, aq, entropy, threshold):
    x = x.reshape(1, -1)
    p = mdl.predict_proba(x)
    e = stats.distributions.entropy(p.T)
    entropy =np.append(entropy, e)
    if e > threshold and aq>0:
        aq-=1
    return aq,entropy

def customize_model(mdl, W_X, W_Y):
    print("@"*100)
    print(len(W_Y))
    sm = SMOTE(random_state=42)
    # mdl = label_propagation.LabelSpreading(gamma=0.25, max_iter=5)

    X = np.genfromtxt('X.csv', delimiter=',')
    y_train = np.genfromtxt('y_train.csv', delimiter=',')
    # y_train = y_train.reshape(-1,1)
    # W_Y = W_Y.reshape(-1, 1)
    # h = np.arange(len(W_Y))[[i!= -1 for i in W_Y ]]
    # W_Y = W_Y[h]
    # W_X = W_X[h]
    y_train = np.append(y_train, W_Y, axis=0)
    X = np.append(X, W_X, axis=0)
    mdl.fit(X, y_train)

    return mdl

def update_threshold_adaptive_lambda(entropy, k , N, fold_count, AR):

    threshold = sorted(entropy)[::-1][int(AR*k*fold_count/N)]
    return threshold

def update_threshold_static_lambda():
    threshold= 0.65
    return threshold

def customize(mdl, method, name, fold_count, participant_id, AR = 60):
    f_result = []
    acc_result = []
    rec_result =[]
    f_macro_result = []
    rec_macro_result = []
    threshold = 0.691
    entropy = np.array([])

    total_query = 0
    aq = AR
    user_data = np.genfromtxt(join('C:/Users/Marjan/Desktop/Thesis/participants_wild',participant_id, 'wrist_ss.csv'), delimiter=',')
    user_data = user_data[:, 1:]
    data = calc_features(user_data)
    X = preprocessing.scale(data[:, :(user_data.shape[1] - 1) * 9])
    print("*"*20)
    print(X.shape)
    X = np.tile(X, (2, 1))
    #X = np.tile(X, 10) #repeat 10 times
    print(X.shape)
    print("*" * 20)
    y = data[:, -1]
    y = np.tile(y, 2)
    print(y.shape)
    W_Y = np.copy(y)
    print(W_Y.shape)

    N = len(X)
    unlabeled_indices = np.arange(N)
    W_Y [unlabeled_indices] = -1
    # size = int(N//fold_count)
    size = 3600 / 3  # number of instances per hour
    for k, row in enumerate(X):
        if k%size ==  size-1:
            total_query+=(AR- aq)
            aq = AR

            mdl = customize_model(mdl, X[:k+1], W_Y[:k+1])
            predicted_labels = mdl.predict(X[unlabeled_indices])
            # p = mdl.predict_proba(X[unlabeled_indices])
            # predicted_labels = [1 if x > 0.50 else 0 for x in p[:, 1]]

            true_labels = y[unlabeled_indices]
            from collections import  Counter
            print(Counter(true_labels))
            print("size is :: "+ str(size))
            print("k is :: " + str(k))
            print("Iteration :: "+ str((k+1) /size)+ " Queried :: "+ str(total_query))
            print(classification_report(true_labels, predicted_labels))
            f_macro_result.append(f1_score(true_labels, predicted_labels, average='macro'))
            acc_result.append(accuracy_score(true_labels, predicted_labels))
            rec_macro_result.append(recall_score(true_labels, predicted_labels, average='macro'))
            f_result.append(f1_score(true_labels, predicted_labels, average='binary'))
            rec_result.append(recall_score(true_labels, predicted_labels, average='binary'))

        a, entropy = update_data(mdl, row, aq, entropy,threshold)
        if a < aq:
            aq = a
            unlabeled_indices = np.delete(unlabeled_indices, np.where(unlabeled_indices == k))
            W_Y[k] = y[k]
        if method.__name__ == 'update_threshold_adaptive_lambda':
            threshold = method(entropy, k, N, fold_count, AR)
        else:
            threshold = method()
           # print("%.5f" % threshold)
    np.savetxt("en.csv", sorted(entropy), delimiter=",", fmt='%10.5f')
    with open("f1_macro_score_" + name+ '_'+ participant_id + ".csv", 'wb') as afile:
        np.savetxt(afile, f_macro_result, delimiter=",", fmt='%10.5f')
    with open("accuracy_score_" + name + '_' + participant_id + ".csv", 'wb') as afile:
        np.savetxt(afile, acc_result, delimiter=",", fmt='%10.5f')

    with open("recall_macro_score_" + name + '_' + participant_id + ".csv", 'wb') as afile:
        np.savetxt(afile, rec_macro_result, delimiter=",", fmt='%10.5f')

    with open("f1_score_" + name+ '_'+ participant_id + ".csv", 'wb') as afile:
        np.savetxt(afile, f_result, delimiter=",", fmt='%10.5f')

    with open("recall_score_" + name + '_' + participant_id + ".csv", 'wb') as afile:
        np.savetxt(afile, rec_result, delimiter=",", fmt='%10.5f')

    predicted_labels = mdl.predict(X[unlabeled_indices])
    # predicted_labels = [1 if x > 0.5 else 0 for x in p[:, 1]]

    true_labels = y[unlabeled_indices]
    print(classification_report(true_labels, predicted_labels))

def depict_uncertainty(model):
    sensor_data = dataset.load_data()
    y_validate = sensor_data.test_target
    X_validate= sensor_data.test_data
    print(len(X_validate))
    p = model.predict_proba(X_validate)
    e =np.sort(stats.distributions.entropy(p.T))[::-1]
    print(e[0])
    for i in range(10):
        print(e[(i+1)*100])

    plt.scatter(e[:100], np.zeros_like(e[:100]))
    plt.ylabel("Uncertainty")
    plt.legend(loc=2)
    plt.show()

def customize_with_perfect_lambda(mdl, name, fold_count, participant_id, AR=15):
    f_result = []
    acc_result = []

    user_data = np.genfromtxt(join('C:/Users/Marjan/Desktop/Thesis/participants_wild', participant_id,'wrist_ss.csv'), delimiter=',')
    user_data = user_data[:, 1:]
    data = calc_features(user_data)

    X_n = preprocessing.scale(data[:, :(user_data.shape[1] - 1) * 9])
    y_n = data[:, -1]
    rng = np.random.RandomState(0)
    indices = np.arange(len(X_n))
    rng.shuffle(indices)
    X,y = X_n[indices[:]],y_n[indices[:]]
    W_Y = np.copy(y)
    N = len(X)
    unlabeled_indices = np.arange(N)
    W_Y[unlabeled_indices] = -1
    size = int(N // fold_count)
    for i in range(fold_count):
        current_X = np.array(X[i*size: (i+1)*size])
        # current_Y = W_Y[i*size: (i+1)*size]
        p = mdl.predict_proba(current_X)
        e = np.sort(stats.distributions.entropy(p.T))[::-1]

        # select up to 5 digit examples that the classifier is most uncertain about
        uncertainty_index = np.argsort(e)[::-1]

        uncertainty_index = uncertainty_index[
                                np.in1d(uncertainty_index, unlabeled_indices)]

        # keep track of indices that we get labels for
        delete_indices = np.array([])
        eating_counter = 0
        non_eating_counter = 0
        for index in uncertainty_index:
            if eating_counter + non_eating_counter < AR:
                delete_index, = np.where(unlabeled_indices == index)

                is_changed = False
                if y[delete_index] ==0 and non_eating_counter < int(0.5*AR):
                    non_eating_counter += 1
                    is_changed = True
                elif y[delete_index] == 1:
                    eating_counter += 1
                    is_changed = True
                if is_changed:
                    delete_indices = np.concatenate((delete_indices, delete_index))

        unlabeled_indices = np.delete(unlabeled_indices, delete_indices)

        W_Y = np.copy(y)
        W_Y[unlabeled_indices] = -1
        mdl = customize_model(mdl, X[:(i + 1) * size], W_Y[:(i + 1) * size])
        # p = mdl.predict_proba(X[unlabeled_indices])
        # predicted_labels = [1 if x > 0.50 else 0 for x in p[:, 1]]
        predicted_labels = mdl.predict(X[unlabeled_indices])
        true_labels = y[unlabeled_indices]
        print("Iteration :: " + str((i + 1)))
        print(classification_report(true_labels, predicted_labels))
        f_result.append(f1_score(true_labels, predicted_labels, average='binary'))
        acc_result.append(accuracy_score(true_labels, predicted_labels))
    with open("f1_score_"+ name + '_' + participant_id +  ".csv", 'wb') as afile:
        np.savetxt(afile, f_result, delimiter=",", fmt='%10.5f')

if __name__ == "__main__":
    # model= make_model()
    # customize(model)
    print("finished")
    participants = [2, 4, 5, 7]
    model =make_model()

    for p in participants:
        customize(model, update_threshold_adaptive_lambda, 'spal', 10, str(p))

    #depict_uncertainty(model)


#!/usr/bin/python3

from typing import List
import json
from bplib.butil import TreeNode, arr2TreeNode, btreeconnect, aprint

class Solution:
    def canAttendMeetings(self, intervals: List[List[int]]) -> bool:
        intervals = sorted(intervals)
        current_start = -1
        current_end = -1
        for [start, end] in intervals:
            if current_end > start:
                return False
            current_end = end
        return True

arr = json.loads(input())

sol = Solution()
print(sol.canAttendMeetings(arr))


from ast import Interactive
from typing import Collection
import pygame
import logging
from settings import *
from player import Player
from overlay import Overlay
from sprites import GenericSprites, WaterSprites, TreeSprites, WildFlowerSprites, InteractionSprites, ParticleEffects
from pytmx.util_pygame import load_pygame
from support import *
from transition import Transition
from soil import SoilLayer
from sky import Rain, Sky
from random import randint
from menu import Menu

class Level:
    def __init__(self):
        logging.info('The level has started')
        # get the surface for the display
        self.display_surface = pygame.display.get_surface()

        # groups for the sprites
        self.all_sprites = CameraGroup()
        # keep track of things player can collide with
        self.collision_sprites = pygame.sprite.Group()

        # need to know where the trees are relative to player
        self.tree_sprites = pygame.sprite.Group()

        # interaction with objects
        self.interaction_sprites = pygame.sprite.Group()

        # populate soil properties
        self.soil_layer = SoilLayer(self.all_sprites, self.collision_sprites)

        # start it up
        self.setup()
        self.overlay = Overlay(self.player)

        # new transition class
        self.transition = Transition(self.reset_scene, self.player)

        # sky
        self.rain = Rain(self.all_sprites)
        self.raining = randint(0, 10) > 3
        self.soil_layer.raining = self.raining
        logging.info(f'is it raining? {self.raining}')
        self.sky = Sky()

        # shop
        self.menu = Menu(self.player, self.toggle_shop)
        self.shop_active = False

    def setup(self):

        # the map was created in "Tiled" software - mapeditor.org

        # load the map file
        map_file = 'data/map.tmx'
        tmx_data = load_pygame(map_file)

        # import a ton of different things ...

        # house layers need to be separate in pygame
        # bottom (floor) -> walls -> furniture bottom -> furniture top

        # bottom layer in house - order is important, make sure floor is drawn before bottom
        for layer in ['HouseFloor', 'HouseFurnitureBottom']:
            for x, y, surf in tmx_data.get_layer_by_name(layer).tiles():
                GenericSprites((x * TILE_SIZE, y * TILE_SIZE), surf,
                               self.all_sprites, LAYERS['house bottom'])
                if (LOGGINGOPTS == 'DEBUG'):
                    logging.debug(f'layer: {layer} x: {x} y: {y}')

        for layer in ['HouseWalls', 'HouseFurnitureTop']:
            for x, y, surf in tmx_data.get_layer_by_name(layer).tiles():
                GenericSprites((x * TILE_SIZE, y * TILE_SIZE), surf,
                               self.all_sprites, LAYERS['main'])
                if (LOGGINGOPTS == 'DEBUG'):
                    logging.debug(f'layer: {layer} x: {x} y: {y}')

        # fence
        for x, y, surf in tmx_data.get_layer_by_name('Fence').tiles():
            GenericSprites((x * TILE_SIZE, y * TILE_SIZE), surf,
                           [self.all_sprites, self.collision_sprites], LAYERS['main'])
            if (LOGGINGOPTS == 'DEBUG'):
                logging.debug(f'layer: {layer} x: {x} y: {y}')

        # water
        water_folder = 'graphics/water'
        water_frames = import_folder(water_folder)
        for x, y, surf in tmx_data.get_layer_by_name('Water').tiles():
            WaterSprites((x * TILE_SIZE, y * TILE_SIZE),
                         water_frames, self.all_sprites)
            if (LOGGINGOPTS == 'DEBUG'):
                logging.debug(f'frames: {water_frames} x: {x} y: {y}')

        # wildflowers
        # the image layer is slightly different as it was created differently in "Tiled" ...
        # ... but image placement logic is the same
        for obj in tmx_data.get_layer_by_name('Decoration'):
            WildFlowerSprites((obj.x, obj.y), obj.image, [
                              self.all_sprites, self.collision_sprites])
            if (LOGGINGOPTS == 'DEBUG'):
                logging.debug(f'decoration object: {obj} x: {x} y: {y}')

        # trees
        for obj in tmx_data.get_layer_by_name('Trees'):
            TreeSprites(
                pos=(obj.x, obj.y),
                surf=obj.image,
                groups=[
                    self.all_sprites, self.collision_sprites, self.tree_sprites],
                name=obj.name,
                player_add=self.player_add)
            if (LOGGINGOPTS == 'DEBUG'):
                logging.debug(f'tree object: {obj.name} x: {x} y: {y}')

        # collision tiles - these are hard coded in the map layer, not by the objects
        for x, y, surf in tmx_data.get_layer_by_name('Collision').tiles():
            # this is not drawn or updated - just has a non-visible collision surface
            GenericSprites((x * TILE_SIZE, y * TILE_SIZE),
                           pygame.Surface((TILE_SIZE, TILE_SIZE)), self.collision_sprites)

        # player
        # check the map layer and place the player in the 'start' position
        for obj in tmx_data.get_layer_by_name('Player'):
            if obj.name == 'Start':
                self.player = Player(
                    pos=(obj.x, obj.y),
                    group=self.all_sprites,
                    collision_sprites=self.collision_sprites,
                    tree_sprites=self.tree_sprites,
                    interaction=self.interaction_sprites,
                    soil_layer=self.soil_layer,
                    toggle_shop=self.toggle_shop)
            # check map layer and see if player is in 'bed' position
            if obj.name == 'Bed':
                InteractionSprites(
                    pos=(obj.x, obj.y),
                    size=(obj.width, obj.height),
                    groups=self.interaction_sprites,
                    name=obj.name)

            # trader
            if obj.name == 'Trader':
                InteractionSprites(
                    pos=(obj.x, obj.y),
                    size=(obj.width, obj.height),
                    groups=self.interaction_sprites,
                    name=obj.name)

        # create the floor
        floor_image = 'graphics/world/ground.png'
        GenericSprites(
            pos=(0, 0),
            surf=pygame.image.load(floor_image).convert_alpha(),
            groups=self.all_sprites,
            z=LAYERS['ground'])

    def toggle_shop(self):
        self.shop_active = not self.shop_active

    def player_add(self, item):
        self.player.item_inventory[item] += 1
        logging.info(f'inventory: {self.player.item_inventory}')

    def reset_scene(self):

        # plants
        self.soil_layer.update_plant_sprites()

        # soil
        self.soil_layer.remove_water_tiles()

        # rain
        self.raining = randint(0, 10) > 3
        self.soil_layer.raining = self.raining
        if self.raining:
            logging.info(f'is it raining? {self.raining}')
            self.soil_layer.water_all()

        # add apples on trees
        for tree in self.tree_sprites.sprites():
            # look for existing fruit
            for apple in tree.apple_sprites.sprites():
                # remove existing apples
                apple.kill()
            # purged all apples above, now re-add
            tree.create_fruit()

        # reset day/night
        self.sky.start_color = [255, 255, 255]

    def plant_collision(self):
        if self.soil_layer.plant_sprites:
            for plant in self.soil_layer.plant_sprites.sprites():
                if plant.harvestable and plant.rect.colliderect(self.player.hitbox):
                    # 1 add inventory
                    # 2 remove plant
                    # 3 animations
                    # 4 reset grid attribute
                    self.player_add(plant.plant_type)
                    plant.kill()
                    ParticleEffects(plant.rect.topleft, plant.image,
                                    self.all_sprites, z=LAYERS['main'])
                    self.soil_layer.grid[plant.rect.centery //
                                         TILE_SIZE][plant.rect.centerx // TILE_SIZE].remove('P')

    def run(self, dt):
        # remove previous screens and draw new one
        self.display_surface.fill('black')
        # call the new function for the camera group
        self.all_sprites.custom_draw(self.player)

        # updates
        if self.shop_active:
            self.menu.update()
        else:
            self.all_sprites.update(dt)
            self.plant_collision()

        # weather
        self.overlay.display()
        if self.raining and not self.shop_active:
            self.rain.update()

        # day time
        self.sky.display(dt)

        # check if need to change times
        if self.player.sleep:
            self.transition.play()

class CameraGroup(pygame.sprite.Group):
    def __init__(self) -> None:
        super().__init__()
        logging.info('The CameraGroup has started')

        self.display_surface = pygame.display.get_surface()

        # needed to place the player relative to the layers
        self.offset = pygame.math.Vector2()

    def custom_draw(self, player):
        # make the ground offset to the player from the camera pov
        self.offset.x = player.rect.centerx - SCREEN_WIDTH / 2
        self.offset.y = player.rect.centery - SCREEN_HEIGHT / 2

        for layer in LAYERS.values():
            # create a lambda function to sort lower value layers to be drawn below others based on 'y'
            for sprite in sorted(self.sprites(), key=lambda sprite: sprite.rect.centery):
                if sprite.z == layer:
                    offset_rect = sprite.rect.copy()
                    offset_rect.center -= self.offset
                    # changes the sprite's drawing in a position
                    self.display_surface.blit(sprite.image, offset_rect)
                    if (LOGGINGOPTS == 'DEBUG'):
                        logging.debug(f'layer is {sprite.z}')

                    # # find location of target hit
                    if (SPRITEDEBUG=='DEBUG'):
                        if sprite == player:
                            pygame.draw.rect(self.display_surface,'red',offset_rect,5)
                            hitbox_rect = player.hitbox.copy()
                            hitbox_rect.center = offset_rect.center
                            pygame.draw.rect(self.display_surface,'green',hitbox_rect,5)
                            target_pos = offset_rect.center + PLAYER_TOOL_OFFSET[player.status.split('_')[0]]
                            pygame.draw.circle(self.display_surface,'blue',target_pos,5)


# -- coding: utf-8 --
import tensorflow as tf
import pandas as pd
import numpy as np
import pymysql
import sys
sys.path.append(sys.path.append('../'))  # 导入上一级目录中的包

from settings import *

# 查询课程信息
# 课程信息查询,例：离散数学及其应用
# 课程编号，课程名称，公司企业名称，课程编码，课程类别，学分，是否考试，上传时间，标签
def courInfo(courseName):
    db = pymysql.connect("10.134.149.182", "F3235631", "3235631", "university")
    cursor = db.cursor()
    cursor.execute(
        "select c.CourseId,c.Name,cp.Name,c.SerialKey,cs.Name,c.CreditValue,c.Exam,c.PublishDate,c.Tags from Course as c, Company as cp,CourseSerial as cs where c.Name = '%s' and c.SerialId = cs.SerialId and c.CompanyId = cp.CompanyId" % (
            courseName))
    data = cursor.fetchall()
    return data[0]

# 构建课程分析提建议所需数据
# 模型需要的数据：课程类型，学分，是否考试，课件数量，课件类型，课件时长，pdf页数
def getCourseData(courseid):
    db = pymysql.connect("10.134.149.182", "F3235631", "3235631", "university")
    cursor = db.cursor()
    cursor.execute(
        "select CourseId,SerialId,CreditValue,Exam from Course where Deleted=0 and CourseId= %d" % (courseid))
    data = list(cursor.fetchall()[0])
    cursor.execute("select count(*),Type,avg(Duration),avg(Page) from CourseWare where CourseId= %d" % (courseid))
    # data.append(list(cursor.fetchall()[0]))
    data = data + list(cursor.fetchall()[0])
    db.close()

    dic = {"s1": [0], "s2": [0], "s3": [0], "s4": [0], "s5": [0], "s6": [0], "s7": [0], "s8": [0], "s9": [0],
           "creditvalue": [data[2]],
           "exam": [data[3]],
           "warenum": [data[4]],
           "type_1": [0], "type_2": [0], "type_3": [0],
           "duration": [float(data[6])],
           "page": [float(data[7])]}
    dataDF = pd.DataFrame(dic)

    dataDF[dataDF.columns[data[1] - 1]] = 1
    if data[5] == 'pdf':
        dataDF[dataDF.columns[12]] = 1
    elif data[5] == 'mp4':
        dataDF[dataDF.columns[13]] = 1
    else:
        dataDF[dataDF.columns[14]] = 1

    return dataDF

def multilayer_perceptron(_X, _w1, _w2, _b1, _b2, _wout, _bout):
    """
    计算课程属于各个列别的可能性大小
    :param _X: 待预测课程的数据
    :param _w1: 从模型读入的参数
    :return: 课程属于各个列别的可能性大小
    """
    layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(_X, _w1), _b1))
    layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, _w2), _b2))
    return tf.matmul(layer_2, _wout) + _bout

def courPred(data, MOUDEL_PATH):
    """
    读取模型，预测课程类别
    :param data: 待预测课程的数据
    :param MOUDEL_PATH: 模型存储路径
    :return: 预测课程类别0,1,2,3，分别对应报告中课程类别一，二，三，四
             0 —— 一：需改进
             1 —— 二：较受欢迎
             2 —— 四：非常受欢迎，个例
             3 —— 三：非常受欢迎
    """
    data = np.array(data)
    data = data.astype(np.float32)

    tf.reset_default_graph()
    # 允许参数复用 or 判断是否复用了，已经有了就不再重新创建了
    # tf.get_variable_scope().reuse_variables()
    if "w1" not in vars():
        n_hidden_1 = 256
        n_hidden_2 = 128
        n_input = 17
        n_classes = 4
        w1 = tf.Variable(tf.random_normal([n_input, n_hidden_1]), name="w1"),
        w2 = tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2]), name="w2")
        b1 = tf.Variable(tf.random_normal([n_hidden_1]), name="b1")
        b2 = tf.Variable(tf.random_normal([n_hidden_2]), name="b2")
        wout = tf.Variable(tf.random_normal([n_hidden_2, n_classes]), name="wout")
        bout = tf.Variable(tf.random_normal([n_classes]), name="bout")
        saver = tf.train.Saver()
    else:
        pass

    sess = tf.Session()
    saver.restore(sess, MOUDEL_PATH)
    pred = sess.run(multilayer_perceptron(data, w1[0], w2, b1, b2, wout, bout))
    sess.close()
    return pred.argmax()

# 类别为二，三，四：受欢迎
def courWithoutAdvise():
    return "预测课程受欢迎！"

# 类别为一：需改进
# 维度：是否考试，课件类型、课件数量、时长/页数
def courAdvise(data, STD_COURSE_DATA):
    advice = '该课程仍需改进。\n'
    # 考试
    if data['exam'][0] == 0:
        advice += '课程可增加考试，提升学员对课程的参与度。\n'

    # 课件类型
    if data['type_1'][0] == 0:
        advice += "在受欢迎的课程中，有%s的课程使用了pdf类型的课件，建议课程添加pdf课件便于学员学习复习使用。\n" % (STD_COURSE_DATA['courWarePDFRate'])

    # pdf类，课件少，页码数多
    if data['warenum'][0] < STD_COURSE_DATA['courAvgWareNum'] and data['page'][0] > STD_COURSE_DATA['courAvgPage']:
        advice += "在受欢迎课程中，平均课件数量为%s，平均页码数为%s，本课程有%s的课件数量和%s的平均页码数，建议酌情减少每个课件的\
        页数，增加课件数量，便于学员使用碎片化时间学习。\n" % (
            STD_COURSE_DATA['courAvgWareNum'], STD_COURSE_DATA['courAvgPage'], data['warenum'][0], data['page'][0])

    # MP4类，课件少，平均时间长
    if data['warenum'][0] < STD_COURSE_DATA['courAvgWareNum'] and data['duration'][0] > STD_COURSE_DATA['courAvgTime']:
        advice += "在受欢迎课程中，平均课件数量为%s，视频课程平均时长为%s，本课程有%s个课件数量，每个课件平均时长%s分钟，建议酌情减少每个课件的\
        时长，增加课件数量，便于学员使用碎片化时间学习。\n" % (
            STD_COURSE_DATA['courAvgWareNum'], STD_COURSE_DATA['courAvgTime'], data['warenum'][0],
            round(data['duration'][0] / 60))

    if len(advice) < 10:
        advice = '暂无改进建议'

    return advice

def getAdvise(courseid):
    data = getCourseData(courseid)
    # 2.2 预测课程是否受欢迎
    courTypePred = courPred(data, MOUDEL_PATH)
    # 2.3 如果是受欢迎的类别or不受欢迎的类别分别调用不同函数
    if courTypePred != 0:
        advice = courWithoutAdvise()
    else:
        advice = courAdvise(data, STD_COURSE_DATA)
    return advice

if __name__ == '__main__':
    # 1、查询课程基本信息
    courBasicInfo = courInfo('离散数学及其应用')
    print(courBasicInfo)
    # 2、获得改进建议
    # 2.1 获得需要分析的课程的数据
    data = getCourseData(9477)
    # 2.2 预测课程是否受欢迎
    courTypePred = courPred(data, MOUDEL_PATH)
    # 2.3 如果是受欢迎的类别or不受欢迎的类别分别调用不同函数
    if courTypePred != 0 :
        advice = courWithoutAdvise()
    else :
        advice = courAdvise(data, STD_COURSE_DATA)
    print(advice)


__author__ = "Michael Chambers"
__copyright__ = "Copyright 2019, Michael Chambers"
__email__ = "greenkidneybean@gmail.com"
__license__ = "MIT"

from snakemake.shell import shell

log = snakemake.log_fmt_shell(stdout=False, stderr=True)

shell(
    "samtools faidx {snakemake.params} {snakemake.input[0]} > {snakemake.output[0]} {log}"
)


from keras.utils import to_categorical
import numpy as np
import pandas as pd
import time

TRAIN_SIZE = 30000
TEST_SIZE = 2

def label_generator(num_of_labels, size):
	"""
	Create a two coloum label set
	:param size1:
	:param size2:
	:return:
	"""
	lab = np.zeros((size, num_of_labels), dtype=np.float)
	length_per_label = int(size / num_of_labels)
	for i in range(num_of_labels):
		lab[(i * length_per_label):((i+1) * length_per_label), i] = 1.0

	return lab

def suffle_data_label(a, b):
	"""
	Utility function to shuffle data and labels
	:param a:
	:param b:
	:return:
	"""
	c = np.c_[a.reshape(len(a), -1), b.reshape(len(b), -1)]
	np.random.shuffle(c)
	a2 = c[:, :a.size // len(a)].reshape(a.shape)
	b2 = c[:, a.size // len(a):].reshape(b.shape)
	return a2, b2

def split_train_test(selex_data, train_size, test_size):
	"""
	split the selex inputs into train and test sets
	:param selex0:
	:param selex1:
	:param train_size:
	:return:
	"""
	split_size = len(selex_data)
	x_train = np.array([])
	x_test = np.array([])
	for selex in selex_data:
		np.random.shuffle(selex)
		if len(x_test) == 0:
			x_train = selex[0:int(train_size/split_size), :, :, :]
			x_test = selex[int(train_size / split_size)+1:int(train_size / split_size)+1+int(test_size/split_size), :, :, :]
			continue
		x_train = np.concatenate((x_train, selex[0:int(train_size/split_size), :, :, :]), axis=0)
		x_test = np.concatenate((x_test, selex[int(train_size / split_size)+1:int(train_size / split_size)+1+int(test_size/split_size), :, :, :]), axis=0)

	y_train = label_generator(split_size, train_size)
	y_test = label_generator(split_size, test_size)

	return x_train, x_test, y_train, y_test

def oneHot(string):
	"""
	One Hot encoding of a DNA sequence ACGT -> 0123 -> [1, 0, 0, 0], [0, 1, 0, 0], ...
	:param string: DNA sequence with A, C, G, or T
	:return: a matrix of the encoded sequence
	"""

	string = string.replace('A', '0')
	string = string.replace('C', '1')
	string = string.replace('G', '2')
	string = string.replace('T', '3')
	data = list(map(int, string))
	return to_categorical(data, 4)

def pbm_dataset_generator(filename):
	"""
	Open a PBM file andtransform it to a numpy encoded list
	:param filename: a PBM file with N lines of DNA sequences
	:return: an Nx4 numpy array
	"""
	f = open(filename, 'r')
	data = []
	for line in f:
		data.append(oneHot(line[0:60]))
	f.close()
	arr = np.array(data)
	return arr

def selex_dataset_generator(filename, data_to_load=TRAIN_SIZE+TEST_SIZE, selex_size=36):
	"""
	Open a selex file and transform it to two numpy lists.
	One for the onehot encoded DNA sequence
	One for the number of occurences
	:param filename:
	:return:
	"""
	t = time.time()
	f = open(filename, 'r')  # pd.read_csv(filename, delimiter='\t', usecols=[0], header=-1)
	dat = f.read().split()[0::2]
	data = []
	n = len(dat)
	idx = np.arange(n)
	# np.random.shuffle(idx)

	for i in range(data_to_load):
		encoded_line = oneHot(dat[idx[i]])
		padding = int((selex_size - len(encoded_line)) / 2)
		encoded_line = np.concatenate((np.concatenate((0.25 * np.ones((padding, 4)), encoded_line)), 0.25 * np.ones((padding, 4))))
		data.append(encoded_line)
		""" Reverse complement"""
		# encoded_line_rev = np.zeros(encoded_line.shape)
		# encoded_line_rev[:, 0] = encoded_line[:, 2]
		# encoded_line_rev[:, 1] = encoded_line[:, 3]
		# encoded_line_rev[:, 2] = encoded_line[:, 0]
		# encoded_line_rev[:, 3] = encoded_line[:, 1]
		# encoded_line_rev = encoded_line_rev[::-1]
		# data.append(encoded_line_rev)

	data = np.asarray(data)
	print('Took ', time.time() - t, ' seconds to encode data, for ', filename)
	return data, None

def save_dataset(x_train, x_test, y_train, y_test):
	import h5py
	hdf5_file = h5py.File('data_tf1.hdf5', mode='w')
	hdf5_file.create_dataset("x_train", data = x_train)
	hdf5_file.create_dataset("y_train", data = y_train)
	hdf5_file.create_dataset("x_test", data=x_test)
	hdf5_file.create_dataset("y_test", data=y_test)
	hdf5_file.close()

def load_dataset():
	import h5py
	f = h5py.File('data_tf1.hdf5', 'r')
	x_train = f["x_train"].value
	y_train = f["y_train"].value
	x_test = f["x_test"].value
	y_test = f["y_test"].value
	return x_train, x_test, y_train, y_test

def generate_data(PBM_FILE, SELEX_FILES, GENERATE_DATASET=True, train_size=100000, SELEX_SIZE=36, test_size=100000):
	"""
	Generate entire dataset
	:param PBM_FILE: PBM file name
	:param SELEX_FILES: SELEX filename list
	:param GENERATE_DATASET: Boolean to load from file dataset or not
	:param train_size: Size of output training data
	:param SELEX_SIZE: To how much to pad the selex sequence
	:param test_size: Size of output testing data
	:return:
	"""
	pbm_data = pbm_dataset_generator(PBM_FILE)
	if GENERATE_DATASET:  # load data and OneHot encode data
		print(pbm_data.shape)
		selex_4, _ = selex_dataset_generator(SELEX_FILES[-1], int((train_size+test_size)/2+1), selex_size=SELEX_SIZE)
		# selex_3, _ = selex_dataset_generator(SELEX_FILES[-2], (train_size+test_size)/5+1, selex_size=SELEX_SIZE)
		# selex_2, _ = selex_dataset_generator(SELEX_FILES[-3], (train_size+test_size)/5+1, selex_size=SELEX_SIZE)
		# selex_1, _ = selex_dataset_generator(SELEX_FILES[-4], (train_size+test_size)/5+1, selex_size=SELEX_SIZE)
		selex_0, _ = selex_dataset_generator(SELEX_FILES[0], int((train_size+test_size)/2+1), selex_size=SELEX_SIZE)

		selex_data = list()
		selex_data.append(selex_0.reshape((len(selex_0), SELEX_SIZE, 4, 1)))
		# selex_data.append(selex_1.reshape((len(selex_1), SELEX_SIZE, 4, 1)))
		# selex_data.append(selex_2.reshape((len(selex_2), SELEX_SIZE, 4, 1)))
		# selex_data.append(selex_3.reshape((len(selex_3), SELEX_SIZE, 4, 1)))
		selex_data.append(selex_4.reshape((len(selex_4), SELEX_SIZE, 4, 1)))

		x_train, x_test, y_train, y_test = split_train_test(selex_data, train_size, test_size)
		save_dataset(x_train, x_test, y_train, y_test)
	else:  # Load from data_tf1.hdf5 file
		t = time.time()
		x_train, x_test, y_train, y_test = load_dataset()
		print('Took ', time.time() - t, ' seconds to load data from h5py file')
	return x_train, x_test, y_train, y_test, pbm_data

import sys

def get_argv():
    """
    Get input from sys.argv
    :return:
    """
    if len(sys.argv) < 3:
        print("Length of input arguments is ", len(sys.argv))
        print("\nUsage:\n python main.py pbm_file SELEX_FILE_0 SELEX_FILE_1 ...")
        print("\nUsage2:\n python main.py pbm_file #of_selex_0 #of_selex_1 ...")
        sys.exit(0)

    PBM_FILE = sys.argv[1]
    SELEX_FILES = [sys.argv[i] for i in range(2, len(sys.argv))]
    if SELEX_FILES[0].isdigit():
        SELEX_FILES = list(map(int, SELEX_FILES))
    return PBM_FILE, SELEX_FILES

def parse_args(PBM_FILE, SELEX_FILES):
    """
    Transform selex numbers to filenames.
    :param PBM_FILE: Required for full path
    :param SELEX_FILES: List of numbers of SELEX cycles, or filenames
    :return: Filenames of everything
    """
    selex = SELEX_FILES
    if len(SELEX_FILES) < 1:
        return None, None
    if type(SELEX_FILES[0]) == int:
        base = PBM_FILE.split('_')[0]
        selex = [base+'_selex_'+str(i)+'.txt' for i in SELEX_FILES]

    return PBM_FILE, selex

if __name__ == '__main__':
	PBM_FILE, SELEX_FILES = 'train/TF1_pbm.txt', [0, 1, 2, 3, 4]  # get_argv()
	print(PBM_FILE)
	print(SELEX_FILES)
	# PBM_FILE, SELEX_FILES =
	PBM_FILE, SELEX_FILES = parse_args(PBM_FILE, SELEX_FILES)
	generate_data(PBM_FILE, SELEX_FILES, test_size=TEST_SIZE, train_size=TRAIN_SIZE)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

def trim_fields(apps, schema_editor):
    trim_extra_account(apps, schema_editor, "facebook")
    trim_extra_account(apps, schema_editor, "github")
    trim_extra_account(apps, schema_editor, "twitter")

def trim_extra_account(apps, schema_editor, types):
    Users = apps.get_model('accounts', 'User')
    for user in Users.objects.all():
        field = getattr(user, types)
        if not field:
            continue
        tmpString = field.split("/")[-1]
        if len(field) >= 15:
            setattr(user, types, tmpString[:14])
            user.save()

class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0003_auto_20141218_1755'),
    ]

    operations = [
        migrations.RunPython(trim_fields),
        migrations.AlterField(
            model_name='user',
            name='facebook',
            field=models.CharField(max_length=15, null=True, blank=True),
            preserve_default=True,
        ),

        migrations.AlterField(
            model_name='user',
            name='github',
            field=models.CharField(max_length=39, null=True, verbose_name='github', blank=True),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='user',
            name='twitter',
            field=models.CharField(max_length=15, null=True, blank=True),
            preserve_default=True,
        ),
    ]


import logging
import re
from datetime import datetime, timezone
from zipfile import ZipFile
import pandas

from jal.widgets.helpers import g_tr
from jal.db.update import JalDB
from jal.constants import Setup, DividendSubtype, PredefinedCategory, PredefinedAsset

# -----------------------------------------------------------------------------------------------------------------------
class UralsibCapital:
    Header = '  Брокер: ООО "УРАЛСИБ Брокер"'
    PeriodPattern = "  за период с (?P<S>\d\d\.\d\d\.\d\d\d\d) по (?P<E>\d\d\.\d\d\.\d\d\d\d)"
    DividendPattern = "> (?P<DESCR1>.*) \((?P<REG_CODE>.*)\) (?P<DESCR2>.*) налог в размере (?P<TAX>\d+\.\d\d) удержан. НДС не облагается."
    BondInterestPattern = "Погашение купона №( -?\d+)? (?P<NAME>.*)"

    def __init__(self, parent, filename):
        self._parent = parent
        self._filename = filename
        self._statement = None
        self._account_id = 0
        self._settled_cash = {}

    def load(self):
        self._settled_cash = {}
        with ZipFile(self._filename) as zip_file:
            contents = zip_file.namelist()
            if len(contents) != 1:
                logging.error(g_tr('Uralsib', "Archive contains multiple files, only one is expected for import"))
                return False
            with zip_file.open(contents[0]) as r_file:
                self._statement = pandas.read_excel(io=r_file.read(), header=None, na_filter=False)
        if not self.validate():
            return False
        self.load_cash_balance()
        self.load_stock_deals()
        self.load_cash_tranactions()         # FIXME Routing raises errors if section is absent in report
        logging.info(g_tr('Uralsib', "Uralsib Capital statement loaded; Planned cash: ")
                     + f"{self._settled_cash[self._account_id]}")
        return True

    def validate(self):
        if self._statement[2][0] != self.Header:
            logging.error(g_tr('Uralsib', "Can't find Uralsib Capital report header"))
            return False
        account_name = self._statement[2][7]
        parts = re.match(self.PeriodPattern, self._statement[2][2], re.IGNORECASE)
        if parts is None:
            logging.error(g_tr('Uralsib', "Can't parse Uralsib Capital statement period"))
            return False
        statement_dates = parts.groupdict()
        report_start = int(datetime.strptime(statement_dates['S'], "%d.%m.%Y").replace(tzinfo=timezone.utc).timestamp())
        if not self._parent.checkStatementPeriod(account_name, report_start):
            return False
        logging.info(g_tr('Uralsib', "Loading Uralsib Capital statement for account ") +
                     f"{account_name}: {statement_dates['S']} - {statement_dates['E']}")
        self._account_id = self._parent.findAccountID(account_name)
        if self._account_id is None:
            return False
        return True

    def find_section_start(self, header, subtitle, columns, header_height=2) -> (int, dict):
        header_found = False
        start_row = -1
        headers = {}
        for i, row in self._statement.iterrows():
            if not header_found and (row[0] == header):
                header_found = True
            if header_found and ((subtitle == '') or (row[0] == subtitle)):
                start_row = i + 1  # points to columns header row
                break
        if start_row > 0:
            for col in range(self._statement.shape[1]):  # Load section headers from next row
                for row in range(header_height):
                    headers[self._statement[col][start_row+row]] = col
            start_row += header_height
        column_indices = {column: headers.get(columns[column], -1) for column in columns}
        for idx in column_indices:
            if column_indices[idx] < 0:
                logging.error(g_tr('Uralsib', "Column not found: ") + idx)
                start_row = -1
        return start_row, column_indices

    def load_stock_deals(self):
        cnt = 0
        columns = {
            "number": "Номер сделки",
            "date": "Дата сделки",
            "time": "Время сделки",
            "isin": "ISIN",
            "B/S": "Вид сделки",
            "price": "Цена одной ЦБ",
            "qty": "Количество ЦБ, шт.",
            "amount": "Сумма сделки",
            "accrued_int": "НКД",
            "settlement": "Дата поставки, плановая",
            "fee_ex": "Комиссия ТС"
        }

        row, headers = self.find_section_start("СДЕЛКИ С ЦЕННЫМИ БУМАГАМИ",
                                               "Биржевые сделки с ценными бумагами в отчетном периоде", columns)
        if row < 0:
            return False
        asset_name = ''
        while row < self._statement.shape[0]:
            if self._statement[0][row] == '' and self._statement[0][row + 1] == '':
                break
            if self._statement[0][row] == 'Итого по выпуску:' or self._statement[0][row] == '':
                row += 1
                continue
            try:
                deal_number = int(self._statement[0][row])
            except ValueError:
                asset_name = self._statement[0][row]
                row += 1
                continue

            asset_id = self._parent.findAssetID('', isin=self._statement[headers['isin']][row], name=asset_name)
            if self._statement[headers['B/S']][row] == 'Покупка':
                qty = self._statement[headers['qty']][row]
                bond_interest = -self._statement[headers['accrued_int']][row]
            elif self._statement[headers['B/S']][row] == 'Продажа':
                qty = -self._statement[headers['qty']][row]
                bond_interest = self._statement[headers['accrued_int']][row]
            else:
                row += 1
                logging.warning(g_tr('Uralsib', "Unknown trade type: ") + self._statement[headers['B/S']][row])
                continue

            price = self._statement[headers['price']][row]
            fee = self._statement[headers['fee_ex']][row]
            amount = self._statement[headers['amount']][row]
            if abs(abs(price * qty) - amount) >= Setup.DISP_TOLERANCE:
                price = abs(amount / qty)
            ts_string = self._statement[headers['date']][row] + ' ' + self._statement[headers['time']][row]
            timestamp = int(datetime.strptime(ts_string, "%d.%m.%Y %H:%M:%S").replace(tzinfo=timezone.utc).timestamp())
            settlement = int(datetime.strptime(self._statement[headers['settlement']][row],
                                               "%d.%m.%Y").replace(tzinfo=timezone.utc).timestamp())
            JalDB().add_trade(self._account_id, asset_id, timestamp, settlement, deal_number, qty, price, -fee)
            if bond_interest != 0:
                JalDB().add_dividend(DividendSubtype.BondInterest, timestamp, self._account_id, asset_id,
                                     bond_interest, "НКД", deal_number)
            cnt += 1
            row += 1
        logging.info(g_tr('Uralsib', "Trades loaded: ") + f"{cnt}")

    def load_cash_tranactions(self):
        cnt = 0
        columns = {
            "number": "№ операции",
            "date": "Дата",
            "type": "Тип операции",
            "amount": "Сумма",
            "description": "Комментарий"
        }
        operations = {
            'Ввод ДС': self.transfer_in,
            'Вывод ДС': self.transfer_out,
            'Налог': self.tax,
            'Доход по финансовым инструментам': self.dividend,
            'Погашение купона': self.interest
        }

        row, headers = self.find_section_start("ДВИЖЕНИЕ ДЕНЕЖНЫХ СРЕДСТВ ЗА ОТЧЕТНЫЙ ПЕРИОД", '',
                                               columns, header_height=1)
        if row < 0:
            return False

        while row < self._statement.shape[0]:
            if self._statement[0][row] == '' and self._statement[0][row + 1] == '':
                break

            operation = self._statement[headers['type']][row]
            if operation not in operations:   # not supported type of operation
                row += 1
                continue
            number = self._statement[headers['number']][row]
            timestamp = int(datetime.strptime(self._statement[headers['date']][row],
                                              "%d.%m.%Y").replace(tzinfo=timezone.utc).timestamp())
            amount = self._statement[headers['amount']][row]
            description = self._statement[headers['description']][row]

            operations[operation](timestamp, number, amount, description)

            cnt += 1
            row += 1
        logging.info(g_tr('Uralsib', "Cash operations loaded: ") + f"{cnt}")

    def transfer_in(self, timestamp, _number, amount, description):
        currency_name = JalDB().get_asset_name(JalDB().get_account_currency(self._account_id))
        text = g_tr('Uralsib', "Deposit of ") + f"{amount:.2f} {currency_name} " + \
               f"@{datetime.utcfromtimestamp(timestamp).strftime('%d.%m.%Y')}\n" + \
               g_tr('Uralsib', "Select account to withdraw from:")
        pair_account = self._parent.selectAccount(text, self._account_id)
        if pair_account == 0:
            return
        JalDB().add_transfer(timestamp, pair_account, amount, self._account_id, amount, 0, 0, description)

    def transfer_out(self, timestamp, _number, amount, description):
        currency_name = JalDB().get_asset_name(JalDB().get_account_currency(self._account_id))
        text = g_tr('Uralsib', "Withdrawal of ") + f"{-amount:.2f} {currency_name} " + \
               f"@{datetime.utcfromtimestamp(timestamp).strftime('%d.%m.%Y')}\n" + \
               g_tr('Uralsib', "Select account to deposit to:")
        pair_account = self._parent.selectAccount(text, self._account_id)
        if pair_account == 0:
            return                                       # amount is negative in XLS file
        JalDB().add_transfer(timestamp, self._account_id, -amount, pair_account, -amount, 0, 0, description)

    def dividend(self, timestamp, number, amount, description):
        parts = re.match(self.DividendPattern, description, re.IGNORECASE)
        if parts is None:
            logging.error(g_tr('Uralsib', "Can't parse dividend description ") + f"'{description}'")
            return
        dividend_data = parts.groupdict()
        asset_id = self._parent.findAssetID('', reg_code=dividend_data['REG_CODE'])
        if asset_id is None:
            logging.error(g_tr('Uralsib', "Can't find asset for dividend ") + f"'{description}'")
            return
        try:
            tax = float(dividend_data['TAX'])
        except ValueError:
            logging.error(g_tr('Uralsib', "Failed to convert dividend tax ") + f"'{description}'")
            return
        amount = amount + tax   # Statement contains value after taxation while JAL stores value before tax
        shortened_description = dividend_data['DESCR1'] + ' ' + dividend_data['DESCR2']
        JalDB().add_dividend(DividendSubtype.Dividend, timestamp, self._account_id, asset_id,
                             amount, shortened_description, trade_number=number, tax=tax)

    def interest(self, timestamp, number, amount, description):
        parts = re.match(self.BondInterestPattern, description, re.IGNORECASE)
        if parts is None:
            logging.error(g_tr('Uralsib', "Can't parse bond interest description ") + f"'{description}'")
            return
        interest_data = parts.groupdict()
        asset_id = JalDB().find_asset_like_name(interest_data['NAME'], asset_type=PredefinedAsset.Bond)
        if asset_id is None:
            logging.error(g_tr('Uralsib', "Can't find asset for bond interest ") + f"'{description}'")
            return
        JalDB().add_dividend(DividendSubtype.BondInterest, timestamp, self._account_id, asset_id,
                             amount, description, number)

    def tax(self, timestamp, _number, amount, description):
        JalDB().add_cash_transaction(self._account_id, self._parent.getAccountBank(self._account_id), timestamp,
                                     amount, PredefinedCategory.Taxes, description)

    def load_cash_balance(self):
        columns = {
            "settled_cash": "Плановый исходящий остаток"
        }

        row, headers = self.find_section_start("ПОЗИЦИЯ ПО ДЕНЕЖНЫМ СРЕДСТВАМ", '', columns)
        if row < 0:
            return False

        self._settled_cash[self._account_id] = self._statement[headers['settled_cash']][row]


from config import data_db_schema,data_ocr
from repositories import DataRepo
from utilities import LANG_CODES
import logging
from logging.config import dictConfig
log = logging.getLogger('file')

repo = DataRepo()

class OcrModel:
    def __init__(self):
        self.db     =   data_db_schema
        self.col    =   data_ocr

    def compute_ocr_data_filters(self,ocr_data):
        log.info("Updating ocr filter params!")
        try:

            for filter in ocr_data["filters"]:

                if filter["filter"]         ==  "collectionMethod.collectionDescription":
                    collection_method       =   repo.distinct("collectionMethod.collectionDescription",self.db,self.col)
                    filter["values"]        =   self.get_collection_details(collection_method)
                    log.info("collected availabel collection methods")
                elif filter["filter"]       ==  "sorceLanguage":
                    langres                 =   repo.distinct("sourceLanguage",self.db,self.col)
                    filter["values"]        =   self.get_language_filter(langres)
                    log.info("collected available languages")
                else:
                    response                =   repo.distinct(filter["filter"],self.db,self.col)
                    filter["values"]        =   self.get_formated_data(response)
                    log.info(f"collected available {filter['label']}")

            return ocr_data
        except Exception as e:
            log.info(f"Exception on OcrModel :{e}")

    def get_language_filter(self,lang_list):
        log.info("formatting language filter")
        values = []
        for data in lang_list:
            attribute = {}
            attribute["value"] = data
            attribute["label"] = LANG_CODES.get(data)
            values.append(attribute)
        return values

    def get_collection_details(self,collection_methods):
        log.info("formatting collection method,details filter")
        values = []
        for data in collection_methods:
            collection = {}
            collection["value"] = data
            collection["label"] = ' '.join(data.split('-')).title()
            tools =[]
            if data in ['web-scraping-ocr']:
                ocrtools = repo.distinct("collectionMethod.collectionDetails.ocrTool",self.db,self.col)
                tools.append({"OCR Tool" : ocrtools })

            collection["tool/method"] = tools
            values.append(collection)
        return values

    def get_formated_data(self, attribute_data):
        log.info("formatting filter attribute")
        values = []
        for data in attribute_data:
            attribute = {}
            attribute["value"] = data
            attribute["label"] = str(data).title()
            values.append(attribute)
        return values

# Log config
dictConfig({
    'version': 1,
    'formatters': {'default': {
        'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(threadName)s %(levelname)s in %(module)s: %(message)s',
    }},
    'handlers': {
        'info': {
            'class': 'logging.FileHandler',
            'level': 'DEBUG',
            'formatter': 'default',
            'filename': 'info.log'
        },
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'DEBUG',
            'formatter': 'default',
            'stream': 'ext://sys.stdout',
        }
    },
    'loggers': {
        'file': {
            'level': 'DEBUG',
            'handlers': ['info', 'console'],
            'propagate': ''
        }
    },
    'root': {
        'level': 'DEBUG',
        'handlers': ['info', 'console']
    }
})


from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector

from cssspy.utils import domains_from_urls, absolute_urls
from cssspy.cssscrapy.items import CssFilesItem
from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor

class CssSpiderMixin(object):
    name = "css"

    def __init__(self, urls=None, *args, **kwargs):
        self.start_urls = urls
        self.allowed_domains = domains_from_urls(*self.start_urls)

    def parse_item(self, response):
        hxs = HtmlXPathSelector(response)
        css_links = hxs.select('//link[@rel="stylesheet"]/@href').extract()
        item = CssFilesItem()
        item['page_url'] = response.url
        item['page_body'] = response.body
        item['css_urls'] = absolute_urls(response.url, css_links)
        item['dead_selectors'] = []
        return item

class CssCrawlSpider(CssSpiderMixin, CrawlSpider):

    rules = (
        Rule(HtmlParserLinkExtractor(), callback='parse_item', follow=True
         ),
    )

    def __init__(self, *args, **kwargs):
        CrawlSpider.__init__(self, *args, **kwargs)
        CssSpiderMixin.__init__(self, *args, **kwargs)

class CssBaseSpider(CssSpiderMixin, BaseSpider):

    def __init__(self, *args, **kwargs):
        BaseSpider.__init__(self, *args, **kwargs)
        CssSpiderMixin.__init__(self, *args, **kwargs)

    def parse(self, response):
        return self.parse_item(response)


from typing import Dict, List

import asana

from giges.models.team import Team
from giges.slack import SlackClient
from giges.tasks.app import app
from giges.util import validate_uuid

def _add_ds_class_item(custom_field: Dict[str, str]) -> str:
    """
    Returns the slack representation for a Data Science item.

    :param custom_field: the unprocessed value from asana API
    :return: a string with an emoji and the name
    """
    return f':large_green_square: {custom_field["name"]}'

def _add_pf_class_item(custom_field: Dict[str, str]) -> str:
    """
    Returns the slack representation for a Platform item.

    :param custom_field: the unprocessed value from asana API
    :return: a string with an emoji and the name
    """
    return f':large_blue_square: {custom_field["name"]}'

def _add_section(memberships: List[Dict], team_projects: List[str]) -> str:
    """
    Returns the name of the section of the workflow project
    the selected task is currently on.

    :param memberships: the memberships response from asana API
    :param team_projects: a list of the ids of the team projects
    :return: a string with the name of the section
    """
    section = ""
    for membership in memberships:
        if membership.get("project"):
            if membership["project"].get("gid") in team_projects:
                section = membership["section"]["name"]

    return section

def _add_class_of_service(custom_field: Dict[str, str]) -> str:
    """
    Returns the slack representation for the class of service of an item.

    :param custom_field: the unprocessed value from asana API
    :return: a string with an emoji
    """
    emojis = {
        "1200760323624685": ":rocket:",
        "1200760323624695": ":bomb:",
        "1200760323630880": ":black_square_button:",
        "1200760323631982": ":brain:",
    }

    gid = custom_field["gid"] if custom_field else ""

    return f'{emojis.get(gid, ":chipmunk:")} '

def daily_stick(team_id: str = None) -> None:
    """
    Wrap the stick inside the app context to be called from SNS
    :param team_id: the giges UUID of the team
    """
    with app.app_context():
        stick(team_id)

def stick(team_id: str = None) -> None:
    """
    For all the projects that belongs to a team:
        - list all the current tasks
        - assigned to each person in the team
        - annoy the sufficient amount

    :param team_id: the giges UUID of the team
    """
    asana_client = asana.Client.access_token(app.config["ASANA_TOKEN"])
    slack_client = SlackClient()
    workspace_id = app.config["ASANA_WORKSPACE"]

    if validate_uuid(team_id):
        team = Team.query.filter_by(id=team_id).one_or_none()
    else:
        # Search for the Tech team by default
        team = Team.query.filter_by(name="Tech").one_or_none()

    projects_ids = [p.external_id for p in team.projects]

    for human in team.tesseras:
        message = f"<@{human.slack_id}|{human.name}> :robot_face: :qoobe_hi:"
        tasks = asana_client.tasks.search_tasks_for_workspace(
            workspace_gid=workspace_id,
            params={
                "assignee.any": human.asana_id,
                "projects.any": ",".join(projects_ids),
                "completed": False,
                "is_subtask": False,
                # 1200200605652838 = In Progress
                "custom_fields.1200200605652836.value": 1200200605652838,
            },
            options={"fields": "followers,assignee", "opt_pretty": True},
        )
        for t in tasks:
            task = asana_client.tasks.find_by_id(t["gid"])
            message += "\n\t -  <https://app.asana.com/0/"
            message += f"{workspace_id}/{task['gid']}/f|{task['name']}> \t"

            fields = {
                "section": _add_section(task["memberships"], projects_ids)
            }
            for field in task["custom_fields"]:
                if field["gid"] == "1200760323623705":
                    fields["class"] = _add_class_of_service(
                        field["enum_value"]
                    )
                elif (
                    field["gid"] == "1200760343663992" and field["enum_value"]
                ):
                    fields["item"] = _add_ds_class_item(field["enum_value"])
                elif (
                    field["gid"] == "1200765868551790" and field["enum_value"]
                ):
                    fields["item"] = _add_pf_class_item(field["enum_value"])
            message += f"{fields.get('class', ':memo:')} "
            message += f"{fields.get('section', ':fairy:')} "
            message += f"{fields.get('item', ':dragon:')}"

        message += "\n:speech_balloon: -> :thread:\n"
        slack_client.send_to_road_blocks(message)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

from absl import flags
import tensorflow as tf

import dataloader
import retinanet_model
from tensorflow.contrib.tpu.python.tpu import tpu_config
from tensorflow.contrib.tpu.python.tpu import tpu_estimator
from tensorflow.contrib.training.python.training import evaluation

# Cloud TPU Cluster Resolvers
flags.DEFINE_string(
    'tpu', default=None,
    help='The Cloud TPU to use for training. This should be either the name '
    'used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 url.')
flags.DEFINE_string(
    'gcp_project', default=None,
    help='Project name for the Cloud TPU-enabled project. If not specified, we '
    'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
    'tpu_zone', default=None,
    help='GCE zone where the Cloud TPU is located in. If not specified, we '
    'will attempt to automatically detect the GCE project from metadata.')

# Model specific paramenters
flags.DEFINE_string(
    'eval_master', default='',
    help='GRPC URL of the eval master. Set to an appropiate value when running '
    'on CPU/GPU')
flags.DEFINE_bool('use_tpu', True, 'Use TPUs rather than CPUs')
flags.DEFINE_bool(
    'use_xla', False,
    'Use XLA even if use_tpu is false.  If use_tpu is true, we always use XLA, '
    'and this flag has no effect.')
flags.DEFINE_string('model_dir', None, 'Location of model_dir')
flags.DEFINE_string('resnet_checkpoint', '',
                    'Location of the ResNet50 checkpoint to use for model '
                    'initialization.')
flags.DEFINE_string('hparams', '',
                    'Comma separated k=v pairs of hyperparameters.')
flags.DEFINE_integer(
    'num_shards', default=8, help='Number of shards (TPU cores)')
flags.DEFINE_integer('train_batch_size', 64, 'training batch size')
flags.DEFINE_integer('eval_steps', 5000, 'evaluation steps')
flags.DEFINE_integer(
    'iterations_per_loop', 100, 'Number of iterations per TPU training loop')
flags.DEFINE_string(
    'training_file_pattern', None,
    'Glob for training data files (e.g., COCO train - minival set)')
flags.DEFINE_string(
    'validation_file_pattern', None,
    'Glob for evaluation tfrecords (e.g., COCO val2017 set)')
flags.DEFINE_string(
    'val_json_file',
    '',
    'COCO validation JSON containing golden bounding boxes.')
flags.DEFINE_integer('num_examples_per_epoch', 120000,
                     'Number of examples in one epoch')
flags.DEFINE_integer('num_epochs', 15, 'Number of epochs for training')
flags.DEFINE_string('mode', 'train',
                    'Mode to run: train or eval (default: train)')
flags.DEFINE_bool('eval_after_training', False, 'Run one eval after the '
                  'training finishes.')
# For Eval mode
flags.DEFINE_integer('min_eval_interval', 180,
                     'Minimum seconds between evaluations.')
flags.DEFINE_integer(
    'eval_timeout', None,
    'Maximum seconds between checkpoints before evaluation terminates.')

FLAGS = flags.FLAGS

def main(argv):
  del argv  # Unused.

  if FLAGS.use_tpu:
    tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
        FLAGS.tpu,
        zone=FLAGS.tpu_zone,
        project=FLAGS.gcp_project)
    tpu_grpc_url = tpu_cluster_resolver.get_master()
    tf.Session.reset(tpu_grpc_url)
  else:
    tpu_cluster_resolver = None

  if FLAGS.mode is 'train' and FLAGS.training_file_pattern is None:
    raise RuntimeError('You must specify --training_file_pattern for training.')
  if FLAGS.mode is 'eval':
    if FLAGS.valid_data_dir is None:
      raise RuntimeError('You must specify --valid_data_dir for evaluation.')
    if FLAGS.val_json_file is None:
      raise RuntimeError('You must specify --val_json_file for evaluation.')

  # Parse hparams
  hparams = retinanet_model.default_hparams()
  hparams.parse(FLAGS.hparams)

  params = dict(
      hparams.values(),
      num_shards=FLAGS.num_shards,
      num_examples_per_epoch=FLAGS.num_examples_per_epoch,
      use_tpu=FLAGS.use_tpu,
      resnet_checkpoint=FLAGS.resnet_checkpoint,
      val_json_file=FLAGS.val_json_file,
      mode=FLAGS.mode,
  )
  config_proto = tf.ConfigProto(
      allow_soft_placement=True, log_device_placement=False)
  if FLAGS.use_xla and not FLAGS.use_tpu:
    config_proto.graph_options.optimizer_options.global_jit_level = (
        tf.OptimizerOptions.ON_1)

  run_config = tpu_config.RunConfig(
      cluster=tpu_cluster_resolver,
      evaluation_master=FLAGS.eval_master,
      model_dir=FLAGS.model_dir,
      log_step_count_steps=FLAGS.iterations_per_loop,
      session_config=config_proto,
      tpu_config=tpu_config.TPUConfig(FLAGS.iterations_per_loop,
                                      FLAGS.num_shards))

  # TPU Estimator
  if FLAGS.mode == 'train':
    train_estimator = tpu_estimator.TPUEstimator(
        model_fn=retinanet_model.retinanet_model_fn,
        use_tpu=FLAGS.use_tpu,
        train_batch_size=FLAGS.train_batch_size,
        config=run_config,
        params=params)
    train_estimator.train(
        input_fn=dataloader.InputReader(FLAGS.training_file_pattern,
                                        is_training=True),
        max_steps=int((FLAGS.num_epochs * FLAGS.num_examples_per_epoch) /
                      FLAGS.train_batch_size))

    if FLAGS.eval_after_training:
      # Run evaluation after training finishes.
      eval_params = dict(
          params,
          use_tpu=False,
          input_rand_hflip=False,
          skip_crowd=False,
          resnet_checkpoint=None,
          is_training_bn=False,
          use_bfloat16=False,
      )
      eval_estimator = tpu_estimator.TPUEstimator(
          model_fn=retinanet_model.retinanet_model_fn,
          use_tpu=False,
          train_batch_size=FLAGS.train_batch_size,
          eval_batch_size=1,
          config=run_config,
          params=eval_params)
      eval_results = eval_estimator.evaluate(
          input_fn=dataloader.InputReader(FLAGS.validation_file_pattern,
                                          is_training=False),
          steps=FLAGS.eval_steps)
      tf.logging.info('Eval results: %s' % eval_results)

  elif FLAGS.mode == 'eval':
    # eval only runs on CPU or GPU host with batch_size = 1

    # Override the default options: disable randomization in the input pipeline
    # and don't run on the TPU.
    eval_params = dict(
        params,
        use_tpu=False,
        input_rand_hflip=False,
        skip_crowd=False,
        resnet_checkpoint=None,
        is_training_bn=False,
        use_bfloat16=False,
    )

    eval_estimator = tpu_estimator.TPUEstimator(
        model_fn=retinanet_model.retinanet_model_fn,
        use_tpu=False,
        eval_batch_size=1,
        train_batch_size=FLAGS.train_batch_size,
        config=run_config,
        params=eval_params)

    def terminate_eval():
      tf.logging.info('Terminating eval after %d seconds of no checkpoints' %
                      FLAGS.eval_timeout)
      return True

    # Run evaluation when there's a new checkpoint
    for ckpt in evaluation.checkpoints_iterator(
        FLAGS.model_dir,
        min_interval_secs=FLAGS.min_eval_interval,
        timeout=FLAGS.eval_timeout,
        timeout_fn=terminate_eval):

      tf.logging.info('Starting to evaluate.')
      try:
        eval_results = eval_estimator.evaluate(
            input_fn=dataloader.InputReader(FLAGS.validation_file_pattern,
                                            is_training=False),
            steps=FLAGS.eval_steps)
        tf.logging.info('Eval results: %s' % eval_results)

        # Terminate eval job when final checkpoint is reached
        current_step = int(os.path.basename(ckpt).split('-')[1])
        total_step = int((FLAGS.num_epochs * FLAGS.num_examples_per_epoch) /
                         FLAGS.train_batch_size)
        if current_step >= total_step:
          tf.logging.info('Evaluation finished after training step %d' %
                          current_step)
          break

      except tf.errors.NotFoundError:
        # Since the coordinator is on a different job than the TPU worker,
        # sometimes the TPU worker does not finish initializing until long after
        # the CPU job tells it to start evaluating. In this case, the checkpoint
        # file could have been deleted already.
        tf.logging.info('Checkpoint %s no longer exists, skipping checkpoint' %
                        ckpt)
  else:
    tf.logging.info('Mode not found.')

if __name__ == '__main__':
  tf.logging.set_verbosity(tf.logging.INFO)
  tf.app.run(main)


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

class TrainTestSplitter:
    def __init__(self, subjects, labels):
        """
        Initializes the TrainTestSplitter class.

        Parameters:
        - subjects (list): List of subjects.
        - labels (list): List of labels.
        """
        self.subjects = subjects
        self.labels = labels

    def train_test_losego(self, df, seg_index=0, num_seg=1, samp_freq=1):
        """
        Leave one segment out for all modalities.

        Parameters:
        - df (pd.DataFrame): Input DataFrame containing raw data.
        - seg_index (int): Index of the segment to be left for the test dataset (default: 0).
        - num_seg (int): Number of segments to include in the test dataset (default: 1).

        Returns:
        - pd.DataFrame, pd.DataFrame: Train and test DataFrames.
        """
        # Initialize empty DataFrames for train and test datasets
        df_train = pd.DataFrame()
        df_test = pd.DataFrame()

        # Loop through each subject
        for i in range(len(self.subjects)):
            # Loop through each label
            for j in range(len(self.labels)):
                # Get the data for the current subject and label
                data = df[(df['Subject'] == self.subjects[i]) & (df['Label_ori'] == self.labels[j])]['Data'].values
                data = np.array([np.array(xi) for xi in data])

                # Select the test data based on the segment index and number of segments
                if samp_freq == 1:
                    #print('Sampling frequency is 1')
                    data_test = data[seg_index:(seg_index) + num_seg]
                elif samp_freq == 8: #
                    #print('Sampling frequency is 8')
                    data_test = data[seg_index*samp_freq:(seg_index*samp_freq) + num_seg]

                # Remove the test data from the training data
                data_train = np.delete(data, seg_index, axis=0)

                # Create DataFrames for the train and test data
                df_one_train = self.create_dataframe(data_train)
                df_one_test = self.create_dataframe(data_test)

                # Concatenate the train and test DataFrames
                df_train = pd.concat([df_train, df_one_train]).reset_index(drop=True)
                df_test = pd.concat([df_test, df_one_test]).reset_index(drop=True)

        # Return the train and test DataFrames
        return df_train, df_test

    def train_test_loso(self, df, subject_index=0):
        """
        Leave one subject out for all modalities.

        Parameters:
        - df (pd.DataFrame): Input DataFrame containing raw data.
        - subject_index (int): Index of the subject to be selected as the test subject (default: 0).

        Returns:
        - pd.DataFrame, pd.DataFrame: Train and test DataFrames.
        """
        # Initialize empty DataFrames for train and test data
        df_train = pd.DataFrame()
        df_test = pd.DataFrame()

        # Get train data by excluding the test subject's data from the DataFrame
        data_train = df[(df['Subject'] != self.subjects[subject_index])]['Data'].values
        data_train = np.array([np.array(xi) for xi in data_train])

        # Get test data for the selected subject
        data_test = df[(df['Subject'] == self.subjects[subject_index])]['Data'].values
        data_test = np.array([np.array(xi) for xi in data_test])

        # Create DataFrames for train and test data
        df_one_train = self.create_dataframe(data_train)
        df_one_test = self.create_dataframe(data_test)

        # Concatenate train and test DataFrames
        df_train = pd.concat([df_train, df_one_train]).reset_index(drop=True)
        df_test = pd.concat([df_test, df_one_test]).reset_index(drop=True)

        # Return the train and test DataFrames
        return df_train, df_test

    def create_dataframe(self, data):
        """
        Convert NumPy arrays back to a DataFrame.

        Parameters:
        - data (np.ndarray): Input NumPy array.

        Returns:
        - pd.DataFrame: DataFrame created from the NumPy array.
        """
        # Create an empty DataFrame
        df_one = pd.DataFrame()

        # Extract the data from the NumPy array and assign it to the 'Data' column
        df_one['Data'] = list(data[:, :-3, :])

        # Extract the label from the NumPy array and assign it to the 'Label' column
        df_one['Label'] = data[:, -3, 0]

        # Extract the original label from the NumPy array and assign it to the 'Label_ori' column
        df_one['Label_ori'] = data[:, -2, 0]

        # Extract the subject from the NumPy array and assign it to the 'Subject' column
        df_one['Subject'] = data[:, -1, 0]

        # Return the created DataFrame
        return df_one

    def train_val_split(self, df, test_size=0.2):
        """
        Split the DataFrame into train and test sets.

        Parameters:
        - df (pd.DataFrame): Input DataFrame containing raw data.
        - test_size (float): Proportion of the dataset to include in the test split (default: 0.2).

        Returns:
        - pd.DataFrame, pd.DataFrame: Train and test DataFrames.
        """
        # Split the DataFrame into train and test sets using the specified test size
        df_train, df_val = train_test_split(df, test_size=test_size)

        # Return the train and test DataFrames
        return df_train, df_val


import h2o
import numpy as np

# Start H2O on your local machine
h2o.init ( ip='localhost', port=54321, nthreads=-1, max_mem_size='25g' )
# Import the train_numeric
train_numeric = h2o.import_file ( path="/Users/avinashbarnwal/Desktop/Kaggle/Bosch/train_numeric.csv" )
# train_categorical = h2o.import_file(path = "/Users/avinashbarnwal/Desktop/Kaggle/Bosch/train_categorical.csv")
# train_date = h2o.import_file(path = "/Users/avinashbarnwal/Desktop/Kaggle/Bosch/train_date.csv")

# train_numeric.describe()
train_numeric["Response"] = train_numeric["Response"].asfactor ( )

# train = train_numeric.cbind(train_categorical)
# train=  train.cbind(train_date)

train_numeric_name = train_numeric.col_names
# train_categorical_name = train_categorical.col_names
# train_date_name = train_date.col_names

# Yielding Colnames of train_numeric
t = train_numeric.shape
# Yielding Dimension of train_numeric
removelist_train_numeric = [0, (t[1] - 1)]
# Removing Id and response
train_numeric_name = [v for i, v in enumerate ( train_numeric_name ) if i not in removelist_train_numeric]

# print train_numeric_name

# removelist_train_categorical = [0]
# train_categorical_name   =     [v for i, v in enumerate(train_categorical_name) if i not in removelist_train_categorical]

# removelist_train_date = [0]
# train_date_name = [v for i, v in enumerate(train_date_name) if i not in removelist_train_date]

# Concatenate the arrays
# train_name =sum([train_numeric_name, train_categorical_name, train_date_name],[])

# Train GBM model

gbm_model = h2o.estimators.gbm.H2OGradientBoostingEstimator ( ntrees=100,
                                                              max_depth=5,
                                                              learn_rate=0.1,
                                                              sample_rate=0.8,
                                                              col_sample_rate=0.8,
                                                              seed=2016,
                                                              nfolds=5,
                                                              fold_assignment="Stratified",
                                                              keep_cross_validation_predictions='TRUE',
                                                              keep_cross_validation_fold_assignment='TRUE',
                                                              score_tree_interval=20,
                                                              stopping_rounds=10,
                                                              stopping_metric="AUC",
                                                              stopping_tolerance=0.01 )

gbm_model.train ( x=train_numeric_name, y="Response", training_frame=train_numeric )

importance = gbm_model.varimp ( )
# print importance

# importance <- as.dataframe(importance)
# importance <- importance[scaled_importance>0]


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import pywt
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
import random
from scipy.spatial import distance
import math
from typing import Union

DataSources = Union[str,pd.Series,np.ndarray]

class TemplateError():
    pass

class DWTTemplate():

    def __init__(self, template: np.ndarray, idxs = (0,0), start_level = 4):
        self.__level = start_level
        self.template = template
        self.template_norm = template/np.sum(template)
        self.idxs = idxs
        self.strart_level = start_level
        coeffs = pywt.wavedec(self.template_norm, 'haar', level = start_level)
        self.ca, self.cd = coeffs[0],coeffs[1:]

    def down(self)->None:
        """
        Reduces the wavelet transform level of the Series Piece.
        """
        self.__level -= 1
        if self.__level == 0:
            raise TemplateError("The wavelet transform is too low. The level cannot be lower than 1.")
        coeffs = pywt.wavedec(self.template_norm, 'haar', level = self.__level)
        self.ca, self.cd = coeffs[0],coeffs[1:]

class DWTTemplates():

    def __init__(self, series:DataSources, start_level = 4, pattern_len = 120,
                 t_count = 10000, temp_type = 'shuffle', acceleration = True):
        self.series = self.__init_series(series)

        self.current_level = start_level
        self.all_patters = False
        self.__ixs = None
        self._templates =  None
        self._acceleration = acceleration
#         self._templates = self.__init_templates(temp_type, start_level, pattern_len, t_count)

    def set_options(self, start_level = 4, pattern_len = 120, t_count = 10000, temp_type = 'shuffle'):
        self._templates = self.__init_templates(temp_type, start_level, pattern_len, t_count)

    def __init_series(self,series:DataSources)->np.ndarray:
        if isinstance(series, str):
            return pd.read_csv(series)
        if isinstance(series,  pd.Series):
            return series.values
        if isinstance(series,  np.ndarray):
            return series
        return None

    def __init_templates(self, temp_type:str, start_level:int, pattern_len:int, t_count = None)->np.ndarray:
        """
        Split time series to Series Pieces.
        """
        piece_len = pattern_len
        slices = []
        ixs = []
        ixs = np.asarray(list(range(0,len(self.series)- piece_len)))
        if temp_type == 'shuffle':
            np.random.shuffle(ixs)
            self.__ixs = ixs[:t_count]
        elif temp_type == 'half_step':
            step = pattern_len//2
            self.__ixs = ixs[::step]
        elif temp_type == 'third_step':
            step = pattern_len//3
            self.__ixs = ixs[::step]
        for i in range(0, len(self.series)- pattern_len):
            part = self.series[i:i+pattern_len]
            part = DWTTemplate(part, start_level = start_level, idxs = (i, i+pattern_len))
            slices.append(part)
        slices = np.asarray(slices)
        return slices

    @property
    def cur_templates(self):
        """
        List all pieces of temporary data in a -Template- format.
        """
        return self._templates[self.__ixs]

    @property
    def templates(self):
        """
        List of all aggregated pieces of temporary format np.ndarray data.
        """
        itemplates = []
        if self.all_patters == False:
            T = self._templates[self.__ixs]
        else:
            T = self._templates
        for t in T:
            itemplates.append(t.ca)
        return  np.asarray(itemplates)

    @property
    def original_templates(self):
        """
        List of all original pieces of temporary format np.ndarray data.
        """
        itemplates = []
        if self.all_patters == False:
            T = self._templates[self.__ixs]
        else:
            T = self._templates
        for t in T:
            itemplates.append(t.template_norm)
        return  np.asarray(itemplates)

    def down(self):
        if self._acceleration == True:
            for i in self.__ixs:
                    self._templates[i].down()
        else:
            for i,t in enumerate(self._templates):
                    self._templates[i].down()
        return

if __name__ == "__main__":
    data = pd.read_csv("data.csv", skiprows = 5, header = None, sep="\s+")
    train_data = data[4]
    templates = DWTTemplates(train_data, pattern_len = 96, start_level = 4)
    t = templates.templates
    to = templates.original_templates

#    n_clusters = 30
#    kmeans = KMeans(n_clusters=120)
#    y_kmeans = kmeans.fit_predict(t)
#
#    mask = np.where(y_kmeans == 5)[0][:-1]
#    t1 = t[mask]
#    to1 = to[mask]
#    sqeres_plot(t1, size = (4,4))
#    sqeres_plot(to1, size = (4,4))


from django.db.models import Manager
from .exceptions import TenantError
from .utils import state

FIELD_NAME = "shop"

class TenantManager(Manager):
    def __init__(self):
        self.state = state
        super().__init__()

    def get_queryset(self):
        current_state = self.state.get_state()
        queryset = super().get_queryset()

        if not current_state.get("enabled", True):
            return queryset

        filter_kwargs = {FIELD_NAME: self.state.get_current_tenant()}

        return queryset.filter(**filter_kwargs)

    def bulk_create(self, objs):
        tenant = self.state.get_current_tenant()

        for obj in objs:
            setattr(obj, FIELD_NAME, tenant)

        return super().bulk_create(objs)


#!/usr/bin/env python3

"""
Sales As Code: randomly selects a sales-y buzzword from a google sheet.
"""

import argparse
import logging
import os
import random
import sys

import gspread
from cachetools import TTLCache
from dotenv import load_dotenv
from oauth2client.service_account import ServiceAccountCredentials

logging.basicConfig(level=logging.INFO)

cache = TTLCache(maxsize=1, ttl=3600)

def authenticate():
    """Authenticate with Google Sheets API and return the gsheet object."""
    if args.credentials:
        creds = args.credentials
    else:
        load_dotenv()
        creds = os.getenv("SALES_AS_CODE_CREDS")

    if not creds:
        logging.error("No credentials found. Exiting...")
        sys.exit(1)

    credential = ServiceAccountCredentials.from_json_keyfile_name(
        creds,
        [
            "https://spreadsheets.google.com/feeds",
            "https://www.googleapis.com/auth/drive",
            "https://www.googleapis.com/auth/drive.file",
            "https://www.googleapis.com/auth/spreadsheets",
        ],
    )

    client = gspread.authorize(credential)
    gsheet = client.open("Sales As Code").sheet1

    return gsheet

def fetch_sales_terms(gsheet):
    """Fetch sales terms from Google Sheets."""
    try:
        sales_terms = gsheet.col_values(1)[1:]
        return sales_terms
    except Exception as err:
        logging.error("Failed to fetch sales terms: %s", err)
        return None

def cache_sales_terms(sales_terms):
    """Cache sales terms locally."""
    try:
        cache["sales_terms"] = sales_terms
    except Exception as err:
        logging.error("Failed to cache sales terms: %s", err)

def get_sales_terms():
    """Get sales terms from Google Sheets, cache them locally, and pick one at random."""
    cached_terms = cache.get("sales_terms")
    if cached_terms:
        sales_terms = cached_terms
    else:
        gsheet = authenticate()
        if not gsheet:
            logging.error("Failed to authenticate with Google Sheets. Exiting...")
            return None
        sales_terms = fetch_sales_terms(gsheet)
        if not sales_terms:
            return None
        cache_sales_terms(sales_terms)
    return random.choice(sales_terms)

def spongecase(term):
    """SpOngECAse tHe oUtPUt."""
    return "".join(
        char.upper() if random.randint(0, 1) else char.lower() for char in term
    )

def ask_if_rep_talking():
    """A while loop asking if the rep is still talking."""
    while True:
        try:
            answer = input("Is the rep still talking? (y/n) ").lower()
            if answer != "y":
                break
            sales_term = get_sales_terms()
            if args.spongecase:
                print(spongecase(sales_term))
            else:
                print(f"{sales_term}")
        except Exception as err:
            logging.error("Failed to get sales term: %s", err)

if __name__ == "__main__":
    argparser = argparse.ArgumentParser()
    argparser.add_argument(
        "-s",
        "--spongecase",
        required=False,
        action="store_true",
        help="SpONgECasE ThE oUtPut",
    )
    argparser.add_argument(
        "-c",
        "--credentials",
        required=False,
        help="Path to credentials file",
    )
    argparser.add_argument(
        "--cachebust",
        required=False,
        action="store_true",
        help="Bust the cache",
    )
    args = argparser.parse_args()

    if not authenticate():
        logging.error("Failed to authenticate with Google Sheets. Exiting...")
        sys.exit(1)

    ask_if_rep_talking()


from getmac import get_mac_address as gma
import sys
import os
import getopt

#varibles
mac = None
mac_vendor = ""
#argv
argv = sys.argv[1:]
opts, args = getopt.getopt(argv, "i:m:h", ["ip=", "mac=", "help"])

archivo = open("archivo.txt", "r")
def getmac(ipadress):
    mac = gma(ip=ipadress)
    return mac

def Vendor_Check(mac):
    lista = mac.split(sep=":")
    mac_vendor = ""
    for x in range(4):
        mac_vendor = mac_vendor + lista[x] + ":"
    print(mac_vendor)

for opt, arg in opts:
    if opt in ('-i', '--ip'):
        Vendor_Check(getmac(arg))

def vendors():
    lineas = archivo.read()
    for linea in lineas:
        print(linea)

vendors()


from functools import wraps

import numpy as np

def handle_0D_1D_input(
    patched_kwargs: [], patched_argpos: [], return_scalar=False
):
    """
    A decorator that handles 0D, 1D inputs and transforms them to 2D.

    Parameters
    ----------
    kwarg : list of str
        The names of the keyword arguments that should be handeled.
    argpos : list of int
        The positions of the arguments that should be handeled.
    return_scalar : bool, optional
        If the input is 1D the function should return a scalar,
        by default False.
    """

    def decorator(function):
        @wraps(function)
        def wrapper(self, *args, **kwargs):
            for idx, patched_kwarg in enumerate(patched_kwargs):
                if patched_kwarg in kwargs:
                    # for keyword arguments
                    input_dims = np.ndim(kwargs.get(patched_kwarg))
                    kwargs[patched_kwarg] = np.atleast_2d(
                        kwargs.get(patched_kwarg)
                    )

                elif len(args) > patched_argpos[idx]:
                    # for positional arguments
                    arg_list = list(args)
                    input_dims = np.ndim(args[patched_argpos[idx]])
                    arg_list[patched_argpos[idx]] = np.atleast_2d(
                        args[patched_argpos[idx]]
                    )
                    args = tuple(arg_list)

            res = function(self, *args, **kwargs)

            # return value 1D or scalar when boolean set
            if input_dims == 1:
                # handle functions with multiple return values
                if type(res) is tuple:
                    returnvalues = list(res)
                    returnvalues = [o.ravel() for o in returnvalues]
                    if return_scalar:
                        returnvalues = [o[0] for o in returnvalues]
                    return tuple(returnvalues)

                elif return_scalar and np.ndim(res) != 0:
                    return res.ravel()[0]
                elif np.ndim(res) != 0:
                    return res.ravel()

            return res

        return wrapper

    return decorator