import shutil
import gc
import copy
import numpy
import random
import cv2
from PIL import Image, ImageDraw
import os
from functools import partial
from scipy.ndimage.filters import gaussian_filter
import time
import pickle
import re
from sklearn import preprocessing
import scipy.io as sio
from Att_BiLSTM_training import test_att_BiLSTM
from keras.models import *
from statistics import mode
from scipy import interpolate
from scipy.io import loadmat
import sys
import matplotlib.pyplot as plt
from SMGaccessSample import GestureSample
from scipy.special import softmax

'''
load SMG training dataset
'''
def loader_SMG(parsers):

    # start counting tim
    time_tic = time.time()

    #counting feature number
    HMM_state_feature_count = 0
    n_HMM_state_feature_count = 0

    #how many joints are used
    njoints = parsers['njoints']

    used_joints = parsers['used_joints']

    #HMM temporal steps
    Time_step_NO = parsers['STATE_NUM']#3

    #feature dimension of the LSTM input
    featurenum = int((njoints*(njoints-1)/2 + njoints**2)*3)

    #get sample list
    Sample_list = sorted(os.listdir(parsers['data']))

    dictionaryNum = parsers['class_count']*parsers['STATE_NUM']+1

    '''pre-allocating the memory '''
    #gesture features
    Feature_all_states = numpy.zeros(shape=(400000, featurenum), dtype=numpy.float32)
    Targets_all_states = numpy.zeros(shape=(400000, dictionaryNum), dtype=numpy.uint8)

    n_Feature_all_states = numpy.zeros(shape=(400000, featurenum), dtype=numpy.float32)
    n_Targets_all_states = numpy.zeros(shape=(400000, dictionaryNum), dtype=numpy.uint8)

    # HMM pror and transition matrix
    Prior = numpy.zeros(shape=(dictionaryNum))
    Transition_matrix = numpy.zeros(shape=(dictionaryNum,dictionaryNum))

    #start traversing samples
    for sampleID in Sample_list[0:35]:

        print("\t Processing file " + str(sampleID))

        smp=GestureSample(os.path.join(parsers['data'],sampleID),sampleID)

        gesturesList=smp.getGestures()

        Transition_matrix[-1, -1] += smp.getNumFrames()

        MG_flag = smp.getMGFlag()

        '''traverse all ges in samples'''
        #get the skeletons of the gesture
        for ges_info in gesturesList:
            gestureID, startFrame, endFrame = ges_info
            # print(endFrame - startFrame)
            '''
            1. extract skeleton features of ges
            '''
            Skeleton_matrix, valid_skel = Extract_feature_UNnormalized(smp, used_joints, startFrame, endFrame)
            # to see we actually detect a skeleton:

             ### extract the features according to the CVPR2014 paper
            Feature = Extract_feature_Realtime(Skeleton_matrix, njoints)

            '''generatre the corresponding labels'''
            sample_label = extract_HMMstate_label(Time_step_NO, Feature.shape[0], dictionaryNum, gestureID)

            #assign seg_length number of features to current gesutre
            Feature_all_states[HMM_state_feature_count:HMM_state_feature_count + Feature.shape[0], :] = Feature
            #assign seg_length number of labels to corresponding features
            Targets_all_states[HMM_state_feature_count:HMM_state_feature_count + Feature.shape[0], :] = sample_label
            #update feature count
            HMM_state_feature_count = HMM_state_feature_count + Feature.shape[0]

            '''
            2. extract skeleton features of non ges
            '''
            '''non movement before 5 frames'''
            if 1 not in MG_flag[startFrame-5:startFrame]:
                n_startFrame = startFrame-5
                n_endFrame = startFrame

                Skeleton_matrix, valid_skel = Extract_feature_UNnormalized(smp, used_joints, n_startFrame, n_endFrame)

                Feature = Extract_feature_Realtime(Skeleton_matrix, njoints)

                '''generatre the corresponding labels'''
                #generatre the corresponding labels
                sample_label = extract_nonActionlabel(Feature.shape[0], dictionaryNum)
                #assign seg_length number of features to current gesutre
                n_Feature_all_states[n_HMM_state_feature_count:n_HMM_state_feature_count + Feature.shape[0], :] = Feature
                #assign seg_length number of labels to corresponding features
                n_Targets_all_states[n_HMM_state_feature_count:n_HMM_state_feature_count + Feature.shape[0], :] = sample_label
                #update feature count
                n_HMM_state_feature_count = n_HMM_state_feature_count + Feature.shape[0]

            '''non movement after 5 frames'''

            if 1 not in MG_flag[endFrame:endFrame+5]:
                n_startFrame = endFrame
                n_endFrame = endFrame+5

                Skeleton_matrix, valid_skel = Extract_feature_UNnormalized(smp, used_joints, n_startFrame, n_endFrame)
                Feature = Extract_feature_Realtime(Skeleton_matrix, njoints)

                '''generatre the corresponding labels'''
                #generatre the corresponding labels
                sample_label = extract_nonActionlabel(Feature.shape[0], dictionaryNum)
                #assign seg_length number of features to current gesutre
                n_Feature_all_states[n_HMM_state_feature_count:n_HMM_state_feature_count + Feature.shape[0], :] = Feature
                #assign seg_length number of labels to corresponding features
                n_Targets_all_states[n_HMM_state_feature_count:n_HMM_state_feature_count + Feature.shape[0], :] = sample_label
                #update feature count
                n_HMM_state_feature_count = n_HMM_state_feature_count + Feature.shape[0]

            '''
            3. extract HMM transition info of this ges
            '''

            for frame in range(endFrame-startFrame+1-4):
                # print(gestureID)
                state_no_1,state_no_2 = HMMmatrix(gestureID, frame, endFrame-startFrame+1-4, Time_step_NO)
                # print(state_no_2)
                ## we allow first two states add together:
                Prior[state_no_1] += 1
                Transition_matrix[state_no_1, state_no_2] += 1
                Transition_matrix[-1, -1] -= 1
                if frame<2:
                    Transition_matrix[-1, state_no_1] += 1
                    Prior[-1] += 1
                if frame> (endFrame-startFrame+1-4-2):
                    Transition_matrix[state_no_2, -1] += 1
                    Prior[-1] += 1

    Feature_all_states = Feature_all_states[:HMM_state_feature_count, :]
    #assign seg_length number of labels to corresponding features
    Targets_all_states = Targets_all_states[:HMM_state_feature_count, :]

    print('gesture')
    print(HMM_state_feature_count)

    n_Feature_all_states = n_Feature_all_states[:n_HMM_state_feature_count, :]
    #assign seg_length number of labels to corresponding features
    n_Targets_all_states = n_Targets_all_states[:n_HMM_state_feature_count, :]
    print('non gesture')
    print(n_HMM_state_feature_count)

    num_samples = 20000
    idx = numpy.random.randint(0,len(n_Feature_all_states),size=(num_samples))
    # print(idx)
    n_Feature_all_states = n_Feature_all_states[idx]
    n_Targets_all_states = n_Targets_all_states[idx]
    # save the feature file:
    Feature_all_states = numpy.concatenate((Feature_all_states, n_Feature_all_states))
    Targets_all_states = numpy.concatenate((Targets_all_states, n_Targets_all_states))
    HMM_state_feature_count +=num_samples

    Feature_all,Targets_all,SK_normalizationfilename = process_feature(parsers,Feature_all_states,Targets_all_states,HMM_state_feature_count)

    print ("Processing data done with consuming time %d sec" % int(time.time() - time_tic))

    return Prior, Transition_matrix, Feature_all,Targets_all,SK_normalizationfilename

'''
sparse data list
'''
def process_feature(parsers, Feature_all,Targets_all,action_feature_count):

    # save the feature file:
    print ('total training samples: ' + str(action_feature_count))
    Feature_all = Feature_all[0:action_feature_count, :]
    Targets_all = Targets_all[0:action_feature_count, :]

    #random the samples
    rand_num = numpy.random.permutation(Feature_all.shape[0])
    Feature_all = Feature_all[rand_num]
    Targets_all  = Targets_all[rand_num]

    #[train_set_feature_normalized, Mean1, Std1]  = preprocessing.scale(train_set_feature)
    scaler = preprocessing.StandardScaler().fit(Feature_all)
    Mean1 = scaler.mean_
    Std1 = scaler.scale_
    Feature_all = normalize(Feature_all,Mean1,Std1)

    # save the normalization files
    SK_normalizationfilename = parsers['outpath']+ parsers['experi_ID'] +'SK_normalization.pkl'

    f = open(SK_normalizationfilename,'wb')
    pickle.dump( {"Mean1": Mean1, "Std1": Std1 },f)
    f.close()

    return Feature_all,Targets_all, SK_normalizationfilename

'''
using DNN to get HMM emission probability
'''
def emission_prob(modelname, parsers, Feature, Mean1, Std1,best_threshold,modeltype,AES=True,mu=0.0,lamd=1.0):
    Feature_normalized = normalize(Feature, Mean1, Std1)
    #print Feature_normalized.max()
    #print Feature_normalized.min()

    # feed to  Network
    y_pred_4 = test_att_BiLSTM(modelname, parsers, Feature_normalized, best_threshold)
    #y_pred_4 = test_torch_BiLSTM(modelname, parsers, Feature_normalized,best_threshold, modeltype, iflog)

    if AES:

        y_pred_4 = y_pred_4.T
        observ_likelihood = y_pred_4
        # print(y_pred_4)
        atten_value = y_pred_4[:-1, :]* y_pred_4[:-1, :]/9
        # print(atten_value)softmax(x, axis=0)
        SFMX = softmax(atten_value, axis=0)
        #
        # print(SFMX)
        observ_likelihood[:-1, :] = SFMX* y_pred_4[:-1, :]*mu + y_pred_4[:-1, :]
        # print(observ_likelihood)
        observ_likelihood= numpy.log(observ_likelihood)
        # print(observ_likelihood)
        observ_likelihood[-1, :] = observ_likelihood[-1, :] *lamd

    else:
        observ_likelihood = numpy.log(y_pred_4.T)

    return observ_likelihood

'''
save features
'''
def datasaver(parsers, Prior, Transition_matrix,Feature_all,Targets_all):

    '''define name'''

    Prior_Transition_matrix_filename = parsers['outpath']+ parsers['experi_ID'] + str(parsers['STATE_NUM']) + 'Prior_Transition_matrix.mat'
    Feature_filename = parsers['outpath'] + parsers['experi_ID'] + str(parsers['STATE_NUM']) + 'feature'+ str(parsers['featureNum']) +'Feature_all.pkl'

    '''HMM transition state'''
    #save HMM transition matrix
    sio.savemat( Prior_Transition_matrix_filename, {'Transition_matrix':Transition_matrix, 'Prior': Prior})

    '''feature storage'''
    # save the skeleton file:
    f = open(Feature_filename, 'wb')
    pickle.dump({"Feature_all": Feature_all, "Targets_all": Targets_all }, f,protocol=4)
    f.close()

    return Prior_Transition_matrix_filename, Feature_filename

'''
package Parameters into one file
'''
def packagePara(parsers, model1name, norm_name,HMM_file):

    Paras = {'model1':model1name,
             'norm_para':norm_name,
             'HMM_model':HMM_file,
             }
    path = parsers['outpath'] + parsers['experi_ID'] + '/Paras.pkl'
    afile = open(path, 'wb')
    pickle.dump(Paras, afile)
    afile.close()

'''
test result
'''
def tester_SMG(parsers,best_threshold):

    used_joints = parsers['used_joints']
    dictionaryNum = parsers['class_count']*parsers['STATE_NUM']+1

    MODEL4 = load_model(parsers['outpath']+ parsers['experi_ID'] +'/my_model.h5')
    #print(ges_info_list)
    correct_count = 0.0
    total_count = 0.0
    acc_total = 0.0

    time_tic = time.time()
    datacheck = []

    path = parsers['outpath']+ parsers['experi_ID'] + '/Paras.pkl'
    file2 = open(path, 'rb')
    Paras = pickle.load(file2)
    file2.close()

    ### load the pre-store normalization constant
    f = open(Paras['norm_para'],'rb')
    SK_normalization = pickle.load(f)
    Mean1 = SK_normalization ['Mean1']
    Std1 = SK_normalization['Std1']

    ## Load networks
    modelname1 = Paras['model1']

    ## Load Prior and transitional Matrix
    dic=sio.loadmat(Paras['HMM_model'])
    Transition_matrix = dic['Transition_matrix']

    Transition_matrix[-1, -1]= Transition_matrix[-1, -1]

    Prior = dic['Prior']
    ## Load trained networks
    njoints = parsers['njoints']

    #get sample list
    Sample_list = sorted(os.listdir(parsers['data']))

    total_F1 = 0.0
    total_acc = 0.0
    total_rec = 0.0

    recall_len = 0
    acc_len = 0
    correct_count = 0.0
    threshold_alpha =  parsers['threshold_alpha']
    #start traversing samples
    for sampleID in Sample_list[35:]:

        print(sampleID)
        time_single = time.time()

        '''1 extract skeleton features of this ges'''
        '''process the gesture parts'''
        smp=GestureSample(os.path.join(parsers['data'],sampleID),sampleID)

        gesturesList=smp.getGestures()

        frame_count =smp.getNumFrames()

        MG_flag = smp.getMGFlag()

        Skeleton_matrix, valid_skel = Extract_feature_UNnormalized(smp, used_joints,  1, frame_count - 1)
        # to see we actually detect a skeleton:

         ### extract the features according to the CVPR2014 paper
        Feature = Extract_feature_Realtime(Skeleton_matrix, njoints)
        #ratio = 0.8
        #visiblenumber = int(ratio* (frame_count))

        # sample_feature1 = copy.copy(sample_feature)
        # observ_likelihood1 = emission_prob(modelname1, parsers, sample_feature1, Mean1, Std1, best_threshold, parsers['netmodel'],False)

        sample_feature2 = copy.copy(Feature)
        log_observ_likelihood1 = emission_prob(MODEL4, parsers, sample_feature2, Mean1, Std1, best_threshold, parsers['netmodel'], True,parsers['mu'],parsers['lambda'] )#parsers['AES'])

        log_observ_likelihood1[-1, 0:5] = 0
        log_observ_likelihood1[-1, -5:] = 0
        #
        log_observ_likelihood = log_observ_likelihood1#[:,mask]# + log_observ_likelihood1
        #print("\t Viterbi path decoding " )
        #do it in log space avoid numeric underflow
        [path, _, global_score] = viterbi_path_log(numpy.log(Prior), numpy.log(Transition_matrix), log_observ_likelihood)

        pred_label, begin_frame, end_frame, Individual_score, frame_length = viterbi_colab_MES(path, global_score, state_no = parsers['STATE_NUM'], threshold=-3, mini_frame=15, cls_num = parsers['class_count'])#viterbi_colab_clean_straight(parsers['STATE_NUM'], parsers['class_count'], path, global_score,)

        MG_flag = numpy.zeros(smp.getNumFrames())

        #
        # if True:
        #     # global_score=global_score[:,0:1000]
        #     im  = imdisplay(global_score)
        #     plt.imshow(im, cmap='Greys')## cmap='gray')
        #     plt.plot(range(global_score.shape[-1]), path, color='c',linewidth=2.0)
        #     plt.xlim((0, 2000))#global_score.shape[-1]))
        #     plt.ylim((0, 100))#global_score.shape[-2]))
        #     # plot ground truth
        #     for gesture in gesturesList:
        #     # Get the gesture ID, and start and end frames for the gesture
        #         gestureID,startFrame,endFrame=gesture
        #         frames_count = numpy.array(range(startFrame, endFrame+1))
        #         pred_label_temp = ((gestureID-1) *5+3) * numpy.ones(len(frames_count))
        #         plt.plot(frames_count, pred_label_temp, color='r', linewidth=5.0)
        #
        #     # plot clean path
        #     for i in range(len(begin_frame)):
        #         frames_count = numpy.array(range(begin_frame[i], end_frame[i]+1))
        #         pred_label_temp = ((pred_label[i]-1) *5+3) * numpy.ones(len(frames_count))
        #         plt.plot(frames_count, pred_label_temp, color='b', linewidth=2.0)
        #
        #     plt.show()
        # #

        pred_len = len(pred_label)

        # if True:
        #     # fig = plt.figure()
        #     # fig.set_facecolor("antiquewhite")
        #     im  = imdisplay(global_score)
        #     plt.imshow(im, cmap = 'antiquewhite')
        #     plt.plot(range(global_score.shape[-1]), path, color='c',linewidth=2.0)
        #     plt.xlim((0, 1000))#global_score.shape[-1]))
        #     plt.ylim((0, 100))#global_score.shape[-2]))
        #     # plot ground truth
        #     for gesture in gesturesList[:10]:
        #     # Get the gesture ID, and start and end frames for the gesture
        #         gestureID,startFrame,endFrame=gesture
        #         frames_count = numpy.array(range(startFrame, endFrame+1))
        #         pred_label_temp = ((gestureID-1) *10 +5) * numpy.ones(len(frames_count))
        #         plt.plot(frames_count, pred_label_temp, color='r', linewidth=5.0)
        #
        #     # plot clean path
        #     for i in range(len(begin_frame)):
        #         frames_count = numpy.array(range(begin_frame[i], end_frame[i]+1))
        #         pred_label_temp = ((pred_label[i]-1) *10 +5) * numpy.ones(len(frames_count))
        #         plt.plot(frames_count, pred_label_temp, color='b', linewidth=2.0)
        #
        #     plt.show()

        current_correct_count = 0.0

        for ges in gesturesList:
            gt_label, gt_begin_frame, gt_end_frame = ges
            for pred_i in range(len(begin_frame)):
                # print(pred_i)
                if pred_label[pred_i] == gt_label:
                    alpha = (min(end_frame[pred_i],gt_end_frame)-max(begin_frame[pred_i],gt_begin_frame))/(max(end_frame[pred_i],gt_end_frame)-min(begin_frame[pred_i],gt_begin_frame))
                    # print(pred_i)
                    if alpha> threshold_alpha:
                        # print(pred_label)
                        numpy.delete(pred_label, pred_i)
                        numpy.delete(begin_frame, pred_i)
                        numpy.delete(end_frame, pred_i)
                        #print(pred_label)
                        correct_count +=1
                        current_correct_count +=1
                        break

        recall_len =recall_len + len(gesturesList)
        acc_len = acc_len + pred_len
        print(len(gesturesList))
        print(pred_len)
        # pred_label
        '''recall'''
        current_recall = current_correct_count/len(gesturesList)
        '''precise'''
        current_precision = current_correct_count/pred_len
        current_F1_score = 2*current_recall* current_precision/(current_recall + current_precision+0.0000001)
        print("Used time %d sec, processing speed %f fps, F1 score%f, Recall %f, precision%f" %(int(time.time() - time_single),frame_count/float(time.time() - time_single),current_F1_score, current_recall, current_precision))

    '''recall'''
    recall = correct_count/recall_len
    '''precise'''
    precision = correct_count/acc_len

    total_acc += precision
    total_rec += recall
    F1_score = 2*recall* precision/(recall + precision+0.0000001)
    total_F1 += F1_score

    print ("Processing testing data done with consuming time %d sec" % int(time.time() - time_tic))

    #F1_total = total_F1/len(Sample_list[35:])
    print(parsers['experi_ID']+". The rec for this prediction is " + "{:.12f}".format(total_rec))
    print(parsers['experi_ID']+". The acc for this prediction is " + "{:.12f}".format(total_acc))
    print(parsers['experi_ID']+". The score for this prediction is " + "{:.12f}".format(total_F1))
    print('mu'+str(parsers['mu'])+'lambda'+str(parsers['lambda']))
    numpy.savetxt(parsers['outpath']+ parsers['experi_ID'] +'_score_'+ str(acc_total) +'.txt', [])

def extract_temporal_movingPose(skeleton_all, frame_compens , LSTM_step, njoints):

    frame_count =  skeleton_all.shape[0] - frame_compens*2

    feature_all = Extract_moving_pose_Feature(skeleton_all, njoints)

    feature_dim = feature_all.shape[1]

    feature_n  = numpy.zeros(shape=(frame_count, feature_dim*LSTM_step))

    for frame in range(frame_count):

        for step in range(LSTM_step):

            feature_n[frame, step*feature_dim:(step+1)*feature_dim] = feature_all[frame+step,:]

    return feature_n, feature_n.shape[0]

def extract_HMMstate_label(STATE_NO, action_count, dictionaryNum, gestureID):
    # label the features
    target = numpy.zeros(shape=(action_count, dictionaryNum))
    # HMM states force alignment
    for i in range(STATE_NO):
        # get feature index of the current time step
        begin_feature_index = int(numpy.round(action_count * i / STATE_NO) + 1)
        end_feature_index = int(numpy.round(action_count * (i + 1) / STATE_NO))
        # get feature length of the current time step
        seg_length = end_feature_index - begin_feature_index + 1
        labels = numpy.zeros(shape=(dictionaryNum, 1))
        # assign the one hot labels
        try:
            labels[ i + STATE_NO*gestureID] = 1
        except:
            print(labels.shape)
            print( i + STATE_NO*gestureID)

        target[begin_feature_index-1:end_feature_index,:] = numpy.tile(labels.T, (seg_length, 1))
    return target

def extract_nonActionlabel (action_count, dictionaryNum):
    target_n = numpy.zeros(shape=(action_count, dictionaryNum))
    target_n[:,-1] = 1
    return target_n

'''
record HMM transition matrix
'''
def HMMmatrix(gestureID, frame, frame_count,STATE_NO):
    state_no_1 = numpy.floor(STATE_NO*(frame*1.0/(frame_count+3)))
    state_no_1 = int(state_no_1+STATE_NO*(gestureID))
    state_no_2 = numpy.floor(STATE_NO*((frame+1)*1.0/(frame_count+3)))
    state_no_2 = int(state_no_2+STATE_NO*(gestureID))
    return state_no_1,state_no_2

def normalize(Data, Mean, Std):
    # print(Data.shape)
    # print(Mean.shape)
    Data -= Mean
    Data /= Std
    return Data

def Extract_feature_UNnormalized(smp, used_joints, startFrame, endFrame):
    """
    Extract original features
    """
    frame_num = 0
    Skeleton_matrix  = numpy.zeros(shape=(endFrame-startFrame+1, len(used_joints)*3))

    for numFrame in range(startFrame,endFrame+1):
        # Get the Skeleton object for this frame
        skel=smp.getSkeleton(numFrame)
        for joints in range(len(used_joints)):
            # print((skel.joins[used_joints[joints]][0]))
            # print(Skeleton_matrix[frame_num, joints*3: (joints+1)*3])
            print(skel.joins[used_joints[joints]][0])
            Skeleton_matrix[frame_num, joints*3: (joints+1)*3] =skel.joins[used_joints[joints]][0]

        frame_num += 1

    if numpy.allclose(sum(sum(numpy.abs(Skeleton_matrix))),0):
        valid_skel = False
    else:
        valid_skel = True

    return Skeleton_matrix, valid_skel

def Extract_moving_pose_Feature(Skeleton_matrix_Normalized, njoints):

    #pose
    F_pose = Skeleton_matrix_Normalized

    #velocity
    F_velocity = Skeleton_matrix_Normalized[2:,:] - Skeleton_matrix_Normalized[0:-2,:]

    #accelerate
    F_accelerate = Skeleton_matrix_Normalized[4:,:] + Skeleton_matrix_Normalized[0:-4,:] - 2 * Skeleton_matrix_Normalized[2:-2,:]

    #absolute pose
    FeatureNum = 0
    F_abs = numpy.zeros(shape=(Skeleton_matrix_Normalized.shape[0], int(njoints * (njoints-1)/2)))

    for joints1 in range(njoints-1):
        for joints2 in range(joints1+1,njoints):
            all_X = Skeleton_matrix_Normalized[:, joints1*3] - Skeleton_matrix_Normalized[:, joints2*3]
            all_Y = Skeleton_matrix_Normalized[:, joints1*3+1] - Skeleton_matrix_Normalized[:, joints2*3+1]
            all_Z = Skeleton_matrix_Normalized[:, joints1*3+2] - Skeleton_matrix_Normalized[:, joints2*3+2]

            Abs_distance = numpy.sqrt(all_X**2 + all_Y**2 + all_Z**2)
            F_abs[:, FeatureNum] = Abs_distance
            FeatureNum += 1

    Features = numpy.concatenate((F_pose[2:-2, :], F_velocity[1:-1,:], F_accelerate, F_abs[2:-2, :]), axis = 1)
    return Features

def viterbi_path_log(prior, transmat, observ_likelihood):
    """ Viterbi path decoding
    Wudi first implement the forward pass.
    Future works include forward-backward encoding
    input: prior probability 1*N...
    transmat: N*N
    observ_likelihood: N*T
    """
    T = observ_likelihood.shape[-1]
    N = observ_likelihood.shape[0]

    path = numpy.zeros(T, dtype=numpy.int32)
    global_score = numpy.zeros(shape=(N,T))
    predecessor_state_index = numpy.zeros(shape=(N,T), dtype=numpy.int32)

    t = 1
    global_score[:, 0] =  observ_likelihood[:, 0]
    # print(global_score.shape)
    # need to  normalize the data

    for t in range(1, T):
        for j in range(N):
            # print(global_score[:, t-1].shape)
            # print(prior.shape)

            temp = global_score[:, t-1] + transmat[:, j] + observ_likelihood[j, t]-prior[0]
            global_score[j, t] = max(temp)
            predecessor_state_index[j, t] = temp.argmax()

    path[T-1] = global_score[:, T-1].argmax()

    for t in range(T-2, -1, -1):
        path[t] = predecessor_state_index[ path[t+1], t+1]

    return [path, predecessor_state_index, global_score]

def viterbi_colab_MES(path, global_score, state_no = 5, threshold=-3, mini_frame=15, cls_num = 10):
    """
    Clean the viterbi path output according to its global score,
    because some are out of the vocabulary
    """
    # just to accommodate some frame didn't start right from the begining
    all_label = state_no * cls_num # 20 vocabularies
    start_label = numpy.concatenate((range(0,all_label,state_no), range(1,all_label,state_no),range(2,all_label,state_no)))
    end_label   = numpy.concatenate((range(state_no-3,all_label,state_no), range(state_no-2,all_label,state_no),range(state_no-1,all_label,state_no)))

    begin_frame = []
    end_frame = []
    pred_label = []

    frame = 1
    while(frame < path.shape[-1]-1):
        if path[frame-1]==all_label and path[frame] in start_label:
            begin_frame.append(frame)
            # python integer divsion will do the floor for us :)
            pred_label.append( int(path[frame]/state_no))
            while(frame < path.shape[-1]-1):
                if path[frame] in end_label and path[frame+1]==all_label:
                    end_frame.append(frame)
                    break
                else:
                    frame += 1
        frame += 1

    end_frame = numpy.array(end_frame)
    begin_frame = numpy.array(begin_frame)
    pred_label= numpy.array(pred_label)

    if len(begin_frame)> len(end_frame):
        begin_frame = begin_frame[:-1]
        pred_label = pred_label[:-1]

    elif len(begin_frame)< len(end_frame):# risky hack! just for validation file 668
        end_frame = end_frame[1:]
    ## First delete the predicted gesture less than 15 frames
    frame_length = end_frame - begin_frame
    ## now we delete the gesture outside the vocabulary by choosing
    ## frame number small than mini_frame
    mask = frame_length > mini_frame
    begin_frame = begin_frame[mask]
    end_frame = end_frame[mask]
    pred_label = pred_label[mask]

    Individual_score = []
    for idx, g in enumerate(begin_frame):
            score_start = global_score[path[g], g]
            score_end = global_score[path[end_frame[idx]], end_frame[idx]]
            Individual_score.append(score_end - score_start)

    ## now we delete the gesture outside the vocabulary by choosing
    ## score lower than a threshold
    Individual_score = numpy.array(Individual_score)
    frame_length = end_frame - begin_frame
    # should be length independent
    Individual_score = Individual_score/frame_length

    order = Individual_score.argsort()
    ranks = order.argsort()

    mask = Individual_score > threshold
    begin_frame = begin_frame[mask]
    end_frame = end_frame[mask]
    pred_label = pred_label[mask]
    Individual_score = Individual_score[mask]

    return [pred_label, begin_frame, end_frame, Individual_score, frame_length]

def Extract_feature_Realtime(Pose, njoints):
    #Fcc
    FeatureNum = 0
    Fcc =  numpy.zeros(shape=(Pose.shape[0], int(njoints * (njoints-1)/2*3)))
    for joints1 in range(njoints-1):
        for joints2 in range(joints1+1,njoints):
            Fcc[:, FeatureNum*3:(FeatureNum+1)*3] = Pose[:, joints1*3:(joints1+1)*3]-Pose[:, joints2*3:(joints2+1)*3];
            FeatureNum += 1

    #F_cp
    FeatureNum = 0
    Fcp = numpy.zeros(shape=(Pose.shape[0]-1, njoints **2*3))
    for joints1 in range(njoints):
        for joints2 in range(njoints):
            Fcp[:, FeatureNum*3: (FeatureNum+1)*3] = Pose[1:,joints1*3:(joints1+1)*3]-Pose[0:-1,joints2*3:(joints2+1)*3]
            FeatureNum += 1

    #Instead of initial frame as in the paper Eigenjoints-based action recognition using
    #naive-bayes-nearest-neighbor, we use final frame because it's better initiated
    # F_cf

    Features = numpy.concatenate( (Fcc[0:-1, :], Fcp), axis = 1)
    return Features

def imdisplay(im):
    """ display grayscale images
    """
    im_min = im.min()
    im_max = im.max()
    return (im - im_min) / (im_max -im_min)


# coding=utf-8
from glob import glob
import re
from time import time
import requests
import os
# from xml.dom import minidom
from lxml import etree
import base64
import traceback

import html2text as ht #

from selenium.webdriver.chrome.options import Options
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import configparser

## 配置文件名称
configFileName = "config.ini"

res = "res" # 资源目录名称
head={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.52"}
base64ImgHead = "data:image/png;base64,"

request_url = ""
bypass_tables = False
chromeDriverName = ""
# 初始化配置信息
def init_config():
    global request_url
    global bypass_tables
    global chromeDriverName
    config = configparser.ConfigParser()
    config.read(configFileName)
    request_url = config['base_config']['url']
    bypass_tables = config['base_config'].getboolean('bypass_tables')
    chromeDriverName = config['base_config']['config_fileName']

## 判断当前协议
def http_protocol(url):
    if url.startswith('https://'):
        return "https:"
    elif url.startswith('http://'):
        return "http:"

def get_image_type(Url,Number):
        if Url.find(base64ImgHead) != -1:
            return ("png_base64", str(Number)+".png")
        else:
            if Url.find(".jpg") != -1:
                return("url", str(Number)+".jpg")
            elif Url.find(".png") != -1:
                return ("url", str(Number)+".png")
            else:
                return ("url", str(Number)+".png")

def init_browser(url):
    chrome_options = webdriver.ChromeOptions()
    # # 禁用浏览器弹窗
    prefs = {"profile.default_content_setting_values.notifications": 2}
    chrome_options.add_experimental_option("prefs", prefs)
    # 设置浏览器不显示
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')
    driver = webdriver.Chrome(os.path.join(os.getcwd(),chromeDriverName) ,chrome_options=chrome_options)
    driver.maximize_window()
    driver.get(url)
    return driver

# 创建无头浏览器，来处理浏览器执行 跑 n 秒后的页面状态返回回来
def browser_request(url, loadmore = False, waittime = 2):
    browser = init_browser(url)
    time.sleep(waittime)
    if loadmore:
        while True:
            try:
                next_button = browser.find_element_by_class_name("ant-btn")
                next_button.click()
                time.sleep(waittime)
            except Exception as e:
                # traceback.print_exc()
                # info = traceback.format_exc()
                # print(info)
                break
    else:
        time.sleep(waittime)

    html = browser.page_source
    browser.quit()
    return html

def request(url):
    Result=requests.get(url,headers=head)
    Result.encoding = 'utf-8'
    return Result.text

def save_file(fileName, text):
    fileRef = open(fileName, "w+", encoding='utf-8') # win 环境必须指定编码类型，否则默认会使用系统的 gbk编码
    fileRef.write(text)
    fileRef.close()

def get_csdn_title(root):
    r = root.xpath('//h1[@id="articleContentId"][@class="title-article"]/text()')[0].replace("/","%")
    return r

def get_jianshu_title(root): # //*[@id="__next"]/div[1]/div/div[1]/section[1]/article
    r = root.xpath('//section/h1[@class="_1RuRku"]/text()')[0].replace("/","%")

    return r

# 下载资源到本地
def download_res(img_url,srcType,name):
    # print(img_url)
    if srcType == "png_base64" :
        imageBase64Str = img_url.replace(base64ImgHead,"")
        binary_img_data = base64.b64decode(imageBase64Str)
        with open(name,'wb') as fp:
            fp.write(binary_img_data)
        # print(name,'资源下载成功！！！')
    elif srcType == "url":
        img_data = requests.get(url=img_url,headers=head).content
        with open(name,'wb') as fp:
            fp.write(img_data)
            # print(name,'资源下载成功！！！')

# 初始化目录
def init_file(dirName):
    import shutil
    if not os.path.isdir(dirName):
        # shutil.rmtree(titleName)
        os.mkdir(dirName)
    resPath = os.path.join(dirName,res)
    if not os.path.isdir(resPath):
        os.mkdir(resPath)
    return resPath

# 转换为 makedown 并存盘 titleName/titleName.md 文件
def generate_makedown(root, titleName):
    allItem = root.xpath('*')
    need_str = ""
    for p_item in allItem:
        get_str = etree.tostring(p_item,encoding="utf-8")
        need_str += get_str.decode("utf-8")
    text = makedown(need_str)
    save_file(os.path.join(titleName,titleName+".md"), text)

def makedown(need_str):
    text_maker = ht.HTML2Text()
    text_maker.bypass_tables = bypass_tables # 是否使用表格翻译
    text = text_maker.handle(need_str)
    return text

# 处理好 图片的下载逻辑
def downLoadImgMgr(root, resPath):
    imgL = root.xpath('descendant::img')
    record_list = {}
    Number = 0
    for img in imgL:
        DownloadUrl = ""
        if 'data-original-src' in img.attrib and img.attrib['data-original-src'] != "" :
            DownloadUrl = img.attrib['data-original-src']
        elif 'src' in img.attrib and img.attrib['src'] != "" :
            DownloadUrl = img.attrib['src']
        else:
            continue
        if DownloadUrl.startswith('//'):
            DownloadUrl = http_protocol(request_url) + DownloadUrl
        if DownloadUrl in record_list: # 已经下载过了不处理
            img.attrib['src']=record_list[DownloadUrl]
        else:
            srcType,ImageName = get_image_type(DownloadUrl, Number)
            DownloadImgLocalPath = os.path.join(resPath, ImageName)
            ImageNewSrc = os.path.join(res, ImageName)
            download_res(DownloadUrl,srcType, DownloadImgLocalPath)
            img.attrib['src'] = ImageNewSrc
            record_list[DownloadUrl] = ImageNewSrc
            # if srcType == "png_base64":
            Number+=1

# 请求回一个根目录的节点实例
def request_html(url):
    htmlContent = request(url)
    return etree_html(htmlContent)

# 请求回一个根目录的节点实例，通过虚拟一个无头浏览器处理，拿到让js跑完后的结构
def request_html_by_browser(url):
    htmlContent = browser_request(url, True)
    return etree_html(htmlContent)

def etree_html(htmlContent):
    return etree.HTML(htmlContent, parser=etree.HTMLParser(encoding='utf-8'))

def scoll_down(html_page):# 滚轮下拉到最底部
    html_page.send_keys(Keys.END)
    time.sleep(1)

def generate_article(user, content1, content2, AccHtmlContent):
    userHtml = user.get_attribute('outerHTML')
    content1Html = content1.get_attribute('outerHTML')
    content2Html = ""
    if len(content2) != 0 :
        content2Html = content2[0].get_attribute('outerHTML')
    saveHtml = "<div>"+ userHtml + "\n" + content1Html + "\n" + content2Html + "\n" + "</div> \n"
    AccHtmlContent += saveHtml
    return AccHtmlContent

#  csnd 的相关处理
def start_make_csdn():
    tree = request_html(request_url)
    titleName = get_csdn_title(tree)
    resPath = init_file(titleName)
    makeRoot = tree.xpath('//div[@id="content_views"]')[0]
    downLoadImgMgr(makeRoot, resPath)
    generate_makedown(makeRoot, titleName)

# 处理简书
def make_jianshu():
    tree = request_html_by_browser(request_url) # 拿到html结构
    titleName = get_jianshu_title(tree)     # 获取标题
    makeRoot = tree.xpath('//section/article')[0]
    resPath = init_file(titleName)          # 初始化所需目录
    downLoadImgMgr(makeRoot, resPath)       # 处理资源下载
    generate_makedown(makeRoot, titleName)  # 处理makedown转换

def start():
    init_config()
    print(request_url)
    if request_url == "" :
        print("url配置为空")
        return
    if request_url.find("jianshu") != -1:
        make_jianshu()
    elif request_url.find("csdn") != -1:
        start_make_csdn()
    else:
        print("url 错误 不属于csdn也不属于简书")

    print("处理 完成")

start()


import win32serviceutil
import win32service
import win32event
import servicemanager
import configparser
import os
import inspect
from multiprocessing import Process, Pipe
from db.sqlitemanager import SQLiteManager
from proc.node_client_process import NodeClientProcess
import utils.script_manager as sm
import utils.logging as logutils

def pipe_recv_handler(node_process, parent_pipe):
    node_process._logger.info("Node Pipe Recv Handler Spawned. Listening For Messages")
    while True:
        command = parent_pipe.recv()
        node_process._logger.info("Received Command: " + str(command))

        message_for = command["to"]

        if message_for == "NODE":
            answer = node_process.handle_node_requests(command)
            # send the answer back wherever it came (most likely the http)
            # send answer if it is not None
            if answer is not None:
                parent_pipe.send(answer)
        else:
            node_process._logger.warning("Could Not Determine What Message Is For. Can't Forward Appropriatly")

def bootstrapper(wrapper_object, initialization_tuple):
    instance = wrapper_object(initialization_tuple)
    instance.start()
    exit(0)

class AppServerSvc(win32serviceutil.ServiceFramework):
    _svc_name_ = "VesselNode"
    _svc_display_name_ = "Vessel Service Node"

    _config = configparser.ConfigParser()

    _log_dir = None
    _role = None

    _node_process = None
    _script_dir = None

    def __init__(self, args):
        win32serviceutil.ServiceFramework.__init__(self, args)
        self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)

        self._config.read(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
                          + '/conf/service.ini')

        self._log_dir = self._config["LOGGING"]["log_dir"]
        self._root_dir = self._config["DEFAULT"]["root_dir"]
        self._script_dir = self._config["DEFAULT"].get("scripts_dir", self._root_dir + "/scripts")

        logutils.initialize_all_logging_configuration(self._log_dir)
        self._logger = logutils.node_logger

    def SvcStop(self):
        self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
        win32event.SetEvent(self.hWaitStop)

    def SvcDoRun(self):
        self.ReportServiceStatus(win32service.SERVICE_RUNNING)
        servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
                              servicemanager.PYS_SERVICE_STARTED,
                              (self._svc_name_, ''))

        self._logger.info("Service Is Starting")
        self.main()

    def handle_node_requests(self, command):

        if command["command"] == "SYS" and command["param"] == "SHUTDOWN":
            self._logger.info("Shutdown Request Received. Terminating Node")
            self.SvcStop()
            return None

    def main(self):

        self._logger.info("Service Is Initializing...")

        # setup database
        sqlite_manager = SQLiteManager(self._config, self._logger)

        # catalogue all the scripts in the system
        self._logger.info("Catalogueing Engines On The System")
        sm.catalogue_local_engines(sqlite_manager, self._logger)
        self._logger.info("Catalogueing Scripts On The System")
        sm.catalogue_local_scripts(sqlite_manager, self._script_dir, self._logger)

        # create process for listening for node connections
        #  READ through parent_pipe, WRITE through child_pipe
        try:
            self._logger.info("Now Creating Pipe")
            parent_pipe, child_pipe = Pipe()
            self._logger.info("Now Creating NodeClientProcess Class")
            # node_listener = NodeListenerProcess(to_parent_pipe, to_child_pipe, self._config)
            self._logger.info("Now Creating Process With BootStrapper")
            self._node_process = Process(target=bootstrapper,
                                         args=(NodeClientProcess, (child_pipe, self._config, logutils.logging_queue)))
            self._logger.info("Now Starting Process")
            self._node_process.start()
            self._logger.info("Node Process Has Started Running")
        except Exception as e:
            self._logger.exception("An Exception Was Thrown Starting The Node Listener Process")
            self._logger.error("Later - An Exception Was Thrown")

            return

        # create process for listening for http connections

        # start logging thread
        l_thread = logutils.start_logging_thread()

        rc = None
        while rc != win32event.WAIT_OBJECT_0:
            self._logger.info("Service Is Now Running")

            # hang for 1 minute or until service is stopped - whichever comes first
            rc = win32event.WaitForSingleObject(self.hWaitStop, (1 * 60 * 1000))

        self._node_process.terminate()

if __name__ == '__main__':
    win32serviceutil.HandleCommandLine(AppServerSvc)


import unittest
from typing import List

class Solution:
    def letterCombinations(self, digits: str) -> List[str]:
        if digits is None or len(digits) == 0:
            return []
        res = [""]
        alph = ["", "", "abc", "def", "ghi", "jkl", "mno", "pqrs", "tuv", "wxyz"]
        def combine(res, str):
            temp = []
            for i in range(0, len(str)):
                for j in range(0, len(res)):
                    temp.append(res[j] + str[i])
            return temp
        for digit in digits:
            res = combine(res, alph[int(digit)])
        return res

class UnitTest(unittest.TestCase):
    def test(self):
        digits = "23"
        self.assertEqual(["ad","ae","af","bd","be","bf","cd","ce","cf"], Solution().letterCombinations(digits))

if __name__ == "__main__":
    unittest.main()


from django.http import HttpResponseRedirect
from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, render
from main.models.models import Devices, Like

@login_required
def add_like(request, device_id):
    user=request.user
    device = get_object_or_404(Devices, pk=device_id)

    # Sprawdź, czy użytkownik ma już niepolubienie dla tego urządzenia
    existing_dislike = Like.objects.filter(user_id=user, devices_id=device, dislike=True).first()
    if existing_dislike:
        existing_dislike.delete()  # Usuń dislike

    like = Like(user_id=user, devices_id=device, like=True, dislike=False)
    like.save()
    return HttpResponseRedirect(request.META['HTTP_REFERER'])

@login_required
def add_dislike(request, device_id):
    user=request.user
    device = get_object_or_404(Devices, pk=device_id)

    # Sprawdź, czy użytkownik ma już like dla tego urządzenia
    existing_like = Like.objects.filter(user_id=user, devices_id=device, like=True).first()
    if existing_like:
        existing_like.delete()  # Usuń like

    dislike = Like(user_id=user, devices_id=device, like=False, dislike=True)
    dislike.save()
    return HttpResponseRedirect(request.META['HTTP_REFERER'])

@login_required
def remove_like(request, device_id):
    device = get_object_or_404(Devices, pk=device_id)
    user=request.user
    like = Like.objects.filter(user_id=user, devices_id=device, like=True).first()
    if like:
        like.delete()
    return HttpResponseRedirect(request.META['HTTP_REFERER'])

@login_required
def remove_dislike(request, device_id):
    device = get_object_or_404(Devices, pk=device_id)
    user=request.user
    dislike = Like.objects.filter(user_id=user, devices_id=device, dislike=True).first()
    if dislike:
        dislike.delete()
    return HttpResponseRedirect(request.META['HTTP_REFERER'])


import time

import Adafruit_GPIO.SPI as SPI
import Adafruit_SSD1306
import Adafruit_LSM303

from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont

# Raspberry Pi pin configuration:
RST = 24
DC = 23
SPI_PORT = 0
SPI_DEVICE = 0

disp = Adafruit_SSD1306.SSD1306_128_32(rst=RST)
# Note you can change the I2C address by passing an i2c_address parameter like:
disp = Adafruit_SSD1306.SSD1306_128_64(rst=RST, i2c_address=0x3d)
# Create a LSM303 instance.
lsm303 = Adafruit_LSM303.LSM303()

disp.begin()

disp.clear()
disp.display()

width = disp.width
height = disp.height
image = Image.new('1', (width, height))

# Get drawing object to draw on image.
draw = ImageDraw.Draw(image)
x = 2
top = 2
padding = 12
# Load default font.
font = ImageFont.load_default()

dot_arr = []
dot_max = 50
def map_range(a,b, s):
	(a1, a2), (b1, b2) =a,b
	return b1 + ((s -a1) * (b2 -b1) / (a2 - a1))

while True:
    draw.rectangle((0,0,width,height), outline=0, fill=0)
   # draw graph
    draw.line((padding, height-padding, padding, -height+padding), fill=255)
    draw.line((padding, height-padding, padding+width, height-padding), fill=255)

      # Read the X, Y, Z axis acceleration values and print them.
    accel, mag = lsm303.read()
    # Grab the X, Y, Z components from the reading and print them out.
    accel_x = accel[0]

    if len(dot_arr) <= 115:
    	dot_arr.append(accel_x)
    else:
    	del dot_arr[0]
    	dot_arr.append(accel_x)
    x_pos = 15

    for i in range(1,len(dot_arr)):
    	y_pos = map_range((520, -520), (0 , height - padding), dot_arr[i])
    	last_y_pos = map_range((520, -520), (0 , height - padding), dot_arr[i-1])
    	y_pos = round(y_pos)
    	draw.point((x_pos, y_pos), fill=255)
    	draw.line((x_pos,y_pos,x_pos-1,last_y_pos),fill=255)
    	x_pos += 1

    disp.image(image)

    disp.display()
    # Wait half a second and repeat.
    time.sleep(0.5)


'''
Created on 2012/03/20

@author: hogelog
'''

from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
import threading
import logging

class HttpControlerHandler(SimpleHTTPRequestHandler):
    def do_GET(self):
        mapping = self.mapping()
        path = self.path
        if path in mapping:
            content = mapping[path]()
            if content:
                self.send_response(200)
                self.send_header("Content-type", "text/plain")
                self.end_headers()
                self.wfile.write(content)
            else:
                self.send_error(500, "Server Error")
                self.end_headers()
        else:
            self.send_error(404, "Not Found")
            self.end_headers()

    def mapping(self):
        return {}

class BaseHttpControlerHandler(HttpControlerHandler):
    def index(self):
        return "Hello Http Controler!"

    def mapping(self):
        return {
            "/": self.index,
        }

class HttpControler(threading.Thread, HTTPServer):
    def __init__(self, host, port, handler):
        self.address = (host, port)
        self.handler = handler
        threading.Thread.__init__(self)
        HTTPServer.__init__(self, self.address, self.handler)

    def run(self):
        logging.info("start server: %s", self.address)
        self.serve_forever()

    def __stop(self):
        threading.Thread.__stop(self)
        logging.info("stop server: %s", self.address)

if __name__ == '__main__':
    class Handler(BaseHttpControlerHandler):
        def start(self):
            return "Start!"

        def stop(self):
            return "Stop!"

        def mapping(self):
            mapping = BaseHttpControlerHandler.mapping(self)
            mapping["/start"] = self.start
            mapping["/stop"] = self.stop
            return mapping

    server = HttpControler("0.0.0.0", 12345, Handler)
    server.start()
#    server.shutdown()


import sqlite3

name = input("Name of the pdgm project you are working on: ")

db = sqlite3.connect(name + ".sqlite")
cursor = db.cursor()

print("""Please enter source and target of your dependencies.

After each entry you have to confirm the entry. The question can be answered in four ways:
 y - yes:  Valid entry, continue with next entry
 n - no:   Invalid entry, continue with next entry
 d - done: Valid entry, submit and stop the program
 q - quit: Invalid entry, stop without submitting
The answers are case-insensitive. The default (empty) answer means [y]es.
Every other answer is a [n]o.
""")
source = ""

def newlink():
	global source, cursor
	source = input("Source[" + source + "]: ") or source
	target = input("Target: ")
	check = input("Do you want to add link (" + source + ", " + target + ")? [y]ndq: ") or "y"
	check = check.lower()
	if check == "y":
		cursor.execute("INSERT INTO links VALUES (?,?)", (source, target))
		print("Added link (" + source + ", " + target + ")")
		return 1
	elif check == "d":
		cursor.execute("INSERT INTO links VALUES (?,?)", (source, target))
		print("Added link (" + source + ", " + target + ")")
		return -1
	elif check == "q":
		return -2
	else:
		return 0

additions = 0

while True:
	v = newlink()
	if v == -1:
		additions += 1
		break
	elif v == 1:
		additions += 1
	elif v == -2:
		break

print("Added %d links to the database." % (additions,))

db.commit()
db.close()


########################################################################################################
#
#   Determinado banco possui os dados de histórico de empréstimo, vistos na tabela abaixo. Com esse dados,
#   o banco solicitou que fosse construído um modelo que fornecendo os dados de entrada, indique se deverá
#   fornecer ou não o empréstimo
#
#########################################################################################################

# Importando as bibliotecas
import pandas as pd
from sklearn.naive_bayes import GaussianNB
from sklearn import preprocessing

# Gerando o dataset
dados = {'Tempo': ['Ensolarado', 'Ensolarado', 'Nublado', 'Chuvoso', 'Chuvoso', 'Chuvoso',
                   'Nublado', 'Ensolarado', 'Ensolarado', 'Chuvoso'],
         'Umidade': ['Alta', 'Alta', 'Alta', 'Alta', 'Normal', 'Normal', 'Normal', 'Alta',
                     'Normal', 'Normal'],
         'Vento': ['Fraco', 'Forte', 'Fraco','Fraco', 'Fraco', 'Forte', 'Forte', 'Fraco', 'Fraco', 'Fraco'],
         'Treinou': ['Não', 'Não', 'Sim', 'Sim', 'Sim', 'Não', 'Sim', 'Não', 'Sim','Sim']}

dados = pd.DataFrame(data = dados)

# Criando o LabelEncoder
tempo_lbencoder = preprocessing.LabelEncoder()
umidade_lbencoder = preprocessing.LabelEncoder()
vento_lbencoder = preprocessing.LabelEncoder()
treinou_lbencoder = preprocessing.LabelEncoder()

# Usando o LabelEncoder para atribuir números às variáveis qualitativas
tempo_lbencoder.fit(dados['Tempo'].unique())
umidade_lbencoder.fit(dados['Umidade'].unique())
vento_lbencoder.fit(dados['Vento'].unique())
treinou_lbencoder.fit(dados['Treinou'].unique())

# Transformando o dataset de variáveis qualitativas para variáveis quantitativas
dados['Tempo'] = tempo_lbencoder.transform(dados['Tempo'])
dados['Umidade'] = umidade_lbencoder.transform(dados['Umidade'])
dados['Vento'] = vento_lbencoder.transform(dados['Vento'])
dados['Treinou'] = treinou_lbencoder.transform(dados['Treinou'])

# Separando o nosso dataset nos atributos previsores e na classe objetivo
previsor = dados[['Tempo','Umidade','Vento']]
classe = dados['Treinou']

# Criando o classificador NaiveBayes
gnb = GaussianNB()
gnb.fit(previsor, classe)

# Verificando a precisão
print("\n Precisão = ", gnb.score(previsor, classe)*100,"%")

# Inserindo novos dados para serem previstos
previsao = {'Tempo': ['Ensolarado', 'Nublado', 'Nublado','Chuvoso'],
            'Umidade': ['Normal', 'Alta','Normal','Alta'],
            'Vento':['Forte', 'Forte', 'Fraco', 'Forte']}

previsao = pd.DataFrame(data = previsao)

previsao['Tempo'] = tempo_lbencoder.transform(previsao['Tempo'])
previsao['Umidade'] = umidade_lbencoder.transform(previsao['Umidade'])
previsao['Vento'] = vento_lbencoder.transform(previsao['Vento'])

# Verificando o resultado
print("\n", gnb.predict(previsao))
print("\n", treinou_lbencoder.inverse_transform(gnb.predict(previsao)))

# Verificando as probabilidades
print("\n", gnb.predict_proba(previsao))


# _*_ coding: utf-8 _*_

import os

from datetime import datetime
from scrapy import Spider, Request
from pyquery import PyQuery as pq

class Itjuzi(Spider):
    name = 'itjuzi_spider'

    def start_requests(self):
        meta = {}
        meta['page'] = 0
        meta['url'] = 'http://itjuzi.com/company?page=%d'
        url = 'http://itjuzi.com/company?page=%d' % meta['page']
        yield Request(url, callback=self.parse_list, meta=meta)

    def parse_list(self, response):
        meta = response.meta
        html = response.body_as_unicode()
        html_pq = pq(html)

        divs = html_pq('div.company-list.clearfix.childen-hover div.company-list-item')
        for div in divs:
            url = pq(div)('a:eq(0)').attr['href']
            #url = 'http://itjuzi.com/company/21253'
            yield Request(url, callback=self.parse_detail)
            #return [Request(url, callback=self.parse_detail)]

        if u'下一页' in html_pq('div.pagination.pagination-right').text():
            meta['page'] += 1
            url = meta['url'] % meta['page']
            yield Request(url, callback=self.parse_list, meta=meta)

    def parse_detail(self, response):
        meta = response.meta
        html = response.body_as_unicode()
        html_pq = pq(html)

        keys = [u'网址', u'公司', u'时间', u'地点', u'状态', u'阶段', u'行业', u'子行业', u'TAG', u'简介']
        values = ['' for _ in keys]
        team = []
        products = []
        fund_demand = fund_status = ''
        funds = []
        records = []

        project_name = html_pq('#com_id_value').text()
        divs = html_pq('div.normal-box, div.normal-box-no-pad')
        for div in divs:
            h2_text = pq(div)('h2:eq(0)').text()
            if u'基础信息' in h2_text:
                lis = pq(div)('ul.detail-info li')
                for li in lis:
                    key = pq(li).contents()[0]
                    for i in range(len(keys)):
                        if key.startswith(keys[i]):
                            contents = map(lambda x: pq(x).text().replace(',', u'，').replace('\r', '').replace('\n', '').strip() if hasattr(x, 'text') \
                                    else x.replace(',', u'，').replace('\r', '').replace('\n', '').strip(), pq(li).contents())
                            values[i] = ''.join(contents[1:])
            elif u'团队介绍' in h2_text:
                trs = html_pq('#company-member-list-tbl tr')
                for tr in trs:
                    td = pq(tr)('td')
                    name = td.eq(1).text().strip()
                    position = td.eq(2).text().strip()
                    member = '|'.join([name, position]).replace(',', u'，').replace('\r', '').replace('\n', '')
                    team.append(member)
            elif u'产品介绍' in h2_text:
                product_divs = pq(div)('div.company-product-item')
                for product_div in product_divs:
                    product_name = pq(product_div)('h3').text().strip()
                    product_brief = pq(product_div)('p').text().strip()
                    product = '|'.join([product_name, product_brief]).replace(',', u'，').replace('\r', '').replace('\n', '')
                    products.append(product)
            elif u'融资需求' in h2_text:
                fund_demand = pq(div)('#company-fund-status').text()
            elif u'获投状态' in h2_text:
                fund_status = pq(div)('#company-fund-status').text()
                fund_divs = pq(div)('div.company-fund-item')
                for fund_div in fund_divs:
                    fund_time = ''
                    fund_stage = pq(fund_div)('h3 b').text()
                    tmp = pq(fund_div)('h3').contents()
                    if len(tmp) > 1:
                        fund_time = tmp[1].strip()
                    money = pq(fund_div)('p.company-fund-item-money').text()
                    investor = pq(fund_div)('p:eq(1)').text()
                    fund = '|'.join([fund_time, fund_stage, money, investor]).replace(',', u'，').replace('\r', '').replace('\n', '')
                    funds.append(fund)
            elif u'里程碑' in h2_text:
                record_lis = pq(div)('#company-mile li')
                for li in record_lis:
                    time = pq(li)('b').text()
                    description = pq(li)('p').text()
                    record = '|'.join([time, description]).replace(',', u'，').replace(';', u'。').replace('\r', '').replace('\n', '')
                    records.append(record)

        today = datetime.today().strftime('%Y-%m-%d')
        info = [project_name, ','.join(values), response.url, today, ';'.join(team), ';'.join(products), \
                fund_demand, fund_status, ';'.join(funds), ';'.join(records)]
        result_path = os.path.abspath(__file__).replace('python/itjuzi/itjuzi/spiders/itjuzi_spider.py', 'result/itjuzi.txt')
        fw = open(result_path, 'a')
        fw.write(','.join(info).encode('utf8') + '\n')
        fw.close()


# %% [markdown]
# # Extracting attributes from timedelta columns via a ColumnTransformationPlugin
#
# inspired by Sailu and the following stackoverflow question:
# - https://stackoverflow.com/questions/38355816/pandas-add-timedelta-column-to-datetime-column-vectorized
#
# __Goal:__ extract the number of weeks as float based on the timedelta column

# %%
import pandas as pd
import numpy as np
import bamboolib as bam

# %%
df = pd.DataFrame()

# %%
df["date"] = ["2016-01-10", "2016-05-11", "2016-02-23", "2015-12-08"]
df["date"] = pd.to_datetime(df["date"])

# %%
df["days"] = [28, 7, 15, 30]
df["days"] = pd.to_timedelta(df["days"], "d")

# %%
# # solution:
# df['weeks'] = df['days'] / np.timedelta64(1, 'W')

# %%
import ipywidgets as widgets

from bamboolib.plugins import ColumnTransformationPlugin, DF_OLD, SelectizeDropdown

class TimedeltaExtractAttribute(ColumnTransformationPlugin):

    name = "Timedelta: extract attribute"

    def __init__(self, *args, column=None, **kwargs):
        super().__init__(*args, **kwargs)

        self.column = column

        # based on https://docs.scipy.org/doc/numpy/reference/arrays.datetime.html#datetime-units
        self.attribute = SelectizeDropdown(
            options=[
                ("years", "Y"),
                ("months", "M"),
                ("weeks", "W"),
                ("days", "D"),
                ("hours", "h"),
                ("minutes", "m"),
                ("seconds", "s"),
            ],
            value="D",
            focus_after_init=True,
        )
        self.new_column_name = widgets.Text(value=self.column)

    def render(self):
        self.set_title("Extract attribute")
        self.set_content(
            widgets.HTML(f"Convert <b>{self.column}</b> to"),
            self.attribute,
            widgets.HTML("New column name:"),
            self.new_column_name,
        )

    def get_description(self):
        return (
            f"Extract timedelta attribute {self.attribute.label} from '{self.column}'"
        )

    def get_code(self):
        return f"{DF_OLD}['{self.new_column_name.value}'] = {DF_OLD}['{self.column}'] / np.timedelta64(1, '{self.attribute.value}')"

# %% [markdown]
# __Hint:__ The plugin is shown in bamboolib when clicking on the column header of 'days' and searching for the transformation

# %%
df

# %%