#encoding=utf-8
import sktensor
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys
import os
import scipy.io as scio
import numpy as np
np.seterr(divide='ignore', invalid='ignore')
from random import random
from scipy.sparse import rand as sprand
from compt_methods import *
from road_similar import *
import seaborn as sns

#根据确定的数据维度大小获取数据，对于缺失部分补平均数
def get_tensor(mat_file,ori_file,size_=0):
    ori_data = scio.loadmat(mat_file)['Speed']
    if size_:
        p,q,r = size_
        ori_data = ori_data[:p,:q,:r]
    scio.savemat(ori_file,{'Speed':ori_data})
    return ori_data

#根据缺失率和原始数据进行随机缺失并保存缺少数据
def gene_rand_sparse(ori_data, miss_ratio,miss_file):
    data = ori_data.copy()
    dshape = np.shape(data)
    rand_ts = np.random.random_sample(dshape)
    zero_ts = np.zeros(dshape)
    data = data*(rand_ts>miss_ratio) + zero_ts*(rand_ts<=miss_ratio)
    #if not os.path.exists(miss_file):
    scio.savemat(miss_file,{'Speed':data})
    return data

#根据缺失率和原始数据进行连续缺失并保存缺失数据
def gene_cont_sparse(ori_data,miss_ratio,miss_file):
    data = ori_data.copy()
    dshape = data.shape
    rand_ts = np.random.random_sample(dshape[:-1])
    S = np.rint(rand_ts+0.5-miss_ratio)
    W_cont = np.zeros(dshape)
    for k in range(dshape[2]):
        W_cont[:,:,k] = S[:,:]
    data = data*W_cont
    scio.savemat(miss_file,{'Speed':data})
    return data

gene_sparse = {'rand':gene_rand_sparse,'cont':gene_cont_sparse}

#获取缺失数据，缺失位置和真实缺失率
def get_sparsedata(miss_file):
    #miss_loc = []
    sparse_data = scio.loadmat(miss_file)['Speed']
    dshape = np.shape(sparse_data)
    W_miss = sparse_data==0
    true_miss_ratio = W_miss.sum()/sparse_data.size
    return sparse_data,W_miss,true_miss_ratio

#处理原始的缺失数据
def deal_orimiss(ori_data,shorten = False):
    p,q,r = ori_data.shape
    zero_mat = np.zeros((p,q,r))
    sparse_data = (ori_data>1)*ori_data+(ori_data<=1)*zero_mat
    W = sparse_data>0
    W_miss = np.where(sparse_data<=1)
    M_pos = np.where(W_miss)
    if not shorten:
        for i in range(len(W_miss[0])):
            pos1,pos2,pos3 = W_miss[0][i],W_miss[1][i],W_miss[2][i]
            neigh_info = []
            for n2 in (1,-1):
                for n3 in (1,-1):
                    try:
                        temp = sparse_data[pos1,pos2+n2,pos3+n3]
                        neigh_info.append(temp)
                    except:
                        pass
            if sum(neigh_info)>0:
                sparse_data[pos1,pos2,pos3] = sum(neigh_info)/(np.array(neigh_info)>0).sum()
        return sparse_data,W

    Arr = [set(arr.tolist()) for arr in M_pos]
    Arr_len = [len(arr) for arr in Arr]
    Arr_short = Arr_len.index(min(Arr_len))
    sparse_data = np.delete(sparse_data,list(Arr[Arr_short]),Arr_short)
    return sparse_data,W

#用奇异值比例来确定秩
def tk_rank(data,thre = 0.9):
    rank_set = [0,0,0]
    for i in range(3):
        if np.isnan(data).all():
            print(np.isnan(data).sum())
            sys.exit()
        if np.isnan(data).all():
            print(np.isinf(data).sum())
            sys.exit()
        U,sigma,VT = scipy.linalg.svd(dtensor(data).unfold(i),0)
        for r in range(len(sigma)):
            if sum(sigma[:r])/sum(sigma) > thre:
                rank_set[i] = r
                break
    return rank_set

#预填充，这里采用同一线圈同一时段不同日期的平均数填充
def pre_impute(sparse_data,W,day_axis=1,bias_bool = False):
    if not bias_bool:
        pos = np.where(W==False)
        for p in range(len(pos[0])):
            i,j,k = pos[0][p],pos[1][p],pos[2][p]
            if day_axis == 0:
                if (sparse_data[:,j,k]>0).sum()>0:
                    sparse_data[i,j,k] = np.sum(sparse_data[:,j,k])/(sparse_data[:,j,k]>0).sum()
                else:
                    sparse_data[i,j,k] = np.sum(sparse_data[i,:,:])/(sparse_data[i,:,:]>0).sum()
            elif day_axis == 1:
                if (sparse_data[i,:,k]>0).sum()>0:
                    sparse_data[i,j,k] = np.sum(sparse_data[i,:,k])/(sparse_data[i,:,k]>0).sum()
                else:
                    sparse_data[i,j,k] = np.sum(sparse_data[:,j,:])/(sparse_data[:,j,:]>0).sum()
        return sparse_data
    b,b1 = {},{}
    sp = np.shape(sparse_data)
    impute_data = sparse_data.copy()
    mean = np.sum(impute_data)/(impute_data>0).sum()
    for n in range(3):
        b[n] = np.random.uniform(0,0.1,sp[n])
        b1[n] = np.zeros(sp[n])
    seta,miu = 1e-4,800
    sum_F = sum([np.sum(b[n]**2) for n in range(3)])
    J = 1/2*np.sum(W*(sparse_data-impute_data)**2)+miu/2*sum_F
    ite = 0
    while ite < 100:
        ite += 1
        J_pre = J
        for i in range(sp[0]):
            b1[0][i] = (1-seta*miu)*b[0][i]+seta*np.sum(W[i,:,:]*(sparse_data-impute_data)[i,:,:])
        for j in range(sp[1]):
            b1[1][j] = (1-seta*miu)*b[1][j]+seta*np.sum(W[:,j,:]*(sparse_data-impute_data)[:,j,:])
        for k in range(sp[2]):
            b1[2][k] = (1-seta*miu)*b[2][k]+seta*np.sum(W[:,:,k]*(sparse_data-impute_data)[:,:,k])
        if sum([np.sum((b1[n]-b[n])**2)**0.5 for n in range(3)]) < 0.001:
            pass
        for n in range(3):
            b[n] = b1[n].copy()
        for i in range(sp[0]):
            for j in range(sp[1]):
                for k in range(sp[2]):
                    impute_data[i,j,k] = mean+b[0][i]+b[1][j]+b[2][k]
        sum_F = sum([np.sum(b[n]**2) for n in range(3)])
        J = 1/2*np.sum(W*(sparse_data-impute_data)**2)+miu/2*sum_F
        if abs(J-J_pre) < 1:
            break
    print(np.sum(W*(sparse_data-impute_data)**2))
    sparse_data[W==False] = mean+b[0][i]+b[1][j]+b[2][k]
    return sparse_data

#求rmse,mape,mae,rse
def rmse_mape_rse(est_data,ori_data,W):
    S = np.shape(est_data)
    W_miss = (W==False)
    diff_data = np.zeros(S)+W_miss*(est_data-ori_data)
    #diff_data = ori_data-est_data
    rmse = float((np.sum(diff_data**2)/W_miss.sum())**0.5)
    mre_mat=np.zeros_like(est_data)
    mre_mat[W_miss]=np.abs((est_data[W_miss]-ori_data[W_miss])/ori_data[W_miss])
    mape = float(np.sum(mre_mat)/W_miss.sum())*100
    rse = float(np.sum(diff_data**2)**0.5/np.sum(ori_data[W_miss]**2)**0.5)
    mae = float(np.sum(np.abs(diff_data))/W_miss.sum())
    return round(rmse,4),round(mape,2),round(rse,4),round(mae,4)

def show_img(X,RMSE_list,MAE_list,name_list):
    for i in range(len(name_list)):
        plt.plot(X,RMSE_list[i],'--o')
        plt.savefig(img_dir+'rmse_'+name_list[i])
        plt.close()
        plt.plot(X,MAE_list[i],'--o')
        plt.savefig(img_dir+'mae_'+name_list[i])
        plt.close()
    return 0

def compare_iter(ori_speeddata,miss_data,miss_pos,W):
    sp = np.shape(miss_data)
    rank_set = [0,0,0]
    main_rate = 0.9
    alpha = [1/3,1/3,1/3]
    beta = [0.1,0.1,0.1]
    beta1 = beta.copy()
    gama = [2,2,2]
    lou = 1e-3
    K = 100
    conv = 1e-4
    conv_list = np.arange(1e-4,1e-3,5e-5)
    K_list = [50+10*count for count in range(16)]

    RMSE_lrtc_list,MAE_lrtc_list = [],[]
    RMSE_silrtc_list,MAE_silrtc_list = [],[]
    RMSE_halrtc_list,MAE_halrtc_list = [],[]
    range_list = conv_list
    for conv in conv_list:
        est_lrtc = lrtc_cpt(miss_data,beta,alpha,gama,conv,K,W)
        RMSE_lrtc,MAPE_lrtc,RSE_lrtc,MAE_lrtc = rmse_mape_rse(est_lrtc,ori_speeddata,W)
        RMSE_lrtc_list.append(RMSE_lrtc)
        MAE_lrtc_list.append(MAE_lrtc)
        est_silrtc = silrtc_cpt(miss_data,alpha,beta1,conv,K)
        RMSE_silrtc,MAPE_silrtc,RSE_silrtc,MAE_silrtc = rmse_mape_rse(est_silrtc,ori_speeddata,W)
        RMSE_silrtc_list.append(RMSE_silrtc)
        MAE_silrtc_list.append(MAE_silrtc)
        est_halrtc = halrtc_cpt(miss_data,lou,conv,K,W)
        RMSE_halrtc,MAPE_halrtc,RSE_halrtc,MAE_halrtc = rmse_mape_rse(est_halrtc,ori_speeddata,W)
        RMSE_halrtc_list.append(RMSE_halrtc)
        MAE_halrtc_list.append(MAE_halrtc)
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    ax.plot(range_list,RMSE_lrtc_list,'r--^',label='lrtc')
    ax.plot(range_list,RMSE_silrtc_list,'r--s',label='silrtc')
    ax.plot(range_list,RMSE_halrtc_list,'r--D',label='halrtc')
    ax.legend(loc='best')
    plt.savefig(img_dir+'compare_conv_rmse.png')
    plt.close()
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    ax.plot(range_list,MAE_lrtc_list,'r--^',label='lrtc')
    ax.plot(range_list,MAE_silrtc_list,'r--s',label='silrtc')
    ax.plot(range_list,MAE_halrtc_list,'r--D',label='halrtc')
    ax.legend(loc='best')
    plt.savefig(img_dir+'compare_conv_mae.png')
    plt.close()

    #show_img(K_list,RMSE_tc,MAE_tc,['lrtc','silrtc','halrtc'])
    return 0

def compare_methods(ori_speeddata,ori_W):
    RM_dict,MA_dict,RS_dict,MP_dict = {},{},{},{}
    rt_dict = {}
    miss_list = []
    eva_dict = {'rmse':RM_dict,'mae':MA_dict,'mape':MP_dict}
    for i in range(8):
        print('----'+str(i)+'----')
        miss_ratio = round(0.1*(i+1),2)

        miss_path = data_dir+'miss_'+str(miss_ratio)+''.join(['_'+str(ch) for ch in data_size])+'.mat'
        if not os.path.exists(miss_path):
            gene_rand_sparse(ori_speeddata,miss_ratio,miss_path)
        miss_data,W_miss,tm_ratio = get_sparsedata(miss_path)
        W = (W_miss==False)
        rW = W | (ori_W == False)

        miss_list.append(round(tm_ratio * 100,1))
        #预填充
        time_s = time.time()
        miss_data = pre_impute(miss_data, W)
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(miss_data, ori_speeddata, rW)
        km = 'pre-impute'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s, 1))
        """
        #参数
        p = 0.7
        K = 100     #iterations
        F_thre = 1e-4  #F_norm convergence threshold

        #halrtc
        time_s = time.time()
        lou = 1 / T_SVD(miss_data, p)[0][0]
        est_halrtc = halrtc_cpt(miss_data,lou,F_thre,K,W,alpha=[1/3,1/3,1/3])
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_halrtc,ori_speeddata,rW)
        km = 'HaLRTC'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))

        #Kmeans+halrtc
        time_s = time.time()
        K_n = 4   #cluster_num
        est_kmeans = Kmeans_ha(miss_data, W, K_n, K, F_thre, p)
        time_e = time.time()
        rm,mp,rs,ma = rmse_mape_rse(est_kmeans,ori_speeddata,rW)
        km = 'HaLRTC-CSP'
        if km not in RM_dict:
            RM_dict[km],MP_dict[km],RS_dict[km],MA_dict[km] = [],[],[],[]
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))
        '''
        #STD
        time_s = time.time()
        ap,lm,thre = 2e-10,0.05,0.1
        est_STD = STD_cpt(miss_data, W, thre, ap, lm, p)
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_STD, ori_speeddata, rW)
        km = 'STD'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))
        '''
        #BPCA
        time_s = time.time()
        est_BPCA = BPCA_cpt(miss_data, p)
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_BPCA, ori_speeddata, rW)
        km = 'BPCA'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))
        """

    eva_dict = {'RMSE':RM_dict,'MAE':MA_dict,'MRE':MP_dict,'Run_Time':rt_dict}
    metric_dict = {'RMSE':'km/h', 'MAE':'km/h', 'MRE':'%', 'Run_Time':'s'}
    eva_Ylim = {'RMSE':[2,10],'MAE':[0,5],'MRE':[5,20],'Run_Time':[0,5000]}
    shape = ['r--o','r--*','r--x','r--^','r--s','r--D']
    MK = ['o','o','*','*','x','x']
    CR = ['r','b','y','r','b','y']

    fw = open('compare_methods' + '.txt', 'w')
    fw.write('methods:'+','.join(list(eva_dict['RMSE'].keys()))+'\n')
    fw.write('Missing Rate (%):' + ','.join(list(map(str, miss_list))) + '\n')
    for eva in eva_dict:
        #fig = plt.figure()
        #ax = fig.add_subplot(1,1,1)
        plt.xlabel('Missing Rate (%)')
        plt.ylabel(eva+' ('+metric_dict[eva]+')')
        # xticks([-np.pi, -np.pi / 2, 0, np.pi / 2, np.pi],
        #        [r'$-pi$', r'$-pi/2$', r'$0$', r'$+pi/2$', r'$+pi$'])
        nl = 0
        fw.write(eva+':\n')
        for method in eva_dict[eva]:
            plt.plot(miss_list,eva_dict[eva][method],color=CR[nl],marker=MK[nl],label='$'+method+'$')
            fw.write(','.join(list(map(str, eva_dict[eva][method]))) + '\n')
            nl += 1
        plt.legend(loc='best')
        plt.savefig(img_dir+'compare_mr_'+'_'+eva+'.png')
        plt.close()
    fw.close()

    return 0

def svd_vary(sparse_data):
    ds = sparse_data.shape
    SG = np.zeros((max(ds),3))
    plt.xlabel('Missing Rate (%)')
    for i in range(len(ds)):
        A = dtensor(sparse_data).unfold(i)
        # U,sigma,VT = np.linalg.svd(A,0)
        # SG[:len(sigma),i] = sigma
        for j in range(ds[i]):
            SG[j,i] = np.var(A[j])
        plt.plot(list(range(max(ds))),SG[:,i],'r--o',label='$'+str(i)+'$')

    plt.legend(loc=1)
    plt.savefig(img_dir+'svd_vary.png')
    plt.close()
    return

#比较三维张量和四维张量的填充结果
def compare_3d_4d(ori_speeddata,miss_data):
    time1 = time.time()
    lou = 1 / T_SVD(miss_data, 0.7)[0][0]
    #lou = 0.1
    print(lou)
    est_halrtc = halrtc_cpt(miss_data, lou, 1e-4, 100, W)
    time2 = time.time()
    print('3d_halrtc:', rmse_mape_rse(est_halrtc, ori_speeddata, (W | (ori_W == False))))
    print('3d_time', str(time2 - time1) + 's')
    return

def compare_PI(ori_speeddata,ori_W):
    RM_dict, MA_dict, RS_dict, MP_dict = {}, {}, {}, {}
    rt_dict = {}
    miss_list = []
    # 参数
    p = 0.7
    K = 100  # iterations
    F_thre = 1e-4  # F_norm convergence threshold
    for i in range(8):
        print('----'+str(i)+'----')
        miss_ratio = round(0.1*(i+1),2)
        miss_path = data_dir+'miss_'+str(miss_ratio)+''.join(['_'+str(ch) for ch in data_size])+'.mat'
        if not os.path.exists(miss_path):
            gene_rand_sparse(ori_speeddata,miss_ratio,miss_path)
        miss_data,W_miss,tm_ratio = get_sparsedata(miss_path)
        W = (W_miss==False)
        miss_list.append(round(tm_ratio * 100,1))
        #预填充
        #miss_data = pre_impute(miss_data,W,False)
        rW = W|(ori_W==False)
        time_s = time.time()
        K_n = 4  # cluster_num
        #est_kmeans = Kmeans_ha(miss_data, W, K_n, K, F_thre, p)
        est_halrtc = halrtc_cpt(miss_data, 1e-3, 1e-4, 100, W,[1/3,1/3,1/3])
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_halrtc, ori_speeddata, rW)
        km = 'Without-PI'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))

        miss_data = pre_impute(miss_data, W, False)
        rW = W | (ori_W == False)
        time_s = time.time()
        K_n = 4  # cluster_num
        #est_kmeans = Kmeans_ha(miss_data, W, K_n, K, F_thre, p)
        est_halrtc = halrtc_cpt(miss_data, 1e-3, 1e-4, 100, W,[1/3,1/3,1/3])
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_halrtc, ori_speeddata, rW)
        km = 'With-PI'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))

    eva_dict = {'RMSE': RM_dict, 'MAE': MA_dict, 'MRE': MP_dict, 'Run_Time': rt_dict}
    metric_dict = {'RMSE': 'km/h', 'MAE': 'km/h', 'MRE': '%', 'Run_Time': 's'}
    eva_Ylim = {'RMSE': [2, 10], 'MAE': [0, 5], 'MRE': [5, 20], 'Run_Time': [0, 5000]}
    shape = ['r--o', 'r--*', 'r--x', 'r--^', 'r--s', 'r--D']
    MK = ['o', 'o', '*', '*', 'x', 'x']
    CR = ['r', 'b', 'y', 'r', 'b', 'y']
    fw = open('compare_PI_' + '.txt', 'w')
    fw.write('Missing Rate (%):' + ','.join(list(map(str, miss_list))) + '\n')
    for eva in eva_dict:
        # fig = plt.figure()
        # ax = fig.add_subplot(1,1,1)

        plt.xlabel('Missing Rate (%)')
        plt.ylabel(eva + ' (' + metric_dict[eva] + ')')
        # xticks([-np.pi, -np.pi / 2, 0, np.pi / 2, np.pi],
        #        [r'$-pi$', r'$-pi/2$', r'$0$', r'$+pi/2$', r'$+pi$'])
        nl = 0
        fw.write(eva + ':' + '\n')
        for method in eva_dict[eva]:
            plt.plot(miss_list, eva_dict[eva][method], color=CR[nl], marker=MK[nl], label='$' + method + '$')
            fw.write(','.join(list(map(str, eva_dict[eva][method]))) + '\n')
            nl += 1

        plt.legend(loc='best')
        plt.savefig(img_dir + 'compare_PI_' + '_' + eva + '.png')
        plt.close()
    fw.close()
    return 0

def ori_imputation(miss_data,W,ori_speeddata,ori_W):
    ds = miss_data.shape
    rW = W|(ori_W==False)
    est_Km = Kmeans_ha(miss_data, W, 4, 100, 1e-4, 0.7)
    plt.xlim(0,50)
    plt.ylim(0,50)
    plt.plot([0,50],[0,50],color='b',linewidth=2)
    #plt.scatter(1, 2, s=10, c='r')
    r = 1
    for j in range(ds[1]):
            for k in range(ds[2]):
                if not rW[r,j,k]:
                    plt.scatter(ori_speeddata[r,j,k],est_Km[r,j,k],s=10,c='r')
    plt.savefig('填充真实对比图.png')
    plt.close()
    return

if __name__ == '__main__':
    data_dir = './data/'
    img_dir = './img_test/'
    mat_path = '/home/qiushye/2013_east/2013_east_speed.mat'
    #数据：日期数，线圈数量，时间间隔数
    #data_size = (60,80,144)
    #data_size = (15,35,288)
    data_size = (30,20,72)
    #广州数据

    ori_speeddata = scio.loadmat('../GZ_data/60days_tensor.mat')['tensor']
    #train_sim(ori_speeddata[:,:30,:])
    #assign_group(ori_speeddata[:,:30,:])
    #simMat = multi_sim(ori_speeddata[:,:30,:])
    #scio.savemat('road_sim.mat',{'sim':simMat})
    #sys.exit()
    #print(np.var(ori_speeddata))
    shorten = False
    print((ori_speeddata==0).sum()/ori_speeddata.size)
    ori_speeddata,ori_W = deal_orimiss(ori_speeddata,shorten)
    data_size = np.shape(ori_speeddata)
    print(data_size)

    # compare_methods(ori_speeddata,ori_W)
    # compare_PI(ori_speeddata,ori_W)
    #sys.exit()
    miss_ratio = 0.2
    miss_path = data_dir+'miss_'+str(round(miss_ratio,1))+'_'+'_'.join([str(ch) for ch in data_size])+'.mat'
    #miss_path = data_dir+'cont_miss_'+'_'.join([str(ch) for ch in data_size])+'.mat'
    if not os.path.exists(miss_path):
        gene_rand_sparse(ori_speeddata,miss_ratio,miss_path)
    miss_data,W_miss,tm_ratio = get_sparsedata(miss_path)
    # f, ax = plt.subplots()
    # sns.heatmap(miss_data[0], cmap='RdBu', linewidths=0.05, ax=ax)
    # ax.invert_yaxis()
    # ax.set_xlabel('time interval')
    # ax.set_ylabel('days')
    # plt.savefig(img_dir+'miss_heatmap.png')
    # plt.close()
    # sys.exit()
    W = miss_data>0
    #W1 = miss_data==0
    data_shape = np.shape(ori_speeddata)
    miss_data = pre_impute(miss_data,W)
    print('pre_impute:', rmse_mape_rse(miss_data, ori_speeddata, W | (ori_W == False)))
    #compare_3d_4d(ori_speeddata,miss_data)
    ori_imputation(miss_data, W, ori_speeddata, ori_W)
    #svd_vary(miss_data)

    sys.exit()
    #est_partmiss(ori_speeddata, ori_W, miss_data, W, 0.04)
    #u,sigma,vt = np.linalg.svd(dtensor(miss_data).unfold(1),0)
    #print(sigma[0],sigma[0]/sum(sigma))
    #sys.exit()
    lou = 1e-3
    K = 100
    conv = 1e-4

    halrtc_para = [3e-3,100,1e-4]
    [lou,K,conv_thre] = halrtc_para
    time0 = time.time()
    #est_halrtc = halrtc_cpt(miss_data, 1.3e-3, 1e-4, 100, W, 0)
    time1 = time.time()
    #print('ori_halrtc:', rmse_mape_rse(est_halrtc, ori_speeddata, (W | (ori_W == False))))
    print('ori_time', str(time1- time0) + 's')
    K_n = 2
    labels = SC_1(miss_data, 6, K_n, axis=0)
    est_SC = cluster_ha(labels, miss_data, W, K_n, halrtc_para, axis=0)
    time_e = time.time()
    print('sc_est:',rmse_mape_rse(est_SC, ori_speeddata, W|(ori_W==False)))
    clr_assign,K_n = road_Kmeans(miss_data,ori_W,K_n,W,axis=0,method='cos')
    est_kmeans = cluster_ha(clr_assign,miss_data,W,K_n,halrtc_para,axis=0)
    print('kmeans_est:',rmse_mape_rse(est_kmeans,ori_speeddata,W|(ori_W==False)))
    time2 = time.time()
    print('kmeans_time:',time2-time1,'s')
    sys.exit()
    cr = range(30)


import os
from reportlab.pdfgen import canvas
from reportlab.lib.utils import ImageReader

def img2pdf(filepath):

    name = filepath + '.pdf'
    raw_file = os.listdir(filepath)

    ''' get img file'''
    img_list = []
    for i in raw_file:
        end = i.split('.')[-1]
        if end in ['jpg','jpeg','png']:
            img_list.append(filepath+'/'+i)
    img_list.sort()

    ''' write to file '''
    x = canvas.Canvas(name)
    for img in img_list:
        img_r = ImageReader(img)
        img_size = img_r.getSize()
        x.setPageSize(img_size)
        x.drawImage(img,0,0)
        x.showPage()

    x.save()
    return True

if __name__ == '__main__':

    path = 'kf'
    print('loading...')
    if img2pdf(path):
        print('{} ok.'.format(path))


# detect jolly jumpers
import sys
from functools import reduce

def is_jolly(nums):
    if len(nums) == 1:
        return True
    # else check the diffs
    mask = [False for x in range(len(nums))]
    mask[0] = True # the zero diff does not count
    i = 0
    j = 1
    while j < len(nums):
        diff = abs(nums[j] - nums[i])
        if diff < len(mask):
            mask[diff] = True
        i += 1
        j += 1
    # The first element does not matter
    return reduce(lambda a, b: a and b, mask)

if __name__ == '__main__':
    lines = []
    for line in sys.stdin:
        lines.append(line)

    for line in lines:
        parts = line.replace("\n","").split(" ")
        parts = list(map(lambda k: int(k), parts))
        jolly = is_jolly(parts[1:])
        if jolly:
            print("Jolly")
        else:
            print("Not jolly")


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
files_to_database.py
Create a database with all files in selected directory
"""
import os
import sys
import sqlite3
from pathlib import Path
import hashlib

class Database:
    def __init__(self, name):
        self.conn = sqlite3.connect(name)
        self.cur = self.conn.cursor()
        self.create_table()

    def create_table(self):
        query = """CREATE TABLE IF NOT EXISTS files (
                    id INTEGER PRIMARY KEY,
                    path TEXT NOT NULL UNIQUE,
                    md5 TEXT,
                    mtime TEXT,
                    ctime TEXT,
                    size TEXT
                );"""
        self.cur.execute(query)
        self.conn.commit()

    def insert(self, path, md5, mtime, ctime, size):
        """Insert a path to database"""
        query = f"INSERT OR REPLACE INTO files (path, md5, mtime, ctime, size) " \
                f"VALUES ('{path}', '{md5}', '{mtime}', '{ctime}','{size}');"
        self.cur.execute(query)
        self.conn.commit()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.cur.close()

def main(path):
    """docstring for main"""
    path = Path(path)
    db = Database(f'{path.stem}_files.db')
    for f in path.rglob('*'):
        if f.is_file():
            db.insert(str(f.resolve()),  # path
                      hashlib.md5(f.read_bytes()).hexdigest(),  # md5 checksum
                      f.stat().st_mtime,  # modification time
                      f.stat().st_ctime,  # creation time
                      f.stat().st_size)  # size
            print(f)
    # for root, dirs, files in os.walk(path):
    #     for f in files:
    #         f_path = os.path.join(root, f)
    #         print(f_path)
    #         db.insert(f_path)

if __name__ == '__main__':
    main(sys.argv[1])


import argparse

from colorama import init

from .common import Fore, Messages, Path
from .. import Havok

def parse_args():
    parser = argparse.ArgumentParser(
        description="Compare Havok packfiles.\nAlso works between Wii U and Switch packfiles."
    )
    parser.add_argument(
        "hkFiles", type=Path, help="Paths to Havok packfiles for comparison", nargs="+"
    )

    return parser.parse_args()

def main():
    init(autoreset=True)

    args = parse_args()

    if not args.hkFiles or len(args.hkFiles) <= 1:
        return None

    files = []
    for hkFile in args.hkFiles:
        Messages.loading(hkFile)
        hk = Havok.from_file(hkFile)

        Messages.deserializing(hkFile)
        hk.deserialize()

        files.append(hk)

    for i in range(len(files) - 1):
        print(
            f"{Fore.BLUE}Comparing '{files[i].path.name}' and '{files[i + 1].path.name}'"
        )
        for hkfile0, hkfile1 in zip(files[i].files, files[i + 1].files):
            if hkfile0.data.contents[0] == hkfile1.data.contents[0]:
                print(f"{Fore.GREEN}File contents match!")
            else:
                print(f"{Fore.RED}File contents do not match!")

    Messages.done()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
from StringIO import StringIO
import pycurl
# The BeautifulSoup module
from bs4 import BeautifulSoup
import lxml
import argparse

url = 'http://kotaku.com/'
storage = StringIO()
storage = StringIO()
c = pycurl.Curl()
c.setopt(c.URL, url)
c.setopt(c.WRITEFUNCTION, storage.write)
# again setting headers to prevent bots from scraping
c.setopt(c.HTTPHEADER, ['User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; ',
                                          'header_name2: header_value2'])
c.perform()
c.close()
content = storage.getvalue()
# content lxml
soup = BeautifulSoup(content)

# Beautiful Soup examples
# print soup
spans = soup.find_all('span') # find all spans
# print len(spans)

# find divs with class row
div_rows = soup.find_all('div', {'class' : 'row'})


import numpy as np
import plotly.graph_objs as go

alpha = 0.01
length = 1  # length of the kernel
a = 1
t = 1  # time for modeling
h = 1e-2  # x-axis step of the mesh
tau = 1e-2  # time step of the mesh
M = int(length / h) + 1  # amount of x-axis nodes of the mesh : 11 now
y = np.array([])  # array with y(n, i) values
N = int(t / tau) + 1  # amount of time nodes of the mesh : 11 now

# matrix coefficients
A = a ** 2 * tau ** 2 / h ** 4
B = - 4 * A
C = 1 + 6 * A
D = - 4 * A
E = A
x_arr = np.linspace(0, length, M)

# filling array with initial conditions y(0, x) = alpha * x ** 2
for i in range(M):  # M iterations for M nodes of the mesh
    y = np.append(y, alpha * x_arr[i] ** 2)
y = np.vstack((y, y))
y[1, 1] = 0 # use initial condition for y(1,1)
y = np.vstack((y, np.zeros((N - 2, M))))  # array with initial conditions, boundary conditions and zeroes NxM now: 11x11

for n in range(2, N):
    # matrix creation
    slae_matrix = np.zeros((1, M))  # matrix of the system of the linear algebraic equations
    free_members_column = np.array([])  # column of the free members of the system of the linear algebraic equations
    # fill column of the free members
    for i in range(M):  # M unknown and M equations
        if i < 2 or i > M - 3:  # using the initial and boundary conditions
            free_members_column = np.append(free_members_column, 0)
        else:
            free_members_column = np.append(free_members_column, 2 * y[n - 1, i] - y[n - 2, i])

    # fill SLAE matrix
    for i in range(M):
        matrix_row = np.zeros((1, M))
        if i == 0:  # using boundary condition #1
            slae_matrix[0, i] = 1
            continue
        elif i == 1:    # using boundary condition #2
            matrix_row[0, i] = 1
        elif 1 < i < M - 2:
            matrix_row[0, i - 2] = A
            matrix_row[0, i - 1] = B
            matrix_row[0, i] = C
            matrix_row[0, i + 1] = D
            matrix_row[0, i + 2] = E
        elif i == M - 2:    # using boundary condition #3
            matrix_row[0, i - 1] = 1
            matrix_row[0, i] = -2
            matrix_row[0, i + 1] = 1
        else:   # using boundary condition #4
            matrix_row[0, i - 3] = -1
            matrix_row[0, i - 2] = 3
            matrix_row[0, i - 1] = -3
            matrix_row[0, i] = 1
        slae_matrix = np.vstack((slae_matrix, matrix_row))
    y[n] = np.linalg.solve(slae_matrix, free_members_column)
    print(f'{n} / {N}')

fig = go.Figure()
for n in range(N):
    fig.add_trace(go.Scatter(x=x_arr, y=y[n], name=f't = {n * tau} секунд'))
fig.update_layout(title="Численное решение",
                  xaxis_title="x, м",
                  yaxis_title="y, м",
                  margin=dict(l=0, r=30, t=30, b=0))
fig.show()


#!/usr/bin/env python3

import os
import secrets
import sqlite3

from flask import Flask
from flask import Response
from flask import jsonify
from flask import redirect
from flask import render_template
from flask import request
from flask import url_for

app = Flask(__name__)
app.config["API_URL"] = os.environ.get("API_URL", "http://localhost:5000/api/getQuote/")
app.config["DATABASE"] = "quotes.sqlite"

def get_quote(lang):
    """Return a quote in lang."""
    with sqlite3.connect(app.config["DATABASE"]) as db:
        return db.execute(f"SELECT quote FROM quotes_{lang} ORDER BY random() LIMIT 1").fetchone()[0]

@app.route("/")
@app.route("/<lang>/")
def index(lang=None):
    if lang is None:
        lang = request.accept_languages.best_match(['en', 'de'])

    if lang not in ("de", "en"):
        return redirect(url_for("index", lang="en"))

    nonce = secrets.token_hex(32)

    response = Response(
        render_template("index.html", lang=lang, nonce=nonce, quote=get_quote(lang))
    )
    response.headers["Content-Security-Policy"] = f"default-src 'self'; script-src 'nonce-{nonce}'"

    return response

@app.route("/<lang>/suggest/", methods=["GET", "POST"])
def suggest(lang):
    if lang not in ("de", "en"):
        return redirect(url_for("suggest", lang="en"))

    if request.method == "POST":
        reason = request.form.get("reason")
        thanks = None
        if reason:
            thanks = "yes"

            with sqlite3.connect(app.config["DATABASE"]) as conn:
                conn.execute("INSERT INTO suggestions VALUES (?, CURRENT_TIMESTAMP);", (reason,))

        return redirect(url_for("suggest", lang=lang, thanks=thanks))

    return render_template("suggest.html", lang=lang, thanks=bool(request.args.get("thanks")))

@app.route("/<lang>/api/getQuote/")
def api(lang):
    if lang not in ("de", "en"):
        return Response(
            jsonify({"status": "error", "message": "Unknown language!"}),
            content_type="text/json",
            status=400,
        )

    table = {"de": "quotes_de", "en": "quotes_en"}[lang]
    quote = get_quote(lang)
    return jsonify(
        {"status": "ok", "quote": quote}
    )


# coding:utf-8
import requests
from lxml import html
import os
import datetime
from multiprocessing.dummy import Pool as ThreadPool
import multiprocessing
import re
from multiprocessing import Pool

def header(referer):
    headers = {
        'Host': 'i.meizitu.net',
        'Pragma': 'no-cache',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/59.0.3071.115 Safari/537.36',
        'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
        'Referer': '{}'.format(referer),
    }
    return headers

# 获取主页列表
def getPage(pageNum):
    baseUrl = 'http://www.mzitu.com/page/{}'.format(pageNum)
    try:
        selector = html.fromstring(requests.get(baseUrl,timeout=(3.09,20)).content)
    except:
        return []
    urls = []
    for i in selector.xpath('//ul[@id="pins"]/li/a/@href'):
        urls.append(i)
    return urls

# 图片链接列表， 标题
# url是详情页链接
def getPiclink(url):
    try:
        sel = html.fromstring(requests.get(url,timeout=(3.09,20)).content)
    except:
        return None
    # 图片总数
    total = sel.xpath('//div[@class="pagenavi"]/a[last()-1]/span/text()')[0]
    # 标题
    title = sel.xpath('//h2[@class="main-title"]/text()')[0]
    # 文件夹格式
    dirName = u"[第{}页].[共{}张].{}".format(pag_2,total,title)
    dirName = dirName.replace(":","")
    dirName = dirName.replace("?","")
    dirName = dirName.replace("\\","")
    dirName = dirName.replace("*","")
    dirName = dirName.replace("/","")
    ll = []
    lll = []
    for parent,dirnames,filenames in os.walk(os.path.abspath('.')):
        ll.append(parent)
    for k in ll:
        s = re.findall(r'.+张]\.(.*)',k)
        if s == [] :
            ll.remove(k)
        else:
            lll.append(s[0])

    s = re.findall(r'.+张]\.(.*)',dirName)[0]
    if s not in lll:
        os.mkdir(dirName)
    else:
        print('目录名重复')
        return None
    try:
        total = int(total)
    except:
        total = 1
    n = 1
    for i in range(total):
        try:
            link = '{}/{}'.format(url, i+1)
            try:
                s = html.fromstring(requests.get(link,timeout=(3.09,20)).content)
            except:
                continue
            jpgLink = s.xpath('//div[@class="main-image"]/p/a/img/@src')[0]
            try:
                img = requests.get(jpgLink, headers=header(jpgLink),timeout=(3.09,20)).content
            except:
                continue
            filename = '%s/%s/%s.jpg' % (os.path.abspath('.'), dirName, n)
            print(u'正在下载:第%s页-第%s张-共%s张-%s' % (pag_2, n,total,title))
            with open(filename, "wb+") as jpg:
                jpg.write(img)
            n += 1
        except:
            pass

def start_heiheihei(pageNum):
    global pag_2
    pag_2 = pageNum
    p = getPage(pageNum)
    if p == []:
        return None
    with ThreadPool(5) as pool:
        pool.map(getPiclink, p)
    pool.close()
    pool.join()

if __name__ == '__main__':
    multiprocessing.freeze_support()
    start = datetime.datetime.now()
    result = [x for x in range(1,188)]
    p = Pool(4)
    r = p.map(start_heiheihei,result)
    p.close()
    p.join()
    end = datetime.datetime.now()
    runtime = str((end-start).seconds)
    print(runtime)


import json
import pickle
import random
from os.path import join, dirname

import nltk
from random import shuffle
from string import punctuation
from sklearn.feature_extraction import DictVectorizer
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC

from JarbasModelZoo.features import extract_word_features

MODEL_META = {
    "corpus": "macmorpho",
    "corpus_homepage": "http://www.nilc.icmc.usp.br/lacioweb/",
    "tagset": "Universal Dependencies",
    "lang": "pt",
    "model_id": "sklearn_macmorpho_udep_svm_tagger",
    "algo": "sklearn.svm.LinearSVC",
    "required_packages": ["scikit-learn"]
}

nltk.download('mac_morpho')

def convert_to_universal_tag(t, reverse=True):
    tagdict = {
        'n': "NOUN",
        'num': "NUM",
        'v-fin': "VERB",
        'v-inf': "VERB",
        'v-ger': "VERB",
        'v-pcp': "VERB",
        'pron-det': "PRON",
        'pron-indp': "PRON",
        'pron-pers': "PRON",
        'art': "DET",
        'adv': "ADV",
        'conj-s': "CONJ",
        'conj-c': "CONJ",
        'conj-p': "CONJ",
        'adj': "ADJ",
        'ec': "PRT",
        'pp': "ADP",
        'prp': "ADP",
        'prop': "NOUN",
        'pro-ks-rel': "PRON",
        'proadj': "PRON",
        'prep': "ADP",
        'nprop': "NOUN",
        'vaux': "VERB",
        'propess': "PRON",
        'v': "VERB",
        'vp': "VERB",
        'in': "X",
        'prp-': "ADP",
        'adv-ks': "ADV",
        'dad': "NUM",
        'prosub': "PRON",
        'tel': "NUM",
        'ap': "NUM",
        'est': "NOUN",
        'cur': "X",
        'pcp': "VERB",
        'pro-ks': "PRON",
        'hor': "NUM",
        'pden': "ADV",
        'dat': "NUM",
        'kc': "ADP",
        'ks': "ADP",
        'adv-ks-rel': "ADV",
        'npro': "NOUN",
    }
    if t in ["N|AP", "N|DAD", "N|DAT", "N|HOR", "N|TEL"]:
        t = "NUM"
    if reverse:
        if "|" in t: t = t.split("|")[0]
    else:
        if "+" in t: t = t.split("+")[1]
        if "|" in t: t = t.split("|")[1]
        if "#" in t: t = t.split("#")[0]
    t = t.lower()
    return tagdict.get(t, "." if all(tt in punctuation for tt in t) else t)

dataset = [[(w, convert_to_universal_tag(t)) for (w, t) in sent]
           for sent in nltk.corpus.mac_morpho.tagged_sents()]

shuffle(dataset)

cutoff = int(len(dataset) * 0.8)
train_data = dataset[:cutoff]
test_data = dataset[cutoff:]

def transform_to_dataset(tagged_sentences):
    X, y = [], []

    for tagged in tagged_sentences:
        untagged = [w for w, t in tagged]
        for index in range(len(tagged)):
            X.append(extract_word_features(untagged, index))
            y.append(tagged[index][1])

    return X, y

X, y = transform_to_dataset(train_data)

# Use only the first 10K samples
# numpy.core._exceptions.MemoryError: Unable to allocate 3.21 TiB for an
# array...
X = X[:10000]
y = y[:10000]

clf = Pipeline([
    ('vectorizer', DictVectorizer(sparse=False)),
    ('classifier', LinearSVC())
])

clf.fit(X, y)

print('Training completed')

X_test, y_test = transform_to_dataset(test_data)
X_test = X_test[:10000]
y_test = y_test[:10000]

acc = clf.score(X_test, y_test)
print("Accuracy:", acc)

# Accuracy:  0.9204

MODEL_META["accuracy"] = acc

# save pickle
path = join(dirname(dirname(dirname(__file__))),
            "models", "postag", MODEL_META["model_id"] + ".pkl")
with open(path, "wb") as f:
    pickle.dump(clf, f)
META = join(dirname(dirname(dirname(__file__))), "JarbasModelZoo", "res")
meta_path = join(META, MODEL_META["model_id"] + ".json")
with open(meta_path, "w") as f:
    json.dump(MODEL_META, f)

def pos_tag(sentence):
    tags = clf.predict(
        [extract_word_features(sentence, index)
         for index in range(len(sentence))])
    return zip(sentence, tags)


import os
from collections import namedtuple
from pathlib import Path
import string
import random

from PIL import Image
from PIL.ImageFile import ImageFile

from werkzeug.utils import secure_filename

from configuration import SoccerConfig

Chunk = namedtuple("Chunk", "path url")
STORAGE_PATH = Path(SoccerConfig.STORAGE_PATH)

def url(filename, subdir: str) -> str:
    """Get url file

    Args:
        filename: filename
        subdir: subdirectory

    Returns:
        str url
    """
    if not filename:
        return ""

    static_url = SoccerConfig.STATIC_URL
    if subdir:
        static_url = static_url + "/" + subdir

    return static_url + "/" + filename

def path(filename: str, subdir: str) -> Path:
    """Get full file path

    Args:
        filename: filename
        subdir: subdirectory

    Returns:
        str full path
    """
    upload_dir = Path(SoccerConfig.STORAGE_PATH)
    if subdir:
        upload_dir = upload_dir / subdir

    return upload_dir / filename

def save(file, subdir: str, filename=None, close_after=True) -> str:
    """Save file

    Args:
        file: save able file ex: (FileStorage, ImageFile)
        subdir: sub directory
        filename: filename of file
        close_after: close file after done ?

    Returns:
        str filename
    """
    upload_dir = Path(SoccerConfig.STORAGE_PATH)
    if subdir:
        upload_dir = upload_dir / subdir

    # make sure upload directory exists
    if not upload_dir.is_dir():
        upload_dir.mkdir(parents=True)

    if not filename:
        filename = file.filename

    # make sure filename is safe and no collision
    filename = safe_filename(filename)
    while (upload_dir / filename).is_file():
        filename = safe_filename(file.filename)

    if isinstance(file, ImageFile):
        if file.mode in ('RGBA', 'LA', '1', 'P'):
            file = file.convert("RGB")

        size = (1312, 984) # untuk membatasi ukuran image yang di upload
        if file.size[0] > size[0] or file.size[1] > size[1]:
            file.thumbnail(size, Image.ANTIALIAS)
            file.save(str(upload_dir / filename), quality=90)
        else:
            file.save(str(upload_dir / filename), quality=90)
    else:
        file.save(str(upload_dir / filename))

    if close_after:
        file.close()

    return filename

def safe_filename(filename, maxchar=40):
    """Secure filename and add random string

    Args:
        filename: filename
        maxchar: maximum character

    Example Result:
    [nama_file]_randomgstring - inifilecermin_awkwards.jpg
    """
    name, ext = os.path.splitext(filename)

    random_str = '_' + ''.join(
        random.choice(string.ascii_uppercase + string.ascii_lowercase) for _ in range(10)
    )

    name_max = maxchar - len(ext) - len(random_str)
    if len(name) > name_max:
        name = name[:name_max]

    name = "%s%s%s" % (name, random_str, ext)

    return secure_filename(name)


import math
from typing import Callable
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def GetInterval(start:int,end:int,step:float, fn:Callable[[float],float]) -> dict[str,list[int]]:
    x = np.linspace(start,end,math.ceil((end-start)/step))
    my_dict = {"X":[],"Y":[]}

    for i in x:
        my_dict["X"].append(i)
        my_dict["Y"].append(fn(i))
    return my_dict

def x2(i)->float:
    return i*i

def x3(i)->float:
    return i*i*i

a  = GetInterval(-20,20,1,x2)
b = GetInterval(-20,20,1,x3)

fig, axis = plt.subplots(nrows = 1, ncols = 2, figsize = (15,10))
axis[0].plot(a["X"],a["Y"],color='b',linewidth=4)
axis[1].plot(b["X"],b["Y"],color='r',linewidth=4)

axis[0].set_title("f(x)= x²")
axis[1].set_title("f(x)= x³")

axis[0].grid()
axis[1].grid()

plt.show()

Bosch/Revisão Python - Queila/Arquivos Prova/untitled1.py


# -*- coding: utf-8 -*-

import multiprocessing
import os

def do_this(what):
    whoami(what)

def whoami(what):
    print("进程 %s 说： %s" % (os.getpid(), what))

if __name__ == '__main__':
    whoami('我是主程序')
    for n in range(4):
        p = multiprocessing.Process(target=do_this, args=("我是 #%s 进程" % n,))
        p.start()


'''
using :keyword:`with` statement
'''
from easyprocess import EasyProcess
from pyvirtualdisplay.smartdisplay import SmartDisplay

if __name__ == "__main__":
    with SmartDisplay(visible=0, bgcolor='black') as disp:
        with EasyProcess('xmessage hello'):
            img = disp.waitgrab()

    img.show()


from http import HTTPStatus

from django.test import TestCase
from konekt.forms import CandidateForm, EducationFormset, ExperienceFormset, SkillsFormset

class TestForms(TestCase):

    def test_candidate_form(self):
        form = CandidateForm({
            'name': 'Pierre Dupont',
            'job': 'DevOps',
            'address': 'Paris',
            'phone': '0123465789',
            'email': 'pierre@bti.com',
            'status': 'Freelance',
            'availability': 'Sous 3 mois',
            'mobility': 'Paris',
            'price': '600',
            'salary': ''
        })
        self.assertTrue(form.is_valid())

    def instantiate_formset(self, formset_class, data, instance=None, initial=None):
        prefix = formset_class().prefix
        formset_data = {}
        for i, form_data in enumerate(data):
            for name, value in form_data.items():
                if isinstance(value, list):
                    for j, inner in enumerate(value):
                        formset_data['{}-{}-{}_{}'.format(prefix, i, name, j)] = inner
                else:
                    formset_data['{}-{}-{}'.format(prefix, i, name)] = value
        formset_data['{}-TOTAL_FORMS'.format(prefix)] = len(data)
        formset_data['{}-INITIAL_FORMS'.format(prefix)] = 0

        if instance:
            return formset_class(formset_data, instance=instance, initial=initial)
        else:
            return formset_class(formset_data, initial=initial)

    def test_education_formset(self):
        formset = self.instantiate_formset(EducationFormset, [
            {
                'date': '01/01/2021',
                'name': 'Xxx',
            },
        ])
        self.assertTrue(formset.is_valid())

    def test_experiences_formset(self):
        formset = self.instantiate_formset(ExperienceFormset, [
            {
                'job_title': 'DevOps',
                'company_name': 'BTI',
                'job_location': 'Levallois',
                'duration': '12 mois',
                'description': 'cicd, cloud',
                'tools': 'azure, aws, jenkins',
            },
        ])
        self.assertTrue(formset.is_valid())

    def test_skills_formset(self):
        formset = self.instantiate_formset(SkillsFormset, [
            {
                'name': 'aws',
            },
        ])
        self.assertTrue(formset.is_valid())

class AddCandidateViewTests(TestCase):
    def test_get(self):
        response = self.client.get("/add_candidate/")

        self.assertEqual(response.status_code, HTTPStatus.OK)
        self.assertContains(response, '
BTI Konekt - Ajouter un·e candidat·e
', html=True)


from django.db import models
from django.contrib.auth.models import AbstractUser
from django.utils.translation import gettext_lazy as _
# Create your models here.

def user_avatar_path(user, filename: str):
    return 'avatar/user_{0}/{1}'.format(user.username, filename)

def user_thumbnail_path(user, filename: str):
    return 'avatar/thumbnail/{0}_{1}'.format(user.pk, filename)

class User(AbstractUser):
    email = models.EmailField(_('email address'), unique=True)
    avatar = models.ImageField(
        verbose_name='Avatar',
        # width_field=100,
        # height_field=100,
        upload_to=user_avatar_path,
        null=True,
        blank=True
    )
    birth = models.DateField(
        verbose_name='Birthday',
        null=True,
        blank=True,
    )
    thumbnail = models.ImageField(
        verbose_name='thumbnail',
        # width_field=100,
        # height_field=100,
        upload_to=user_thumbnail_path,
        null=True,
        blank=True
    )

    class Meta:
        verbose_name = 'User'
        verbose_name_plural = verbose_name
        ordering = ('-id', )

    def __str__(self):
        return self.username

    @property
    def avatar_url(self):
        if not self.thumbnail:
            return '/static/image/default_avatar.png'
        else:
            return self.thumbnail.url


import logging
import json

from flask import request, jsonify;

from codeitsuisse import app;

logger = logging.getLogger(__name__)

@app.route('/encryption', methods=['POST'])
def encrypt():
    data = request.get_json();
    logging.info("data sent for evaluation {}".format(data))
    result = []
    for test_case in data:
    	result.append(encry(test_case["n"],test_case["text"]))
    logging.info("My result :{}".format(result))
    return json.dumps(result);

def encry(n,text):
	p_text = ""
	for ch in text:
		if ch.isalnum():
			p_text += ch.upper()

	start = 0
	list_of_substrings = []
	for i in range(n):
		length = len(p_text) // n
		remainder = len(p_text) % n

		if i < remainder:
			length += 1

		substring = p_text[start: start + length]
		list_of_substrings.append(substring)
		start += length
	result = ""
	for i in range(len(p_text)):
		str_index_in_list = i%n
		str_index = i // n
		result += list_of_substrings[str_index_in_list][str_index]
	return result


from django.urls import path, include
from . import views

urlpatterns = [
    path('', views.IndexView.as_view()),
    path('alumno/', views.AlumnoView.as_view(),name='alumnos'),
    path('alumno/<int:alumno_id>', views.AlumnoDetailView.as_view()),
    path('alumno_horario/', views.Alumno_HorarioView.as_view()),  # Corrección de ruta
    path('alumno_horario/<int:alumno_horario_id>', views.Alumno_HorarioDetailView.as_view()),
]


#Data analaysis using Instagram
from IPython.display import Image
from IPython.display import display

from InstagramAPI import InstagramAPI
username="krisha_mehta"
InstagramAPI = InstagramAPI(username,"prideandprejudice")
InstagramAPI.login()

InstagramAPI.getProfileData()
result = InstagramAPI.LastJson
#print(result)
#All the results are stored in Json format

#print (result['user']['biography'])

InstagramAPI.timelineFeed()
#result1 = InstagramAPI.LastJson
#print(result1[text])

#Get_posts_from_list()
#Get_url()
#image_urls = InstagramAPI.LastJson
#image_urls

import time
myposts=[]
has_more_posts = True
max_id=""

while has_more_posts:
    InstagramAPI.getSelfUserFeed(maxid=max_id)
    if InstagramAPI.LastJson['more_available'] is not True:
        has_more_posts = False #stop condition
        print ("stopped")

    max_id = InstagramAPI.LastJson.get('next_max_id','')
    myposts.extend(InstagramAPI.LastJson['items']) #merge lists
    time.sleep(2) # Slows the script down to avoid flooding the servers

profile_pic = myposts[0]['caption']['user']['profile_pic_url']
img = Image(profile_pic)
display(profile_pic)


from datetime import datetime
from db import db

class BookCopieModel(db.Model):
    __tablename__ = 'book_copie'

    id = db.Column(db.Integer, primary_key=True)

    contribution_date = db.Column(
        db.DateTime(timezone=True), default=datetime.utcnow)

    book_id = contributor_user_id = db.Column(db.Integer,
                                              db.ForeignKey('book.id'))
    book = db.relationship('BookModel')

    contributor_user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    contributor_user = db.relationship('UserModel')

    borrower_list = db.relationship('BorrowModel')

    def __init__(self, book_id, contributor_user_id):
        self.book_id = book_id
        self.contributor_user_id = contributor_user_id

    def json(self):
        return {
            'copy_id': self.id,
            'book_isbn': self.book.isbn,
            'contribution_date': str(self.contribution_date),
            'contributor_user_email': self.contributor_user.email,
            'borowers': [x.json() for x in self.borrower_list]
        }

    def save_to_db(self):
        db.session.add(self)
        db.session.commit()

    def delete_from_db(self):
        db.session.delete(self)
        db.session.commit()

    @classmethod
    def find_by_id(cls, _id):
        return cls.query.filter_by(id=_id).first()


import json

def get_historique_user(user):
    # Load existing user data from the file (if any)
    try:
        with open('users.json', 'r') as f:
            users = json.load(f)
    except FileNotFoundError:
        users = []

    # Search for the user in the list
    for u in users:
        if u['user'] == user:
            # If the user exists, return their history as a string
            history_str = ''
            for entry in u.get('history', []):
                history_str += f"{entry['date']} - ({entry['amount']})\n"
            return history_str.strip()
            break

    # If the user is not found, return an empty string
    return "pas d'action sur votre compte."


"""
Implementation of a language predictor based on a transformer architecture
by the wonderful OpenNMT team at Harvard.

SOURCE: OpenNMT: Open-Source Toolkit for Neural Machine Translation

AUTHORS: Guillaume Klein
         Yoon Kim
         Yuntian Deng
         Jean Senellart
         Alexander M. Rush

EDITORS:  Luke Nonas-Hunter
"""

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.rnn import pad_sequence
from torch.autograd import Variable
import math
import copy
import time

class Transformer(nn.Module):
    """
    A standard Encoder-Decoder transformer architecture.

    Attributes:
        encoder: Instance of the encoder module.
        decoder: Instance of the decoder module.
        src_embed: Instance of embeddings module for the source data.
        tgt_embed: Instance of embeddings module for the target data.
        generator: Instance of generator module.
    """

    def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):
        """
        Instantiate the model.
        """
        super(Transformer, self).__init__()
        self.encoder = encoder
        self.decoder = decoder
        self.src_embed = src_embed
        self.tgt_embed = tgt_embed
        self.generator = generator

    def forward(self, src, tgt, src_mask, tgt_mask):
        """
        Take in and process masked src and target sequences.

        Args:
            src: Source data tensor.
            tgt: Target data tensor.
            src_mask: Boolean tensor illustrating what source data should be
                examined at any given timestep.
            tgt_mask: Boolean tensor illustrating what target data should be
                examined at any given timestep.
        Returns:
            Output of the decode element given the output of the encode element
            and the target data (or memory).
        """
        return self.decode(self.encode(src, src_mask),
                           src_mask,
                           tgt,
                           tgt_mask)

    def encode(self, src, src_mask):
        """
        Run values through the encoder module.

        Args:
            src: Source tensor to run through encoder module.
            src_mask: Boolean tensor indicating what data should be examined
                at any given timestep.
        """
        return self.encoder(self.src_embed(src), src_mask)

    def decode(self, memory, src_mask, tgt, tgt_mask):
        """
        Run values through the decoder.

        Args:
            memory: Data tensor that contains previous words predicted by the
                model.
            src_mask: Boolean tensor indicating what source data should be
                examined at any given timestep.
        """
        return self.decoder(self.tgt_embed(tgt), memory, src_mask, tgt_mask)

    def generate_text(self, tokenized_prompt, vocab):
        """
        Generate text given a text prompt and vocabulary.

        Args:
            tokenized_prompt: List of ordered words from the input sentence.
            vocab: Instance of vocabulary object containing all vocabulary from
                the dataset the model was trained on.
        """
        assert len(tokenized_prompt) <= 510
        numerical_prompt = [torch.tensor([vocab.stoi["<SOS>"]]
                                         + vocab.numericalize(tokenized_prompt,
                                                              tokenize=False)
                                         + [vocab.stoi["<EOS>"]]),
                            torch.tensor([0 for _
                                          in range(512)])]
        numerical_prompt = pad_sequence(numerical_prompt,
                                        batch_first=False,
                                        padding_value=0)
        numerical_prompt = numerical_prompt.transpose(0, 1)
        mask = (numerical_prompt[0] != vocab.stoi["<PAD>"]).unsqueeze(-2)
        model_out = self.greedy_decode(self,
                                       numerical_prompt[0], mask,
                                       max_len=256,
                                       start_symbol=vocab.stoi["<SOS>"],
                                       end_symbol=vocab.stoi["<EOS>"])
        text = ""
        for i in range(1, model_out.size(1)):
            sym = vocab.itos[model_out[0, i].item()]
            if sym == "<EOS>":
                break
            text += f"{sym} "
        return text

    @staticmethod
    def clones(module, N):
        """
        Helper: Produce N identical layers.

        Args:
            module: Module to be duplicated.
            N: Integer number of models to duplicate.

        Returns:
            N identical layers.
        """
        return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])

    @staticmethod
    def subsequent_mask(size):
        """
        Helper: Mask out subsequent positions

        Args:
            size: Size of model.
        Returns:
            Tensor representing new mask.
        """
        attn_shape = (1, size, size)
        subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
        return torch.from_numpy(subsequent_mask) == 0

    @staticmethod
    def make_model(src_vocab, tgt_vocab, N=6,
                   d_model=512, d_ff=2048, h=8, dropout=0.1):
        """
        Helper: Construct a model from hyperparameters.

        d_model and d_ff must be divisable by h

        Args:
            src_vocab: Integer number of vocabulary words used in the source
                sentences.
            tgt_vocab: Integer number of vocabulary words used in the target
                sentences.
            N: Integer of decode and encode layers.
            d_model: Integer size of model (determines length of input).
            d_ff: Integer size of the feed forward network.
            h: The number of attention heads.
            dropout: Float representing the rate of node deactivation.
        Return:
            An instance of the Transformer class based on the structure
            outlined by the parameters of the function.
        """
        c = copy.deepcopy
        attn = MultiHeadedAttention(h, d_model)
        ff = PositionwiseFeedForward(d_model, d_ff, dropout)
        position = PositionalEncoding(d_model, dropout)
        model = Transformer(
            Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
            Decoder(DecoderLayer(d_model, c(attn), c(attn),
                                 c(ff), dropout), N),
            nn.Sequential(Embeddings(d_model, src_vocab), c(position)),
            nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
            Generator(d_model, tgt_vocab))

        # Initialize parameters with Glorot / fan_avg.
        for p in model.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform_(p)
        return model

    @staticmethod
    def attention(query, key, value, mask=None, dropout=None):
        """
        Helper: Compute 'Scaled Dot Product Attention'

        Args:
            query: Tensor containing query data.
            key: Tensor containing key data.
            value: Tensor containing value data.
            mask: Boolean tensor illustrating what data should be examined.
            dropout: Float representing the rate of node deactivation.
        Return:
            An output tensor of a single attention head.
        """
        d_k = query.size(-1)
        scores = torch.matmul(query, key.transpose(-2, -1)) \
            / math.sqrt(d_k)
        if mask is not None:
            scores = scores.masked_fill(mask == 0, -1e9)
        p_attn = F.softmax(scores, dim=-1)
        if dropout is not None:
            p_attn = dropout(p_attn)
        return torch.matmul(p_attn, value), p_attn

    @staticmethod
    def greedy_decode(model, src, src_mask, max_len,
                      start_symbol, end_symbol=None):
        """
        Helper: Calculate the best word given the model output.

        Args:
            model: Instance of the Transformer class.
            src: Source data (also refered to as the prompt).
            src_mask: Boolean list illustrating what data should be examined.
            max_len: Maximum tokens in output
            start_symbol: Token indicating start of sentence.
            end_symbol: Token indicating end of sentence.
        Return:
            A tensor containing the index representation of the output
            sentence generated by the model given the original prompt.
        """
        memory = model.encode(src, src_mask)
        ys = torch.ones(1, 1).fill_(start_symbol).type_as(src.data)
        for i in range(max_len - 1):
            out = model.decode(memory, src_mask,
                               Variable(ys),
                               Variable(Transformer.subsequent_mask(ys.size(1))
                                        .type_as(src.data)))
            prob = model.generator(out[:, -1])
            _, next_word = torch.max(prob, dim=1)
            next_word = next_word.data[0]
            ys = torch.cat([ys,
                            torch.ones(1, 1)
                            .type_as(src.data)
                            .fill_(next_word)], dim=1)
            if end_symbol is not None and next_word == end_symbol:
                break
        return ys

class Generator(nn.Module):
    """
    Define standard linear + softmax generation step.

    Attributes:
        proj: A linear feed forward module which takes the output of the decode
            block and projects it onto a series of nodes which represents all
            the possible vocabulary words the model knows.
    """

    def __init__(self, d_model, vocab):
        """
        Instantiate standard linear + softmax neural network.

        Args:
            d_model: Integer dimension of model input.
            vocab: Integer size of vocabulary
        """
        super(Generator, self).__init__()
        self.proj = nn.Linear(d_model, vocab)

    def forward(self, x):
        """
        Feed data through the layer.

        Args:
            x: The data to be passed through the network.
        Returns:
            The output of the self.proj layer after being processed by a
            softmax function.
        """
        return F.log_softmax(self.proj(x), dim=-1)

class Encoder(nn.Module):
    """
    Core encoder is a stack of N layers.

    Attributes:
        layers: List of encoder layers which make up the module.
        norm: Instance of LayerNorm.
    """

    def __init__(self, layer, N):
        """
        Instantiate encoder module.

        Args:
            layer: Instance of EncoderLayer to be duplicated.
            N: Integer of times to duplicate the given layer.
        """
        super(Encoder, self).__init__()
        self.layers = Transformer.clones(layer, N)
        self.norm = LayerNorm(layer.size)

    def forward(self, x, mask):
        """
        Pass the input (and mask) through each later in turn.

        Args:
            x: Source data tensor.
            mask: Boolean tensor illustrating what data should be examined.
        Returns:
            Output of all the encoder layers.
        """
        for layer in self.layers:
            x = layer(x, mask)
        return self.norm(x)

class LayerNorm(nn.Module):
    """
    Construct a normalization layer.

    Attributes:
        a_2: Parameter tensor which represents the constant that scales the
            output of the normalization function.
        b_2: Parameter tensor which represents the constant that offsets the
            output of the normalization function.
        eps: Float point representing learning rate.
    """

    def __init__(self, features, eps=1e-6):
        """
        Instantiate layer norm.

        Args:
            features: Integer number of features in layer.
            eps: Floating point representation of learning rate.
        """
        super(LayerNorm, self).__init__()
        self.a_2 = nn.Parameter(torch.ones(features))
        self.b_2 = nn.Parameter(torch.zeros(features))
        self.eps = eps

    def forward(self, x):
        """
        Feed data through the layer.

        Args:
            x: Source data tensor.
        Returns:
            Normalized data.
        """
        mean = x.mean(-1, keepdim=True)
        std = x.std(-1, keepdim=True)
        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2

class SublayerConnection(nn.Module):
    """
    A residual connection followed by a layer norm.
    Note for code simplicity the norm is first as opposed to last.

    Attributes:
        norm: Instance of LayerNorm.
        dropout: Dropout function defined by pytorch.
    """

    def __init__(self, size, dropout):
        """
        Initalize a sublayer connection.

        Args:
            size: Size of the layer.
            dropout: Floating point representation of dropout rate.
        """
        super(SublayerConnection, self).__init__()
        self.norm = LayerNorm(size)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, sublayer):
        """
        Apply residual connection to any sublayer with same size.

        Args:
            x: Source tensor.
            sublayer: Previous layer to run data through.
        """
        return x + self.dropout(sublayer(self.norm(x)))

class EncoderLayer(nn.Module):
    """
    Assemble an encoder layer for use inside an encoder.

    Attributes:
        self_attn: Instance of the MultiHeadedAttention class.
        feed_forward: Linear feed forward neural network.
        sublayer: List of SublayerConnection modules which normalize the data.
        size: Size of the model.
    """

    def __init__(self, size, self_attn, feed_forward, dropout):
        """
        Instantiate an encoder layer.

        Args:
            size: Integer size of the layer.
            self_attn: Instance of the MultiHeadedAttention class.
            feed_forward: Linear feed forward neural network.
            dropout: Floating point representation of dropout rate.
        """
        super(EncoderLayer, self).__init__()
        self.self_attn = self_attn
        self.feed_forward = feed_forward
        self.sublayer = Transformer.clones(SublayerConnection(size, dropout), 2)
        self.size = size

    def forward(self, x, mask):
        """
        Feed data through layer.

        Args:
            x: Source data tensor.
            mask: Boolean tensor illustrating what data should be examined.
        """
        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))
        return self.sublayer[1](x, self.feed_forward)

class Decoder(nn.Module):
    """
    Generic N layer decoder with masking.

    Attributes:
        layers: List of decoder layers which make up the module.
        norm: Instance of LayerNorm.
    """

    def __init__(self, layer, N):
        """
        Instantiate a decoder module.

        Args:
            layer: Instance of DecoderLayer to be duplicated.
            N: Integer of times to duplicate the given layer.
        """
        super(Decoder, self).__init__()
        self.layers = Transformer.clones(layer, N)
        self.norm = LayerNorm(layer.size)

    def forward(self, x, memory, src_mask, tgt_mask):
        """
        Feed data through the decoder.

        This includes running the data through all the decoder layers and the
        normalization later.

        Args:
            x: Source data tensor.
            memory: Tensor containing words generated by the model so far.
            src_mask: Boolean tensor illustrating what source data should be
                examined.
            tgt_mask: Boolean tensor illustrating what memory data should be
                examined.
        Returns:
            Output from the decoder.
        """
        for layer in self.layers:
            x = layer(x, memory, src_mask, tgt_mask)
        return self.norm(x)

class DecoderLayer(nn.Module):
    """
    Decoder layer is made of self-attn, src-attn, and feed forward.

    Attributes:
        size: Integer dimension of model.
        self_attn: Instance of the MultiHeadedAttention class.
        src_attn: Instance of the MultiHeadedAttention class.
        feed_forward: Linear feed forward neural network.
        sublayer: List of connecting sublayers.
    """

    def __init__(self, size, self_attn, src_attn, feed_forward, dropout):
        """
        Instantiate a decoder layer.

        Args:
            size: Integer dimension of model.
            self_attn: Instance of the MultiHeadedAttention class.
            src_attn: Instance of the MultiHeadedAttention class.
            feed_forward: Linear feed forward neural network.
            dropout: Floating point representation of dropout rate.
        """
        super(DecoderLayer, self).__init__()
        self.size = size
        self.self_attn = self_attn
        self.src_attn = src_attn
        self.feed_forward = feed_forward
        self.sublayer = Transformer.clones(SublayerConnection(size, dropout), 3)

    def forward(self, x, memory, src_mask, tgt_mask):
        """
        Feeds data through decoder layer.

        Args:
            x: Source data tensor.
            memory: Tensor containing words generated by the model so far.
            src_mask: Boolean tensor illustrating what source data should be
                examined.
            tgt_mask: Boolean tensor illustrating what memory data should be
                examined.
        Returns:
            Output of a single decoder layer.
        """
        m = memory
        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))
        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))
        return self.sublayer[2](x, self.feed_forward)

class MultiHeadedAttention(nn.Module):
    """
    Combines multiple attention functions together into one layer.

    Attributes:
        d_k: Integer dimension of query, key, and value vectors.
        h: Integer number of attention heads.
        linears: List of linear feed forward layers.
        attn: Function representing a single attention head.
    """

    def __init__(self, h, d_model, dropout=0.1):
        """
        Take in model size and number of heads and instantiate
        MultiHeadedAttention object.

        Args:
            h: Integer number of attention heads.
            d_model: Integer dimension of model.
            dropout: Floating point representation of dropout rate.
        """
        super(MultiHeadedAttention, self).__init__()
        assert d_model % h == 0
        # We assume d_v always equals d_k
        self.d_k = d_model // h
        self.h = h
        self.linears = Transformer.clones(nn.Linear(d_model, d_model), 4)
        self.attn = None
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, query, key, value, mask=None):
        """
        Feed data through a multiheaded attention layer.

        Args:
            query: Data tensor of query words.
            key: Data tensor of key words.
            value: Data tensor of value words.
            mask: Boolean tensor which indicates which data should be excluded.
        Returns:
            Output of multiheaded attention layer.
        """
        if mask is not None:
            # Same mask applied to all h heads.
            mask = mask.unsqueeze(1)
        nbatches = query.size(0)

        # 1) Do all the linear projections in batch from d_model => h x d_k
        query, key, value = \
            [linear(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
                for linear, x in zip(self.linears, (query, key, value))]

        # 2) Apply attention on all the projected vectors in batch
        x, self.attn = Transformer.attention(query, key, value, mask=mask,
                                             dropout=self.dropout)

        # 3) "Concat" using a view and apply a final linear
        x = x.transpose(1, 2).contiguous() \
            .view(nbatches, -1, self.h * self.d_k)
        return self.linears[-1](x)

class PositionwiseFeedForward(nn.Module):
    """
    Implementats FFN.

    Attributes:
        w_1: First liner layer in the network.
        w_2 Second linear layer in the network
        dropout: Dropout function defined by pytorch.
    """

    def __init__(self, d_model, d_ff, dropout=0.1):
        """
        Instantiate PositionwiseFeedForward object.

        d_model: Integer dimension of the input/output layers.
        d_ff: Integer dimension of the hidden layers.
        dropout: Floating point representation of dropout layer.
        """
        super(PositionwiseFeedForward, self).__init__()
        self.w_1 = nn.Linear(d_model, d_ff)
        self.w_2 = nn.Linear(d_ff, d_model)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        """
        Feed data through the network.

        Args:
            x: Source data tensor.
        Returns:
            Output of the feed forward network.
        """
        return self.w_2(self.dropout(F.relu(self.w_1(x))))

class Embeddings(nn.Module):
    """
    Embed text as vectors.

    Attributes:
        lut: A look up table defined by the pytorch Embedding module.
        d_model: Integer dimension of the model.
    """

    def __init__(self, d_model, vocab):
        """
        Instantiates an embedding module.

        d_model: Integer dimension of the model.
        vocab: Integer number of vocabulary words.
        """
        super(Embeddings, self).__init__()
        self.lut = nn.Embedding(vocab, d_model)
        self.d_model = d_model

    def forward(self, x):
        """
        Feed data through the embedding module.

        Args:
            x: Source data tensor.
        Returns:
            Output of embeddings layer.
        """
        return self.lut(x) * math.sqrt(self.d_model)

class PositionalEncoding(nn.Module):
    """
    Implement the PE function.

    Attributes:
        dropout: Dropout function defined by pytorch.
    """

    def __init__(self, d_model, dropout, max_len=5000):
        """
        Instantiate positional encoding module.

        Args:
            d_model: Integer dimension of model.
            dropout: Floating point representation of dropout rate.
            max_len: Maximum position value.
        """
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        # Compute the positional encodings once in log space
        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2) *
                             -(math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x):
        """
        Feed data through the positional encoding layer.

        Args:
            x: Source data tensor.
        Returns:
            The source data now encoded with each words relative position in
            the sentence.
        """
        x = x + Variable(self.pe[:, :x.size(1)],
                         requires_grad=False)
        return self.dropout(x)

if __name__ == "__main__":
    tmp_model = Transformer.make_model(10, 10, 2)


##### Sales Prediction with Linear Regression

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

pd.set_option("display.float_format", lambda x: "%.2f" % x)

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.model_selection import train_test_split, cross_val_score

##### Simple Linear Regression with OLS Using Scikit-Learn

df = pd.read_csv("advertising.csv")

print(df.shape)

print(df.describe().T)

##### Model

X = df[["TV"]]
y = df[["sales"]]

reg_model = LinearRegression().fit(X, y)

## b + w*TV
# tv katsayısı (w)
print(reg_model.coef_[0][0])

# sabit (b)
print(reg_model.intercept_[0])

##### Tahmin

# 150 birimlik TV harcaması olsa satıs ne olur?
print(reg_model.intercept_[0] + reg_model.coef_[0][0] * 150)

## modelin görselleştirilmesi
vis = sns.regplot(x=X, y=y, scatter_kws={"color": "g", "s": 9},
                  ci=False, color="b")  # model tahmin değerleri mavi, gerçek değerler yesil
vis.set_title(f"Model Denklemi: Sales = {round(reg_model.intercept_[0], 2)} + TV*{round(reg_model.coef_[0][0], 2)}")
vis.set_ylabel("Satıs Sayısı")
vis.set_xlabel("TV Harcamaları")
plt.xlim(-10, 310)
plt.ylim(bottom=0)
plt.show()

##### Tahmin Başarısı

# MSE
y_pred = reg_model.predict(X)  # tahmin edilen değerler
mean_squared_error(y, y_pred)  # gerçek değerler ve tahmin değerleri prmteler

y.mean()  # gercek bagımlı değişken değerlerin ort

y.std()  # gercek bagımlı değişken degerlerin std sapması, degerler 9-19 arasında değişiyor

# RMSE
np.sqrt(mean_squared_error(y, y_pred))

# MAE
mean_absolute_error(y, y_pred)

# R-KARE, veri setindeki bagımsız degisklerin bagımlı degiskeni acıklama yuzdesidir
print(reg_model.score(X, y))

##### *Değişken sayısı arttıkca R-kare şişebilir. bu yuzden duzeltilmiş r karenin de değeri göz onune alınmalıdır.
###### *Katsayıların anlamlılığı, modelin anlamlılığı ile ilgilenmiyoruz, optimizasyon, makine öğrenmesi açısından yüksek tahmin başarısıyla ilgileniyoruz. doğrusal formda tahmin etmek.
##### *Gelişmiş regresyon problemleri çözmek için doğrusal değil ağaca dayalı yöntemleri daha cok kullanacağız.

##### Multiple Linear Regression

X = df.drop("sales", axis=1)
y = df[["sales"]]

##### Model

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

print(X_test.shape)
print(y_train.shape)

reg_model = LinearRegression().fit(X_train, y_train)

# b
print(reg_model.intercept_[0])

# w
print(reg_model.coef_[0])

##### Tahmin

# regresyon cıktısı
print(reg_model.intercept_[0] + reg_model.coef_[0][0] * 30 + reg_model.coef_[0][1] * 10 + reg_model.coef_[0][2] * 40)

veri_deneme = [[30], [10], [40]]
veri_deneme = pd.DataFrame(veri_deneme).T
reg_model.predict(veri_deneme)  # tahmin

###### Tahmin Başarısı

y_pred = reg_model.predict(X_train)
np.sqrt(mean_squared_error(y_train, y_pred))  # train RMSE

reg_model.score(X_train, y_train)  # train R-Kare

y_pred = reg_model.predict(X_test)
np.sqrt(mean_squared_error(y_test, y_pred))  # test RMSE

reg_model.score(X_test, y_test)  # test R-Kare

# 10 katlı cv RMSE
np.mean(np.sqrt(-cross_val_score(reg_model, X, y, cv=10, scoring="neg_mean_squared_error")))

# crossun basında - olmasının sebebi negatif sonuclar vermesi, negatif hata olmayacağından - ile carpılarak pozitif sayılar elde edilir.
# veri setinin boyutu az oldugundan capraz dogrulamaya bakmak daha mantıklı olabilir

### BONUS

##### Simple Linear Regression with Gradient Descent from Scratch

# Cost function MSE
def cost_function(Y, b, w, X):
    num_of_ob = len(Y)
    sse = 0

    for i in range(0, num_of_ob):
        y_pred = b + w * X[i]  # bagimli tahmin degerleri
        y_sum = (Y[i] - y_pred) ** 2
        sse += y_sum

    mse = sse / num_of_ob
    return mse

# gradient descent
def update_weights(Y, b, w, X, learning_rate):
    num_of_ob = len(Y)
    b_derivative_sum = 0
    w_derivative_sum = 0

    for i in range(0, num_of_ob):
        y_pred = b + w * X[i]  # bagimli tahmin degerleri
        y = Y[i]  # bagimli gercek degerleri
        b_derivative_sum += (y_pred - y)  # b ye gore kismi turev
        w_derivative_sum += (y_pred - y) * X[i]  # w ye gore kismi turev

    new_b = b - (learning_rate * b_derivative_sum * 1 / num_of_ob)
    new_w = w - (learning_rate * w_derivative_sum * 1 / num_of_ob)
    return new_b, new_w

# train fonksiyonu
def train(Y, initial_b, initial_w, X, learning_rate, num_iters):
    print('b:{} w:{} MSE:{}'.format(initial_b, initial_w, cost_function(Y, initial_b, initial_w, X)))

    b = initial_b
    w = initial_w
    cost_h = []

    for i in range(num_iters):
        b, w = update_weights(Y, b, w, X, learning_rate)
        mse = cost_function(Y, b, w, X)
        cost_h.append(mse)
        if i % 500 == 0:
            print('iter:{} b:{:.2f}, w:{:.2f}, MSE:{:.2f}'.format(i, b, w, mse))

    print('after {} iterations b: {:.2f}, w: {:.2f}, MSE: {:.2f}'.format(num_iters, b, w, cost_function(Y, b, w, X)))
    return cost_h, b, w

df = pd.read_csv('advertising.csv')
X = df['radio']
Y = df['sales']

# hyperparameters

initial_b = 0.001
initial_w = 0.001
learning_rate = 0.001
num_iters = 10000

cost_h, b, w = train(Y, initial_b, initial_w, X, learning_rate, num_iters)


from flask import Flask, request, jsonify, make_response
from functools import lru_cache
import json

app = Flask(__name__)

"""
comma_separtaed_params_to_list splits string when comma occurs and append splitted tokens to a list
input_str: input string
Returns: a list of tokens separated by commas
"""

def comma_separated_params_to_list(input_str: str) -> list:
    result = []
    for val in input_str.split(','):
        if val:
            result.append(val)
    return result

@app.route('/api/ping')
def get_ping():
    return make_response(jsonify({'success': True}), 200)

class Posts:
    JSON_FIlE = "data.json"

    @classmethod
    @lru_cache()
    def load(cls):
        with open(cls.JSON_FIlE, "r") as postsFile:
            return json.load(postsFile).get("posts", [])

    @classmethod
    def write(cls, posts: list):
        with open(cls.JSON_FIlE, "w") as postsFile:
            json.dump({"posts": posts}, postsFile, indent=2)

    @classmethod
    def get_by_id(cls, posts: list, post_id: str):
        return next(filter(lambda pos: pos.get("id") == post_id, posts), None)

    @classmethod
    def get_by_tag(cls, posts: list, post_tags: list) -> list:
        ret = []
        for post in posts:
            for tag in post_tags:
                if tag in post.get("tags"):
                    # print(post.get("id"), tag, ret)
                    ret.append(post)
                    break
        return ret

    @classmethod
    def sort_by_param(cls, posts: list, param: str, reverse: bool) -> list:
        return sorted(posts, key=lambda k: k[param], reverse=reverse)

@app.route('/api/posts', methods=['GET', 'POST'])
def get_post_posts():
    if request.method == "POST":
        print("POST")
        data = request.get_json()
        posts = Posts.load()
        existing_post = Posts.get_by_id(posts, data.get("id"))

        if existing_post:
            return {"error": "Post with same id already exists"}, 400

        posts.append(data)
        Posts.write(posts)
        return "", 201

    if request.method == "GET":
        print("GET")
        response = {}
        post_tag = request.args.getlist('tags') or request.form.getlist('tags')

        if len(post_tag) == 1 and ',' in post_tag[0]:
            post_tag = comma_separated_params_to_list(post_tag[0])

        sort_by = request.args.get('sortBy', default="id")
        direction = request.args.get('direction', default="asc")

        if not post_tag:
            return {"error": "Tags parameter is required"}, 400

        if sort_by not in ["id", "reads", "likes", "popularity"]:
            return {"error": "sortBy parameter is invalid"}, 400

        if direction not in ["asc", "desc"]:
            return {"error": "direction parameter is invalid"}, 400

        reverse = False if direction == "asc" else True

        posts = Posts.load()
        post = Posts.get_by_tag(posts, post_tag)
        post = Posts.sort_by_param(post, sort_by, reverse)

        if post:
            response["posts"] = post
        return response
    return None

if __name__ == "__main__":
    app.run(debug=True, host='0.0.0.0', port=80)


#!/usr/bin/python3

import sys

from q import run, run_text

if len(sys.argv) == 1:
    while True:
        text = input('> ')
        if text == 'exit': break
        print(run_text(text))
elif len(sys.argv) == 2:
    run(sys.argv[1])
else:
    print('Usage: q [file]')


import glob
import os
import json
import re
import tqdm
import pickle
import pandas as pd
import arguments as args

files = glob.glob(os.path.join(args.scopes_dir, '*.scopes.json'))
os.makedirs(args.cooc_dir, exist_ok=True)

def str_normalize(value):
    return re.sub(r'\s+', ' ', value).lower()

def get_item_string_by_coveredtext(scope, item):
    p = item['properties']
    begin, end = p['begin'], p['end']
    return str_normalize(scope['text'][begin:end])

def get_item_string(scope, item, prop):
    return item['properties'][prop] if prop else get_item_string_by_coveredtext(scope, item)

cooccurrences = {}
for file in tqdm.tqdm(files):
    file_id = os.path.basename(file).split('.')[0]
    with open(file, encoding='utf-8') as f:
        data = json.load(f)
    scopes = [d for d in data if d['type'] == args.scope_type]
    for scope_id, scope in enumerate(scopes):
        if args.event_type and not any([a for a in scope.get('sub', []) if a['type'] == args.event_type]):
            continue
        items = sorted([a for a in scope.get('sub', []) if a['type'] == args.item_type], key=lambda a: a['properties']['begin'])
        cooccurrences_ = set()
        for i in range(len(items) - 1):
            for j in range(i + 1, len(items)):
                cooc = tuple(get_item_string(scope, items[i_], args.item_property) for i_ in [i, j])
                cooccurrences_.add(cooc)
        for cooc in cooccurrences_:
            cooccurrences.setdefault(cooc, []).append((file_id, scope_id))

# save cooccurrences with scopes ids
with open(os.path.join(args.cooc_dir, 'cooccurrences.scope_ids.pkl'), mode='wb') as f:
    pickle.dump(cooccurrences, f)

# build dataframe
processed = set()
data = []
for a, b in cooccurrences:
    if (b, a) not in processed:
        forward_count = len(cooccurrences[(a, b)])
        backward_count = len(cooccurrences.get((b, a), []))
        total_count = forward_count + backward_count
        ratio = forward_count / total_count
        data.append((a, b, forward_count, backward_count, total_count, ratio))
        processed.add((a, b))

df = pd.DataFrame(data, columns=['item_left', 'item_right', 'forward_count', 'backward_count', 'total_count', 'ratio'])

df.to_csv(os.path.join(args.cooc_dir, 'cooccurrences.csv'))


import numpy as np
import h5py as hp
import sys

first = sys.argv[1]
second = sys.argv[2]
third= sys.argv[3]
fourth = sys.argv[4]

process = sys.argv[5]
grid = (2048,2048,2048)

def get_mass(path):
    try:
        f=hp.File(path,'r')
    except IOError:
        print('files not found')
        return np.zeros(grid, dtype=np.float32)
    else:
        try:
            mass = f["mass"][:].astype(np.float32)
        except KeyError:
            print('mass field not found - creating substitute')
            return np.zeros(grid, dtype=np.float32)
        else:
            print('correctly found the mass')
            return mass

total = get_mass('subtotal4_ptl_'+first+'.hdf5')
print(sys.getsizeof(total))
sf = get_mass('subtotal4_ptl_'+second+'.hdf5')
total = np.add(total,sf,dtype=np.float32)
del sf
sf = get_mass('subtotal4_ptl_'+third+'.hdf5')
total = np.add(total,sf, dtype=np.float32)
del sf
sf = get_mass('subtotal3_ptl_13.hdf5')
total = np.add(total,sf,dtype=np.float32)
del sf
print('finished adding, new file:')
print(sys.getsizeof(total))
w = hp.File('final_ptl_'+process+'.hdf5','w')
w.create_dataset("mass",data=total)

#is this working


from collections import Counter

def MostPopularNumbers(array, size):
    c = Counter(array)
    values = c.most_common(size)
    return min(values)

a = [1, 1, 2, 2, 3, 4, 5, 6]
MostPopularNumbers(a, a.count)


from __future__ import print_function
import os,sys,inspect
from termcolor import colored

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)

import models as Models
import global_vars as Global
from utils.args import args

import categories.classifier_setup as CLSetup
import categories.kway_logistic_setup as KLogisticSetup
import categories.deep_ensemble_setup as DeepEnsembleSetup
import categories.ae_setup as AESetup
import categories.pixelcnn_setup as PCNNSetup

if args.exp != 'model_ref':
    print(colored('The exp is NOT model_ref!', 'yellow'))

def needs_processing(args, dataset_class, models, suffix):
    """
        This function checks whether this model is already trained and can be skipped.
    """
    for model in models:
        for suf in suffix:
            home_path = Models.get_ref_model_path(args, model.__name__, dataset_class.__name__, model_setup=True, suffix_str=suf)
            hbest_path = os.path.join(home_path, 'model.best.pth.done')
            if not os.path.isfile(hbest_path):
                return True
    return False

task_list = [
    # The list of models,   The function that does the training,    Can I skip-test?,   suffix of the operation.
    # The procedures that can be skip-test are the ones that we can determine
    # whether we have done them before without instantiating the network architecture or dataset.
    # saves quite a lot of time when possible.
    (Global.dataset_reference_classifiers, CLSetup.train_classifier,            True, ['base']),
    (Global.dataset_reference_classifiers, KLogisticSetup.train_classifier,     True, ['KLogistic']),
    (Global.dataset_reference_classifiers, DeepEnsembleSetup.train_classifier,  True, ['DE.%d'%i for i in range(5)]),
    (Global.dataset_reference_autoencoders, AESetup.train_BCE_AE,               False, []),
    (Global.dataset_reference_autoencoders, AESetup.train_MSE_AE,               False, []),
    (Global.dataset_reference_vaes, AESetup.train_variational_autoencoder,      False, []),
    (Global.dataset_reference_pcnns, PCNNSetup.train_pixelcnn,                  False, []),
]

# Do a for loop to run the training tasks.
for task_id, (ref_list, train_func, skippable, suffix) in enumerate(task_list):
    target_datasets = ref_list.keys()
    print('Processing %d datasets.'%len(target_datasets))
    for dataset in target_datasets:
        print('Processing dataset %s with %d networks for %d-%s.'%(colored(dataset, 'green'), len(ref_list[dataset]), task_id, colored(train_func.__name__, 'blue')))
        if skippable and not needs_processing(args, Global.all_datasets[dataset], ref_list[dataset], suffix=suffix):
            print(colored('Skipped', 'yellow'))
            continue
        ds = Global.all_datasets[dataset]()
        for model in ref_list[dataset]:
            model = model()
            print('Training %s'%(colored(model.__class__.__name__, 'blue')))
            train_func(args, model, ds.get_D1_train())


# -*- coding: utf-8 -*-
'''从自己选出的优质股中，读取excel值，查看每个股票的K图，和证券公司的研报，筛选出好的股票存入excel'''
import time
import pandas as pd
from selenium import webdriver
from openpyxl import load_workbook

class ShowK:

    def __init__(self):
        self.share_list0 = []
        self.share_list1 = []

    def read_from_excel(self):
        wb = load_workbook("C:\\Users\\Zhouxiong\\OneDrive\\桌面\\share_of_fund.xlsx")
        ws = wb.get_sheet_by_name('before_check_k')
        first_column = ws['A']
        for x in range(len(first_column)):
            print(first_column[x].value)
            self.share_list0.append(first_column[x].value)
        # return share_list0

    @staticmethod
    def save_to_excel(share_list1):
        wb = load_workbook('C:\\Users\\Zhouxiong\\OneDrive\\桌面\\share_of_fund.xlsx')
        ws = wb.get_sheet_by_name('after_check_k')
        j = ws.max_column
        for z in range(len(share_list1)):
            ws.cell(row=z+1, column=j+1).value = share_list1[z]
        wb.save("C:\\Users\\Zhouxiong\\OneDrive\\桌面\\share_of_fund.xlsx")

    # def save_to_excel():
    #     df1 = pd.DataFrame({
    #         's_name': s_name,
    #         's_code': s_code
    #     })
    #     df1.to_excel('C:/Users/Zhouxiong/OneDrive/桌面/K1.xlsx', sheet_name='after_check')

    def begin(self):
        # s_name = []
        # s_code = []
        # d = pd.read_excel('C:/Users/Zhouxiong/OneDrive/桌面/K0.xlsx', sheet_name='before_check')

        driver = webdriver.Chrome(r'C:\Users\Zhouxiong\AppData\Local\Google\Chrome\Application\chromedriver.exe')  # 打开谷歌浏览器
        num = 0
        self.read_from_excel()
        print(self.share_list0)
        for i in self.share_list0:
            try:
                driver.get('http://quote.cfi.cn/quote_'+str(i)+'.html')
                time.sleep(1)
                driver.find_element_by_link_text(u"日K").click()
                time.sleep(5)
                driver.find_element_by_link_text(u"周K").click()
                time.sleep(5)
                driver.find_element_by_link_text(u"月K").click()
                time.sleep(5)
                driver.find_element_by_link_text(u"研报一览").click()
                time.sleep(5)
                c = input('该股票是否优质,y or n:')
                if c == 'y':
                    num += 1
                    self.share_list1.append(i)
            except:
                print(''+i+'网页获取失败')
                continue
        print('选出的股票数为:'+str(num)+'')
        self.save_to_excel(self.share_list1)
        driver.quit()

if __name__ == '__main__':
    show_k = ShowK()
    show_k.begin()