from sklearn.manifold import Isomap
from sklearn.decomposition import FastICA
import statsmodels.api as sm
import sys
from typing import Tuple, Optional, Dict, Callable, Union

# JAX SETTINGS
import jax
import jax.numpy as np
import jax.random as random
from jax.ops import index, index_add, index_update
from jax.scipy.special import ndtri
import numpyro
import numpyro.distributions as dist
from numpyro.infer import SVI
from jax.scipy import linalg
from jax.scipy.stats import norm
from jax import grad, value_and_grad
from jax.experimental import optimizers as jax_opt
from funcs_LNC import *

# NUMPY SETTINGS
import numpy as onp
import time
onp.set_printoptions(precision=3, suppress=True)

# LOGGING SETTINGS
import logging
logging.basicConfig(
    level=logging.INFO,
    stream=sys.stdout,
    format='%(asctime)s:%(levelname)s:%(message)s'
)
logger = logging.getLogger()
#logger.setLevel(logging.INFO)

@jax.jit
def rrModel(lam, X, y, ws):

    L = linalg.cho_factor(np.dot(X.T,X) + lam * np.diag(1 / ws))

    # weights
    weights = linalg.cho_solve(L, np.dot(X.T,y))

    # save the params

    y_hat = np.dot(X, weights)

    resids = y - y_hat

    # return the predictions
    return weights, resids, y_hat

@jax.jit
def linRegCoefStat(lam, X, y, ws):

    invMat = np.linalg.inv(np.dot(X.T, X)+ lam * np.diag(1 / ws))
    #L = linalg.cho_factor(np.dot(X.T,X) + lam * np.diag(1 / ws))

    # weights
    #weights = linalg.cho_solve(L, np.dot(X.T,y))
    weights = np.dot(invMat, np.dot(X.T,y))

    # save the params

    y_hat = np.dot(X, weights)

    resids = y - y_hat
    mses = mse(y, y_hat)

    var_b = mses*(invMat).diagonal()
    sd_b = np.sqrt(var_b)

    ts_b = weights[:,0] / sd_b
    return ts_b

@jax.jit
def getDataLoss_LNC_lin(smpl, loss_data):

    x, y, Z, U, U_ica,  K_u, D_x, K_y, D_y, K_x, idxs, beta_real, stds, idx_min, idx_x_c, idx_y_c, mixMat = loss_data

    D_x_aux = D_x[smpl, :]
    D_x_aux = D_x_aux[:, smpl]
    K_x_aux = K_x[smpl, :]
    K_x_aux = K_x_aux[:, smpl]
    K_y_aux = K_y[smpl,:]
    K_y_aux = K_y_aux[:,smpl]
    K_u_aux = K_u[smpl,:]

    U_aux = U[smpl,:]
    U_ica_aux = U_ica[smpl,:]

    x_aux = x[smpl,]
    y_aux = y[smpl,]

    if Z is not None:
        Z_aux = Z[smpl,]
    else:
        Z_aux = None

    return x_aux, y_aux, Z_aux, U_aux, U_ica_aux, K_u_aux, D_x_aux, K_y_aux, None, K_x, idxs, beta_real, stds, idx_min, idx_x_c, idx_y_c, mixMat

def getIniPar_LNC_lin(reps, loss_data, pars, smplsParts): #N, m, reps, y, M

    _, _, lam, _ , _ = pars
    x, y, Z, U, U_ica,  K_u, D_x, K_y, D_y, K_x, idxs, beta_real, stds, idx_min, idx_x_c, idx_y_c, mixMat = loss_data

    n = x.shape[0]

    # initialize z_x to x, z_y to y and z_c manifold of z_c
    # we actually have to initialize alpha_x, alpha_y and alpha_c so in each case
    # we have to work out what z = U*alpha -> cholesky decomposition of U

    #zx_ini = x
    #zy_ini = y
    xy = onp.hstack([x, y])

    #embedding = Isomap(n_components=10)
    #xy_transformed = embedding.fit_transform(xy)
    #zc_ini = np.array(xy_transformed[:,3])[:,None]

    zc_ini = np.array(U_ica[:,idx_min][:,None])
    p_ica = U_ica.shape[1]

    alpha_x_ini = onp.zeros((p_ica, p_ica))
    alpha_x_ini[idx_x_c, idx_x_c] = 1
    alpha_x_ini = alpha_x_ini[:,idx_x_c]
    alpha_x_ini = np.array(alpha_x_ini)

    alpha_y_ini = onp.zeros((p_ica, p_ica))
    alpha_y_ini[idx_y_c, idx_y_c] = 1
    alpha_y_ini = alpha_y_ini[:,idx_y_c]
    alpha_y_ini = np.array(alpha_y_ini)

    zx_ini = (U_ica@alpha_x_ini)
    zy_ini = (U_ica@alpha_y_ini)

    p = U.shape[1]
    L = linalg.cho_factor(np.dot(U.T, U) + lam * np.eye(p))
    #alpha_x_ini = linalg.cho_solve(L, np.dot(U.T, zx_ini))
    #alpha_y_ini = linalg.cho_solve(L, np.dot(U.T, zy_ini))
    alpha_c_ini = linalg.cho_solve(L, np.dot(U.T, zc_ini))

    params = {
            'alpha_x': alpha_x_ini,
            'alpha_y': alpha_y_ini,
            'alpha_c': alpha_c_ini,
    }

    #onp.random.seed(seed=4)

    #zx_ini = (zx_ini - np.mean(zx_ini)) / (np.std(zx_ini))
    #zy_ini = (zy_ini - np.mean(zy_ini)) / (np.std(zy_ini))
    #zc_ini = (zc_ini - np.mean(zc_ini)) / (np.std(zc_ini))
    zx_ini = np.array(onp.apply_along_axis(stdrze, 0, zx_ini))
    zy_ini = np.array(onp.apply_along_axis(stdrze, 0, zy_ini))
    zc_ini = np.array(onp.apply_along_axis(stdrze, 0, zc_ini))

    sigma_x_med = 1 / np.median(D_x)
    sigma_y = 1 / np.median(D_y)

    D_zx = covariance_matrix(sqeuclidean_distance, zx_ini, zx_ini)
    sigma_zx_med = 1 / np.median(D_zx)

    D_zy= covariance_matrix(sqeuclidean_distance, zy_ini, zy_ini)
    sigma_zy_med = 1 / np.median(D_zy)

    D_zc = covariance_matrix(sqeuclidean_distance, zc_ini, zc_ini)
    sigma_zc_med = 1 / np.median(D_zc)

    K_zx = rbf_kernel_matrix({'gamma': sigma_zx_med}, zx_ini, zx_ini)
    K_zy = rbf_kernel_matrix({'gamma': sigma_zy_med}, zy_ini, zy_ini)
    K_zc = rbf_kernel_matrix({'gamma': sigma_zc_med}, zc_ini, zc_ini)
    K_yhat = K_x*K_zc*K_zy
    K_xhat = K_zc*K_zx

    smplPart1 = smplsParts[0]
    N_part1 = smplPart1.shape[0]
    y_part1 = y[smplPart1,]
    x_part1 = x[smplPart1,]
    K_yhat_part1 = K_yhat[smplPart1, :]
    K_yhat_part1 = K_yhat_part1[:, smplPart1]
    K_xhat_part1 = K_xhat[smplPart1, :]
    K_xhat_part1 = K_xhat_part1[:, smplPart1]
    ws = np.ones(N_part1)

    weights_y, resids_y, y_hat = krrModel(lam, K_yhat_part1, y_part1, ws)
    weights_x, resids_x, x_hat = krrModel(lam, K_xhat_part1, x_part1, ws)

    # right now we simply overwrite residual lengthscale sig_r_h to be median heuristic
    distsRx = covariance_matrix(sqeuclidean_distance, resids_x, resids_x)
    sigma_rx_med = 1 / np.quantile(distsRx, 0.5)
    distsRy = covariance_matrix(sqeuclidean_distance, resids_y, resids_y)
    sigma_ry_med = 1 / np.quantile(distsRy, 0.5)

    sigs_zx_h= sigma_zx_med * np.ones(reps)
    sigs_zy_h= sigma_zy_med * np.ones(reps)
    sigs_zc_h= sigma_zc_med * np.ones(reps)
    sigs_x_h= sigma_x_med * np.ones(reps)
    sigs_rx_h= sigma_rx_med * np.ones(reps)
    sigs_ry_h= sigma_rx_med * np.ones(reps)

    params['ln_sig_zx_h']= np.log(sigs_zx_h)
    params['ln_sig_zy_h']= np.log(sigs_zy_h)
    params['ln_sig_zc_h']= np.log(sigs_zc_h)
    params['ln_sig_x_h']= np.log(sigs_x_h)
    params['ln_sig_rx_h']= np.log(sigs_rx_h)
    params['ln_sig_ry_h']= np.log(sigs_ry_h)

    return params

@jax.jit
def getParamsForGrad_LNC_lin(params, rep, smpl):

    ln_sig_x_h = params["ln_sig_x_h"][rep]
    ln_sig_zx_h = params["ln_sig_zx_h"][rep]
    ln_sig_zy_h = params["ln_sig_zy_h"][rep]
    ln_sig_zc_h = params["ln_sig_zc_h"][rep]
    ln_sig_rx_h = params["ln_sig_rx_h"][rep]
    ln_sig_ry_h = params["ln_sig_ry_h"][rep]

    params_aux = params.copy()

    #alpha_x = params["alpha_x"]#[:, rep]
    #alpha_y = params["alpha_y"]#[:, rep]
    #alpha_c = params["alpha_c"]#[:, rep]
    #alpha_x = alpha_x[:,None]
    #alpha_y = alpha_y[:,None]
    #alpha_c = alpha_c[:,None]

    #params_aux['alpha_x'] = alpha_x
    #params_aux['alpha_y'] = alpha_y
    #params_aux['alpha_c'] = alpha_c

    params_aux["ln_sig_x_h"] = ln_sig_x_h
    params_aux["ln_sig_zx_h"] = ln_sig_zx_h
    params_aux["ln_sig_zy_h"] = ln_sig_zy_h
    params_aux["ln_sig_zc_h"] = ln_sig_zc_h
    params_aux["ln_sig_rx_h"] = ln_sig_rx_h
    params_aux["ln_sig_ry_h"] = ln_sig_ry_h

    return params_aux

def updateParams_LNC_lin(params, grad_params, smpl, iteration, rep, learning_rate):

    #alpha_x
    #idx_rows = smpl[:, None]
    p = params['alpha_x'].shape[0]
    idx_rows = np.linspace(0, p - 1, p, dtype=int)[:, None]
    idx_cols = np.linspace(0, p - 1, p, dtype=int)[:, None]
    #idx_cols = np.array(rep)[None, None]
    idx = jax.ops.index[tuple([idx_rows, idx_cols])]
    A = params['alpha_x']#[tuple([idx_rows, idx_cols])]
    B = learning_rate * grad_params['alpha_x']
    #params['alpha_x'] = index_update(params['alpha_x'], idx, A - B)
    #params['alpha_x'] = A - B

    #alpha_y
    p = params['alpha_y'].shape[0]
    idx_rows = np.linspace(0, p - 1, p, dtype=int)[:, None]
    idx_cols = np.linspace(0, p - 1, p, dtype=int)[:, None]
    idx = jax.ops.index[tuple([idx_rows, idx_cols])]
    A = params['alpha_y']#[tuple([idx_rows, idx_cols])]
    B = learning_rate * grad_params['alpha_y']
    #params['alpha_y'] = index_update(params['alpha_y'], idx, A - B)
    #params['alpha_y'] = A - B

    #alpha_c
    p = params['alpha_y'].shape[0]
    idx_rows = np.linspace(0, p - 1, p, dtype=int)[:, None]
    idx_cols = np.linspace(0, 1 - 1, 1, dtype=int)[:, None]
    idx = jax.ops.index[tuple([idx_rows, idx_cols])]

    A = params['alpha_c']#[tuple([idx_rows, idx_cols])]
    B = learning_rate * grad_params['alpha_c']
    #params['alpha_c'] = index_update(params['alpha_c'], idx, A - B)
    params['alpha_c'] = A - B

    gpars = [grad_params["ln_sig_x_h"],
    grad_params["ln_sig_zx_h"],
    grad_params["ln_sig_zy_h"],
    grad_params["ln_sig_zc_h"],
    grad_params["ln_sig_rx_h"],
    grad_params["ln_sig_ry_h"],
    grad_params["ln_sig_x_h"],
    grad_params["ln_sig_zx_h"],
    grad_params["ln_sig_zy_h"],
    grad_params["ln_sig_zc_h"],
    grad_params["ln_sig_rx_h"],
    grad_params["ln_sig_ry_h"]]

    if (onp.sum(onp.isnan(B))!=0) | (onp.sum(onp.isinf(B))!=0) | (onp.sum(onp.isnan(gpars))!=0):
        idx_nan, _ = onp.where(onp.isnan(B))
        print("nans in grad Z, iteration: ", iteration, " rep: ", rep)
        raise ValueError('Nans in gradient.')

    return None

@jax.jit
def kernel_mat_LNC_lin(D_x, x, zx, zy, zc, sig_x_h, sig_zx_h, sig_zy_h, sig_zc_h):

    K_x_h = np.exp(-sig_x_h * D_x)

    K_zx_h = rbf_kernel_matrix({'gamma': sig_zx_h}, zx, zx)
    K_zy_h = rbf_kernel_matrix({'gamma': sig_zy_h}, zy, zy)
    K_zc_h = rbf_kernel_matrix({'gamma': sig_zc_h}, zc, zc)

    # x causes - to compare to rx
    K_ax_h = K_zx_h*K_zc_h
    # y causes - to compare to ry
    K_ay_h = K_x_h*K_zy_h*K_zc_h

    return K_ax_h, K_ay_h, K_x_h, K_zx_h, K_zy_h, K_zc_h

# loss
@jax.jit
def loss_LNC_lin(params, pars, loss_data, ws, alpha_x, alpha_y, alpha_c):

    beta, neta, lam, nu, lu = pars

    x, y, Z, U, U_ica,  K_u, D_x, K_y, D_y, K_x, idxs, beta_real, stds, idx_min, idx_x_c, idx_y_c, mixMat = loss_data

    zxc = U_ica@alpha_x
    zyc = U_ica@alpha_y
    zcc = U@alpha_c

    alpha_x = params["alpha_x"]
    alpha_y = params["alpha_y"]
    alpha_c = params["alpha_c"]

    alphap_x = mixMat@alpha_x
    alphap_y = mixMat@alpha_y

    px = alpha_x.shape[1]
    py = alpha_y.shape[1]
    p = alpha_c.shape[0]
    one_p = np.ones(p)[:,None]
    one_px = np.ones(px)[:,None]
    one_py = np.ones(py)[:,None]
    weight_orth = (one_px.T@alphap_x.T@alpha_c)/px + (one_py.T@alphap_y.T@alpha_c)/py + (one_px.T@alphap_x.T@alphap_y@one_py)/(p*p)
    weight_orth = weight_orth[0,0]

    zx = U_ica@alpha_x
    zy = U_ica@alpha_y
    zc = U@alpha_c

    #zx = (zx - np.mean(zx)) / (np.std(zx))
    #zy = (zy - np.mean(zy)) / (np.std(zy))
    #zc = (zc - np.mean(zc)) / (np.std(zc))
    #zx = np.array([stdrze(zx[:,i]) for i in range(zx.shape[1])]).T
    #zy = np.array([stdrze(zy[:,i]) for i in range(zy.shape[1])]).T

    #zxc = (zxc - np.mean(zxc)) / (np.std(zxc))
    #zyc = (zyc - np.mean(zyc)) / (np.std(zyc))
    #zcc = (zcc - np.mean(zcc)) / (np.std(zcc))
    #zxc = np.array([stdrze(zxc[:,i]) for i in range(zxc.shape[1])]).T
    #zyc = np.array([stdrze(zyc[:,i]) for i in range(zyc.shape[1])]).T

    sig_x_h = np.exp(params["ln_sig_x_h"])
    sig_zx_h = np.exp(params["ln_sig_zx_h"])
    sig_zy_h = np.exp(params["ln_sig_zy_h"])
    sig_zc_h = np.exp(params["ln_sig_zc_h"])
    sig_rx_h = np.exp(params["ln_sig_rx_h"])
    sig_ry_h = np.exp(params["ln_sig_ry_h"])

    K_ax_h, K_ay_h, K_x_h, K_zx_h, K_zy_h, K_zc_h = kernel_mat_LNC_lin(D_x, x, zx, zy, zc, sig_x_h, sig_zx_h, sig_zy_h, sig_zc_h)
    n = K_ax_h.shape[0]

    #X_x = zc
    #X_xc = zcc
    #X_y = np.hstack([x,zc])
    #X_yc = np.hstack([x,zcc])

    X_x = np.hstack([zx, zc])
    X_xc = np.hstack([zxc, zcc])
    X_x_cross = zy

    X_y = np.hstack([x,zy, zc])
    X_yc = np.hstack([x,zyc, zcc])
    X_y_cross = zx

    ws = np.ones(X_x.shape[1])
    weights_x, resids_x, x_hat = rrModel(lam, X_x, x, ws)
    _, _, xc_hat = rrModel(lam, X_xc, x, ws)
    ws = np.ones(X_x_cross.shape[1])
    _, _, x_cross_hat = rrModel(lam, X_x_cross, x, ws)

    ws = np.ones(X_y.shape[1])
    weights_y, resids_y, y_hat = rrModel(lam, X_y, y, ws)
    _, _, yc_hat = rrModel(lam, X_yc,  y, ws)
    ws = np.ones(X_y_cross.shape[1])
    _, _, y_cross_hat = rrModel(lam, X_y_cross, x, ws)

    # stats for reg coeffs
    ws = np.ones(X_x.shape[1])
    stat_x = linRegCoefStat(np.array(0.00001), X_x, x, ws)
    stat_x = stat_x[zx.shape[1]]
    ws = np.ones(X_y.shape[1])
    stat_y = linRegCoefStat(np.array(0.00001), X_y, y, ws)
    stat_y = stat_y[zy.shape[1]+1]
    statss = 1-(stat_x+stat_y)/100

    # right now we simply overwrite residual lengthscale sig_r_h to be median heuristic
    #distsRx = covariance_matrix(sqeuclidean_distance, resids_x, resids_x)
    #sig_rx_h = 1 / np.quantile(distsRx, 0.5)
    distsRy = covariance_matrix(sqeuclidean_distance, resids_y, resids_y)
    sig_ry_h = 1 / np.quantile(distsRy, 0.5)

    #K_rx_h = rbf_kernel_matrix({'gamma': sig_rx_h}, resids_x, resids_x)
    K_ry_h = rbf_kernel_matrix({'gamma': sig_ry_h}, resids_y, resids_y)

    # additivity
    # hsic resids_x, (zx, zc)
    #hsic_rx_ax = hsic(K_rx_h, K_ax_h)
    # hsic resids_y, (x, zx, zc)
    #hsic_ry_ay = hsic(K_ry_h, K_ay_h)
    #hsic_resids = np.log(hsic_rx_ax) +  np.log(hsic_ry_ay)

    # indep causes: between (zx, zy, zc) and between zy and x
    hsic_zx_zy = hsic(K_zx_h, K_zy_h)
    #hsic_zx_zy = np.max(np.array([hsic_zx_zy, 0.02]))
    hsic_zx_zc = hsic(K_zx_h, K_zc_h)
    #hsic_zx_zc = np.max(np.array([hsic_zx_zc, 0.02]))
    hsic_zy_zc = hsic(K_zy_h, K_zc_h)
    #hsic_zy_zc = np.max(np.array([hsic_zy_zc, 0.02]))
    hsic_zy_x  = hsic(K_zy_h, K_x_h)
    #hsic_zy_x = np.max(np.array([hsic_zy_x, 0.02]))
    # conditional independence between zx and y given x.. gonna do this for the additive assumption here
    hsic_ry_zx = hsic(K_zx_h, K_ry_h)
    #hsic_ry_zx = np.max(np.array([hsic_ry_zx, 0.02]))

    hsic_indep =  np.log(hsic_zx_zc+0.02) + np.log(hsic_zy_zc+0.02) + np.log(hsic_zx_zy+0.02) + np.log(hsic_zy_x+0.02)   + np.log(hsic_ry_zx+0.02)

    # desirable hsics of x, y vs zc
    hsic_x_zc = hsic(K_x_h, K_zc_h)
    hsic_y_zc = hsic(K_y, K_zc_h)
    hsic_xy_zc = np.log(hsic_x_zc) + np.log(hsic_y_zc)

    # mse residual
    mse_rx = mse(x, x_hat)
    mse_ry = mse(y, y_hat)
    mse_rx_cross = mse(x, x_cross_hat)
    mse_rx_cross = np.min(np.array([mse_rx_cross, 1]))
    mse_ry_cross = mse(y, y_cross_hat)
    mse_ry_cross = np.min(np.array([mse_ry_cross, 1]))
    mse_rxc = mse(x, xc_hat)
    mse_ryc = mse(y, yc_hat)

    mses = np.log(mse_rx) +  np.log(mse_ry)  - np.log(mse_rx_cross) -  np.log(mse_ry_cross) #+ np.log(mse_rxc) +  np.log(mse_ryc)
    #mses = np.log(statss)

    # CI condition

    # Zs size
    zx_norm = np.dot(zx.T, zx)[1,1]
    zy_norm = np.dot(zy.T, zy)[1,1]
    zc_norm = np.dot(zc.T, zc)[1,1]
    z_norm = np.log(zx_norm) + np.log(zy_norm) + np.log(zc_norm)

    # alpha_c norm
    alphac_norm = np.dot(alpha_c.T, alpha_c)[1,1]

    # calcualte compute los
    loss_value =  beta * mses  + neta*hsic_indep   + nu*z_norm - lu*hsic_xy_zc + weight_orth

    #print("loss value dim: ", loss_value.shape)

    return loss_value[0]

dloss_LNC_lin = jax.grad(loss_LNC_lin, )
dloss_LNC_lin_jitted = jax.jit(dloss_LNC_lin)

# for reporting purposes give back all terms separatley
@jax.jit
def model_LNC_lin(params, lam, loss_data, K_t):

    x, y, Z, U, U_ica,  K_u, D_x, K_y, D_y, K_x, idxs, beta_real, stds, idx_min, idx_x_c, idx_y_c, mixMat = loss_data

    alpha_x = params["alpha_x"]
    alpha_y = params["alpha_y"]
    alpha_c = params["alpha_c"]

    alphap_x = mixMat@alpha_x
    alphap_y = mixMat@alpha_y

    px = alpha_x.shape[1]
    py = alpha_y.shape[1]
    p = alpha_c.shape[0]
    one_p = np.ones(p)[:,None]
    one_px = np.ones(px)[:,None]
    one_py = np.ones(py)[:,None]
    weight_orth = (one_px.T@alphap_x.T@alpha_c)/px + (one_py.T@alphap_y.T@alpha_c)/py + (one_px.T@alphap_x.T@alphap_y@one_py)/(p*p)
    weight_orth = weight_orth[0,0]**2

    zx = U_ica@alpha_x
    zy = U_ica@alpha_y
    zc = U@alpha_c

    #zx = (zx - np.mean(zx)) / (np.std(zx))
    #zy = (zy - np.mean(zy)) / (np.std(zy))
    #zc = (zc - np.mean(zc)) / (np.std(zc))
    #zx = np.array([stdrze(zx[:,i]) for i in range(zx.shape[1])]).T
    #zy = np.array([stdrze(zy[:,i]) for i in range(zy.shape[1])]).T

    sig_x_h = np.exp(params["ln_sig_x_h"])
    sig_zx_h = np.exp(params["ln_sig_zx_h"])
    sig_zy_h = np.exp(params["ln_sig_zy_h"])
    sig_zc_h = np.exp(params["ln_sig_zc_h"])
    sig_rx_h = np.exp(params["ln_sig_rx_h"])
    sig_ry_h = np.exp(params["ln_sig_ry_h"])

    K_ax_h, K_ay_h, K_x_h, K_zx_h, K_zy_h, K_zc_h = kernel_mat_LNC_lin(D_x, x, zx, zy, zc, sig_x_h, sig_zx_h, sig_zy_h, sig_zc_h)
    n = K_ax_h.shape[0]

    #X_x = zc
    #X_y = np.hstack([x,zc])

    X_x = np.hstack([zx, zc])
    X_y = np.hstack([x,zy, zc])
    X_x_cross = zy
    X_y_cross = zx

    ws = np.ones(X_x_cross.shape[1])
    _, _, x_cross_hat = rrModel(lam, X_x_cross, x, ws)
    ws = np.ones(X_y_cross.shape[1])
    _, _, y_cross_hat = rrModel(lam, X_y_cross, x, ws)

    ws = np.ones(X_x.shape[1])
    weights_x, resids_x, x_hat = rrModel(lam, X_x, x, ws)
    ws = np.ones(X_y.shape[1])
    weights_y, resids_y, y_hat = rrModel(lam, X_y, y, ws)

    ws = np.ones(x.shape[1])
    beta_x, _, _ = rrModel(lam, x, y, ws)
    ws = np.ones(U.shape[1])
    beta_u, _, _ = rrModel(lam, U, y, ws)

    beta_ce = weights_y[0]
    beta_x = beta_x[0]
    beta_u = beta_u[0]

    # stats for reg coeffs
    ws = np.ones(X_x.shape[1])
    stat_x = linRegCoefStat(np.array(0.00001), X_x, x, ws)
    stat_x = stat_x[zx.shape[1]]
    ws = np.ones(X_y.shape[1])
    stat_y = linRegCoefStat(np.array(0.00001), X_y, y, ws)
    stat_y = stat_y[zy.shape[1]+1]

    x_std, y_std = stds

    beta_ce = beta_ce/x_std*y_std
    beta_x = beta_x/x_std*y_std
    beta_u = beta_u/x_std*y_std

    bias_beta = np.abs(beta_real-beta_ce)
    bias_beta_u = np.abs(beta_real-beta_u)
    bias_beta_x = np.abs(beta_real-beta_x)
    bias_beta_rel_u = bias_beta/bias_beta_u
    bias_beta_rel_x= bias_beta/bias_beta_x

    # right now we simply overwrite residual lengthscale sig_r_h to be median heuristic
    #distsRx = covariance_matrix(sqeuclidean_distance, resids_x, resids_x)
    #sig_rx_h = 1 / np.quantile(distsRx, 0.5)
    distsRy = covariance_matrix(sqeuclidean_distance, resids_y, resids_y)
    sig_ry_h = 1 / np.quantile(distsRy, 0.5)
    #K_rx_h = rbf_kernel_matrix({'gamma': sig_rx_h}, resids_x, resids_x)
    K_ry_h = rbf_kernel_matrix({'gamma': sig_ry_h}, resids_y, resids_y)

    # additivity
    # hsic resids_x, (zx, zc)
    #hsic_rx_ax = hsic(K_rx_h, K_ax_h)
    # hsic resids_y, (x, zx, zc)
    #hsic_ry_ay = hsic(K_ry_h, K_ay_h)
    #hsic_resids = np.log(hsic_rx_ax) +  np.log(hsic_ry_ay)

    # indep causes: between (zx, zy, zc) and between zy and x
    hsic_zx_zy = hsic(K_zx_h, K_zy_h)
    hsic_zx_zc = hsic(K_zx_h, K_zc_h)
    hsic_zy_zc = hsic(K_zy_h, K_zc_h)
    hsic_zy_x  = hsic(K_zy_h, K_x_h)
    # conditional independence between zx and y given x.. gonna do this for the additive assumption here
    hsic_ry_zx = hsic(K_zx_h, K_ry_h)
    hsic_indep = np.log(hsic_zx_zy) + np.log(hsic_zx_zc) + np.log(hsic_zy_zc) + np.log(hsic_zy_x) + np.log(hsic_ry_zx)

    # desirable hsics of x, y vs zc
    hsic_x_zc = hsic(K_x_h, K_zc_h)
    hsic_y_zc = hsic(K_y, K_zc_h)
    hsic_xy_zc = np.log(hsic_x_zc) + np.log(hsic_y_zc)

    # mse residual
    mse_rx = mse(x, x_hat)
    mse_ry = mse(y, y_hat)
    mse_rx_cross = mse(x, x_cross_hat)
    mse_rx_cross = np.min(np.array([mse_rx_cross, 1]))
    mse_ry_cross = mse(y, y_cross_hat)
    mse_ry_cross = np.min(np.array([mse_ry_cross, 1]))

    mses = np.log(mse_rx) +  np.log(mse_ry) - np.log(mse_rx_cross) -  np.log(mse_ry_cross) #+ np.log(mse_rxc) +  np.log(mse_ryc)

    # hsic (zs, zs_reals)

    idx_x, idx_y, idx_c = idxs

    zx_real = Z[:,idx_x]
    zy_real = Z[:,idx_y]
    zc_real = Z[:,idx_c]

    distsZx_real = covariance_matrix(sqeuclidean_distance, zx_real, zx_real)
    sig_zx_real = 1 / np.quantile(distsZx_real, 0.5)
    K_zx_real = rbf_kernel_matrix({'gamma': sig_zx_real}, zx_real, zx_real)

    distsZy_real = covariance_matrix(sqeuclidean_distance, zy_real, zy_real)
    sig_zy_real = 1 / np.quantile(distsZy_real, 0.5)
    K_zy_real = rbf_kernel_matrix({'gamma': sig_zy_real}, zy_real, zy_real)

    distsZc_real = covariance_matrix(sqeuclidean_distance, zc_real, zc_real)
    sig_zc_real = 1 / np.quantile(distsZc_real, 0.5)
    K_zc_real = rbf_kernel_matrix({'gamma': sig_zc_real}, zc_real, zc_real)

    hsic_zx = hsic(K_zx_h, K_zx_real)
    hsic_zy = hsic(K_zy_h, K_zy_real)
    hsic_zc = hsic(K_zc_h, K_zc_real)

    # correlations
    corr_zc = np.abs(corrcoef(zc_real, zc))

    #impurites
    imp_zx = hsic(K_zx_h, K_zc_real)
    imp_zy = hsic(K_zy_h, K_zc_real)

    monitor = {}
    monitor = {
        #'hsic_rx_ax': hsic_rx_ax,
        #'hsic_ry_ay': hsic_ry_ay,
        #'hsic_resids': hsic_resids,
        'hsic_zx_zy': hsic_zx_zy,
        'hsic_zx_zc': hsic_zx_zc,
        'hsic_zy_zc': hsic_zy_zc,
        'hsic_zy_x': hsic_zy_x,
        'hsic_ry_zx': hsic_ry_zx,
        'hsic_indep': hsic_indep,
        'weight_orth': weight_orth,
        'mse_rx': mse_rx,
        'mse_ry': mse_ry,
        'mse_rx_cross': mse_rx_cross,
        'mse_ry_cross': mse_ry_cross,
        'stat_x': stat_x,
        'stat_y': stat_y,
        'mses': mses,
        'hsic_zx': hsic_zx,
        'hsic_zy': hsic_zy,
        'hsic_zc': hsic_zc,
        'corr_zc': corr_zc,
        'imp_zx': imp_zx,
        'imp_zy': imp_zy,
        'hsic_x_zc': hsic_x_zc,
        'hsic_y_zc': hsic_y_zc,
        'hsic_xy_zc': hsic_xy_zc,
        'beta_ce': beta_ce,
        'beta_u': beta_u,
        'beta_x': beta_x,
        'bias_beta': bias_beta,
        'bias_beta_u':bias_beta_u,
        'bias_beta_x':bias_beta_x,
        'bias_beta_rel_u':bias_beta_rel_u,
        'bias_beta_rel_x':bias_beta_rel_x
    }

    return monitor

def getIniMonitor_LNC_lin(epochs, report_freq, reps, parts):
        num_reports = int(np.ceil(epochs / report_freq))+1 # initial report
        print("num_reports: ", num_reports)
        monitors = {
            'loss': onp.zeros([num_reports, reps, parts]),
            #'hsic_rx_ax': onp.zeros([num_reports, reps, parts]),
            #'hsic_ry_ay': onp.zeros([num_reports, reps, parts]),
            #'hsic_resids': onp.zeros([num_reports, reps, parts]),
            'hsic_zx_zy': onp.zeros([num_reports, reps, parts]),
            'hsic_zx_zc': onp.zeros([num_reports, reps, parts]),
            'hsic_zy_zc': onp.zeros([num_reports, reps, parts]),
            'hsic_zy_x': onp.zeros([num_reports, reps, parts]),
            'hsic_ry_zx': onp.zeros([num_reports, reps, parts]),
            'hsic_indep': onp.zeros([num_reports, reps, parts]),
            'weight_orth': onp.zeros([num_reports, reps, parts]),
            'hsic_x_zc': onp.zeros([num_reports, reps, parts]),
            'hsic_y_zc': onp.zeros([num_reports, reps, parts]),
            'hsic_xy_zc': onp.zeros([num_reports, reps, parts]),
            'mse_rx': onp.zeros([num_reports, reps, parts]),
            'mse_ry': onp.zeros([num_reports, reps, parts]),
            'mse_rx_cross': onp.zeros([num_reports, reps, parts]),
            'mse_ry_cross': onp.zeros([num_reports, reps, parts]),
            'mses': onp.zeros([num_reports, reps, parts]),
            'stat_x': onp.zeros([num_reports, reps, parts]),
            'stat_y': onp.zeros([num_reports, reps, parts]),
            'corr_zc': onp.zeros([num_reports, reps, parts]),
            'hsic_zx': onp.zeros([num_reports, reps, parts]),
            'hsic_zy': onp.zeros([num_reports, reps, parts]),
            'hsic_zc': onp.zeros([num_reports, reps, parts]),
            'imp_zx': onp.zeros([num_reports, reps, parts]),
            'imp_zy': onp.zeros([num_reports, reps, parts]),
            'beta_ce': onp.zeros([num_reports, reps, parts]),
            'beta_u': onp.zeros([num_reports, reps, parts]),
            'beta_x': onp.zeros([num_reports, reps, parts]),
            'bias_beta': onp.zeros([num_reports, reps, parts]),
            'bias_beta_u':onp.zeros([num_reports, reps, parts]),
            'bias_beta_x':onp.zeros([num_reports, reps, parts]),
            'bias_beta_rel_u':onp.zeros([num_reports, reps, parts]),
            'bias_beta_rel_x':onp.zeros([num_reports, reps, parts])

        }
        return monitors

def fillMonitor_LNC_lin(params, params_or, pars, loss_as_par, dloss_as_par_jitted, lossData, iteration, report_freq, rep, part, reps, monitors, K_t):

    print("enters filMonitor_LNC_lin")
    _, _, lam, _,_ = pars

    monitor = model_LNC_lin(params, lam, lossData, K_t)
    print("monitor")
    print(monitor)
    ws = np.ones(params["alpha_x"].shape[0])
    loss_val = loss_as_par(params, pars, lossData,ws, params["alpha_x"], params["alpha_y"], params["alpha_c"])
    indxRep = int(iteration / report_freq)
    print("indxRep:", indxRep, " iteration: ", iteration, " rep: ", rep, " part: ", part)

    monitors['loss'][indxRep, rep, part] = loss_val
    #monitors['hsic_rx_ax'][indxRep, rep, part] = monitor['hsic_rx_ax']
    #monitors['hsic_ry_ay'][indxRep, rep, part] = monitor['hsic_ry_ay']
    #monitors['hsic_resids'][indxRep, rep, part] = monitor['hsic_resids']
    monitors['hsic_zx_zy'][indxRep, rep, part] = monitor['hsic_zx_zy']
    monitors['hsic_zx_zc'][indxRep, rep, part] = monitor['hsic_zx_zc']
    monitors['hsic_zy_zc'][indxRep, rep, part] = monitor['hsic_zy_zc']
    monitors['hsic_zy_x'][indxRep, rep, part] = monitor['hsic_zy_x']
    monitors['hsic_ry_zx'][indxRep, rep, part] = monitor['hsic_ry_zx']
    monitors['hsic_indep'][indxRep, rep, part] = monitor['hsic_indep']
    monitors['hsic_x_zc'][indxRep, rep, part] = monitor['hsic_x_zc']
    monitors['hsic_y_zc'][indxRep, rep, part] = monitor['hsic_y_zc']
    monitors['hsic_xy_zc'][indxRep, rep, part] = monitor['hsic_xy_zc']
    monitors['mse_rx'][indxRep, rep, part] = monitor['mse_rx']
    monitors['mse_ry'][indxRep, rep, part] = monitor['mse_ry']
    monitors['mse_rx_cross'][indxRep, rep, part]= monitor['mse_rx_cross']
    monitors['mse_ry_cross'][indxRep, rep, part]= monitor['mse_ry_cross']
    monitors['mses'][indxRep, rep, part] = monitor['mses']
    monitors['stat_x'][indxRep, rep, part] = monitor['stat_x']
    monitors['stat_y'][indxRep, rep, part] = monitor['stat_y']
    monitors['hsic_zx'][indxRep, rep, part] = monitor['hsic_zx']
    monitors['hsic_zy'][indxRep, rep, part] = monitor['hsic_zy']
    monitors['hsic_zc'][indxRep, rep, part] = monitor['hsic_zc']
    monitors['corr_zc'][indxRep, rep, part] = monitor['corr_zc']
    monitors['imp_zx'][indxRep, rep, part] = monitor['imp_zx']
    monitors['imp_zy'][indxRep, rep, part] = monitor['imp_zy']
    monitors['beta_ce'][indxRep, rep, part] = monitor['beta_ce']
    monitors['beta_x'][indxRep, rep, part] = monitor['beta_x']
    monitors['beta_u'][indxRep, rep, part] = monitor['beta_u']
    monitors['bias_beta'][indxRep, rep, part]= monitor["bias_beta"]
    monitors['bias_beta_u'][indxRep, rep, part]=monitor["bias_beta_u"]
    monitors['bias_beta_x'][indxRep, rep, part]=monitor["bias_beta_x"]
    monitors['bias_beta_rel_u'][indxRep, rep, part]=monitor["bias_beta_rel_u"]
    monitors['bias_beta_rel_x'][indxRep, rep, part]= monitor['bias_beta_rel_x']

    print("exits filMonitor_LNC_lin")

    return monitors

def getLatentZ_LNC_lin(params, loss_as_par, dloss_as_par_jitted, loss_data, pars, epochs, report_freq, reps, batch_size, batch_per_epoch, learning_rate, smplsParts, monitors, batches, batches2, K_t):
    x, _, _, _, _, _, _, _, _, _, _, _, _,_,_,_,_ = loss_data

    N = x.shape[0]
    p = params["alpha_x"].shape[0]

    print("report_freq: ", report_freq)
    parts = len(smplsParts)

    ms = onp.zeros([p, epochs+1, reps])
    vs = onp.zeros([p, epochs+1, reps])
    epochs2 = int(onp.floor(epochs/batch_per_epoch))
    loss_vals = onp.ones([epochs+1, reps])*onp.Inf
    loss_vals_epoch = onp.ones([epochs2, reps])*onp.Inf

    lrs = learning_rate*onp.ones(reps)
    adamOpt = False

    for iteration in range(epochs+1):

        # print("*********************")
        if (iteration % 50 == 0):
            print("iteration: ", iteration)
        #print("nans: ", onp.sum(onp.isnan(onp.array(params["Z"]))))

        epoch = (iteration-1) // batch_per_epoch
        batch = (iteration-1) % batch_per_epoch

        #epoch = bisect.bisect_left(cumbatches_per_epoch, iteration)-1
        #batch = iteration -cumbatches_per_epoch[epoch]-1

        #print("iteration: ", iteration)
        #print("epoch: ", epoch)
        #print("batch: ", batch)

        # get the gradient of the loss
        for rep in range(reps):
            #print("rep: ", rep)

            if(iteration == 0):
                smpl = onp.random.randint(low=0, high=N, size=batch_size)
                smpl2 = onp.random.randint(low=0, high=N, size=batch_size*2)
            else:
                #print(np.unique(np.hstack(batches[rep][epoch])).shape)
                #print(np.unique(np.hstack(batches2[rep][epoch])).shape)
                smpl = batches[rep][epoch][batch]
                smpl2 = batches2[rep][epoch][batch]

            #print("len smpl: ", len(smpl))
            #print("len smpl 2: ", len(smpl2))

            if iteration == 5:
                print("smpl: ", smpl[0:4])

            #smpl2 = onp.random.randint(low=0, high=N, size=batch_size)

            #smpl3 = onp.linspace(0,n-1,n, dtype=int)
            # sampling without replacemnt
            #smpl = onp.random.choice(a=n, size=batch_size, replace=False)

            loss_data_aux = getDataLoss_LNC_lin(smpl, loss_data)
            loss_data_aux2 = getDataLoss_LNC_lin(smpl2, loss_data)

            # equal weights
            ws = np.ones(2)

            # algorithmic independence forcing

            # random weights - so that E[y|x] alg indep of p(x)
            #ws = getWeightsAlgoIndep(loss_data, batch_size, smpl)
            #print("ws: ", ws[0:5])

            # prepare parameters for grad calculation (subsample)

            #params_aux = getParamsForGrad(params, rep, optType, smpl)
            params_aux = getParamsForGrad_LNC_lin(params, rep, smpl)
            params_aux2 = getParamsForGrad_LNC_lin(params, rep, smpl2)
            #params_aux3 = getParamsForGrad(params, rep, smpl3)

            if adamOpt:
                grad_params = grad(loss_as_par, argnums=0)(params_aux, pars, loss_data_aux, ws, params_aux["alpha_x"], params_aux["alpha_y"], params_aux["alpha_c"])
                if iteration > 0:
                    m = ms[smpl,iteration-1,rep][:,None]
                    v = vs[smpl,iteration-1,rep][:,None]
                    m, v = updateParamsAdam2_LNC_lin(params, grad_params, smpl, iteration, rep, lrs[rep], iteration-1, m, v)
                    ms[smpl,iteration,rep] = m[:,0]
                    vs[smpl,iteration,rep] = v[:,0]

            else:
                grad_params = dloss_as_par_jitted(params_aux, pars, loss_data_aux, ws, params_aux["alpha_x"], params_aux["alpha_y"], params_aux["alpha_c"])
                if iteration > 0:
                    updateParams_LNC_lin(params, grad_params, smpl, iteration, rep, lrs[rep])

            ws = np.ones(batch_size*2)
            loss_vals[iteration, rep] = loss_as_par(params_aux2, pars, loss_data_aux2, ws, params_aux["alpha_x"], params_aux["alpha_y"], params_aux["alpha_c"])

            if (iteration % report_freq == 0) & (iteration != 0):
                #print("report")
                print("iteration report: ", iteration)

                for part in range(parts):

                    smplPart = smplsParts[part]
                    params_part = getParamsForGrad_LNC_lin(params, rep, smplPart)
                    #params_part = getParamsSmpl(params, smplPart)
                    loss_data_part = getDataLoss_LNC_lin(smplPart, loss_data)
                    monitors = fillMonitor_LNC_lin(params_part, params, pars, loss_as_par, dloss_as_par_jitted, loss_data_part, iteration, report_freq, rep, part, reps, monitors, K_t)

                    #nPerPart = smplPart.shape[0]
                    #ws = np.ones(nPerPart)
                    #loss_val = loss_as_par(params_part, pars, loss_data_part, ws)

    return params, monitors #, resids, bestResids, bestZ

def getLatentZ_wrapper_lin(x, y, Z, U, idxs, stds, beta_real, nm, pars, num_epochs, report_freq, num_reps, batch_size, learning_rate, job):
    print("nm:", nm)
    N = x.shape[0]

    D_x = covariance_matrix(sqeuclidean_distance, x, x)
    sigma_x_med = 1 / np.median(D_x)
    K_x = rbf_kernel_matrix({'gamma': sigma_x_med}, x, x)

    D_y = covariance_matrix(sqeuclidean_distance, y, y)
    sigma_y = 1 / np.median(D_y)
    K_y = rbf_kernel_matrix({'gamma': sigma_y}, y, y)

    D_u = covariance_matrix(sqeuclidean_distance, U, U)
    sigma_u = 1 / np.median(D_u)
    K_u = rbf_kernel_matrix({'gamma': sigma_u}, U, U)

    numComps = onp.min([Z.shape[1], Z.shape[0]-3])
    transformer = FastICA(n_components=numComps,random_state=0,whiten='unit-variance')
    U_ica = transformer.fit_transform(onp.array(U))
    print("U_ica shape: ", U_ica.shape)
    mixMat = transformer.components_
    mixMat  = mixMat.T
    pvalsy = [sm.OLS(onp.array(y), sm.add_constant(onp.hstack([x, U_ica[:,i][:,None]]))).fit().pvalues[2] for i in range(U_ica.shape[1])]
    pvalsx = [sm.OLS(onp.array(x), sm.add_constant(U_ica[:,i][:,None])).fit().pvalues[1] for i in range(U_ica.shape[1])]
    pvals = np.array([pvalsx, pvalsy])
    sumpvals = onp.apply_along_axis(np.sum, 0, pvals)
    indx_min = onp.argmin(sumpvals)
    print("indx_min: ", indx_min)
    idx_rest  = list(set(onp.arange(0, U_ica.shape[1])).difference(set([indx_min])))
    print("idx_rest: ", idx_rest)
    modMat_aux = sm.add_constant(onp.hstack([x, U_ica[:,indx_min][:,None] ,U_ica[:,idx_rest]] ))
    numRows = modMat_aux.shape[0]
    maxCols = onp.min([numRows, modMat_aux.shape[1]])
    #modMat_aux = modMat_aux[:, 0:maxCols]
    print("modMat shape: ", modMat_aux.shape)
    pvals_resty = sm.OLS(onp.array(y), modMat_aux).fit().pvalues
    pvals_resty = pvals_resty[3:(pvals_resty.shape[0])]
    modMat_aux = sm.add_constant(onp.hstack([U_ica[:,indx_min][:,None], U_ica[:,idx_rest]] ))
    numRows = modMat_aux.shape[0]
    maxCols = onp.min([numRows, modMat_aux.shape[1]])
    #modMat_aux = modMat_aux[:, 0:maxCols]
    print("modMat shape: ", modMat_aux.shape)
    pvals_restx = sm.OLS(onp.array(x), modMat_aux).fit().pvalues
    pvals_restx = pvals_restx[2:(pvals_restx.shape[0])]

    print("pvals_resty: ", pvals_resty)
    print("pvals_restx: ", pvals_restx)
    idx_y_c, = onp.where( (pvals_resty<pvals_restx)&(pvals_resty<0.05))
    idx_y_c = onp.array(idx_rest)[idx_y_c]
    if idx_y_c.shape[0]==0:
        idx_y_c, = onp.where( (pvals_resty<pvals_restx))
        idx_y_c = onp.array(idx_rest)[idx_y_c]
    if idx_y_c.shape[0]==0:
        idx_y_c = onp.argsort(pvals_resty)[0:2]
        idx_y_c = onp.array(idx_rest)[idx_y_c]

    print("idx_y_c: ", idx_y_c)
    idx_x_c, = onp.where( (pvals_resty>pvals_restx)&(pvals_restx<0.05))
    idx_x_c = onp.array(idx_rest)[idx_x_c]
    if idx_x_c.shape[0]==0:
        idx_x_c, = onp.where( (pvals_resty>pvals_restx))
        idx_x_c = onp.array(idx_rest)[idx_x_c]
    if idx_x_c.shape[0]==0:
        idx_x_c = onp.argsort(pvals_restx)[0:2]
        idx_x_c = onp.array(idx_rest)[idx_x_c]
    print("idx_x_c: ", idx_x_c)

    onp.random.seed(seed=job)

    maxMonitor = 1000
    parts = int(onp.ceil(N/maxMonitor))
    smplsParts = onp.random.choice(parts, size=N)

    smplsParts = [myWhere(smplsParts==i) for i in range(parts)]
    #print("smplsParts: ", smplsParts)

    lossData = x, y, Z, U, U_ica, K_u, D_x, K_y, D_y, K_x, idxs, beta_real, stds, indx_min, idx_x_c, idx_y_c, mixMat

    onp.random.seed(seed=job+3)
    params = getIniPar_LNC_lin(num_reps, lossData, pars, smplsParts)
    print("alpha x shape: ",params["alpha_x"].shape)

    res = []

    onp.random.seed(seed=job+123)
    batch_per_epoch = int(onp.ceil(N / batch_size))
    epochs2 = int(onp.ceil(num_epochs / batch_per_epoch))
    num_iters = int(epochs2*batch_per_epoch)
    batches = [[get_batches_one_epoch(batch_per_epoch, batch_size, N) for i in range(epochs2)] for j in range(num_reps)]
    #batches2 = [[get_batches_one_epoch(batch_per_epoch, batch_size, N) for i in range(epochs2)] for j in range(num_reps)]
    batches2 = [[ [onp.random.randint(low=0, high=N, size=batch_size*2) for k in range(batch_per_epoch)] for i in range(epochs2)] for j in range(num_reps)]
    print("N: ", N)
    print("batch size: ", batch_size)
    print("batch per_epoch: ", batch_per_epoch)
    print("epochs2: ", epochs2)
    print("epochs eff: ", len(batches[0]))
    print("num_iters: ", num_iters)

    report_freq2 = num_iters // 1

    path = getIniMonitor_LNC_lin(num_iters, report_freq, num_reps, parts)

    loss_as_par = loss_LNC_lin
    dloss_as_par_jitted = dloss_LNC_lin_jitted

    start = time.process_time()
    params, path = getLatentZ_LNC_lin(params, loss_as_par, dloss_as_par_jitted, loss_data=lossData, pars=pars,epochs=num_iters, report_freq=report_freq,
                                                         reps=num_reps, batch_size=batch_size, batch_per_epoch=batch_per_epoch, learning_rate=learning_rate, smplsParts=smplsParts, monitors=path, batches=batches, batches2=batches2, K_t=None)
    res.append(path)

    res = {k: onp.concatenate([r[k] for r in res], axis=0) for k in res[0].keys()}

    params["zc"] = U@params["alpha_c"]

    print("results")
    res = {"params": params, "path": res}
    #print(res["path"])

    return res


from termcolor import colored
from sqlalchemy import inspect
from ...model.kingdom import Kingdom
from ...model.city import City
from ...model.shop import Shop
from ...model.item import Item
from ...model.item_type import ItemType
from ...model.shop_type import ShopType
import os

def clear_terminal():
    _ = os.system("clear")

def add_entity(session, entity_class, context):
    entity_attributes = {}
    entity_columns = [column.key for column in entity_class.__table__.columns if column.key != 'id']

    parent_entity_name = get_parent_entity_name(entity_columns)
    if parent_entity_name:
        parent_entity = get_parent_entity(session, parent_entity_name, context)
        if parent_entity:
            entity_attributes[parent_entity_name.lower()] = parent_entity
            entity_columns.remove(f'{parent_entity_name.lower()}_id')

    for column in entity_columns:
        if column not in entity_attributes:
            value = get_column_value(session, entity_class, column)
            entity_attributes[column] = value

    new_entity = entity_class(**entity_attributes)
    session.add(new_entity)
    session.commit()
    print(colored(f"{entity_class.__name__} added successfully!", 'green'))
    clear_terminal()

def get_parent_entity_name(entity_columns):
    for column_name in entity_columns:
        if column_name.endswith('_id'):
            return column_name[:-3].capitalize()
    return None

def get_parent_entity(session, parent_entity_name, context):
    entity_mapping = {
    "Kingdom": Kingdom,
    "City": City,
    "Shop": Shop,
    "Item": Item,
    "ItemType": ItemType,
    "ShopType": ShopType
    }
    parent_entity_name_in_context = context.get(parent_entity_name)
    if parent_entity_name_in_context:
        parent_entity_class = entity_mapping[parent_entity_name]
        return session.query(parent_entity_class).filter_by(name=parent_entity_name_in_context).first()
    return None

def get_column_value(session, entity_class, column):
    if column == 'name':
        return get_unique_name(session, entity_class)
    else:
        return input(colored(f"{entity_class.__name__} {column}:", attrs=['bold']))

def get_unique_name(session, entity_class):
    while True:
        name = input(colored(f"{entity_class.__name__} name:", attrs=['bold']))
        existing_entity = session.query(entity_class).filter_by(name=name).first()
        if existing_entity:
            print(colored("Name already exists. Please enter a different name.", 'red'))
        else:
            return name

def get_foreign_key_column(entity_class, parent_entity_class):
    foreign_key_columns = []
    for column in entity_class.__table__.columns:
        if column.foreign_keys:
            for foreign_key in column.foreign_keys:
                if foreign_key.column.table.name == parent_entity_class.__tablename__:
                    foreign_key_columns.append(column)
    if not foreign_key_columns:
        raise ValueError(f"No foreign key columns found in {entity_class.__name__} referencing {parent_entity_class.__name__}")
    return foreign_key_columns[0]

def show_entities(session, entity_class, parent_entity):
    if parent_entity:
        parent_entity_id_field = get_foreign_key_column(entity_class, parent_entity.__class__)
        entities = session.query(entity_class).filter(getattr(entity_class, parent_entity_id_field.key) == parent_entity.id).all()
    else:
        entities = session.query(entity_class).all()

    if not entities:
        print(colored(f"No {entity_class.__name__} entities found.", 'red'))
    else:
        print(colored(f"Available {entity_class.__name__} entities:", 'cyan', attrs=['bold']))
        for entity_obj in entities:
            print(colored(entity_obj.name, 'yellow'))
        print("")


from flask import Flask    # 导入Flask类
from flask import render_template  # 导入render_template模块
app = Flask(__name__)         # 实例化并命名为app实例

# 使用了装饰器来制定路由url
@app.route('/')  # 调用route路由方法，括号里给定参数，/符号默认为首页
# @app.route('/home/user')    # 调用route路由方法，/home/user定位到访问user方法页面
def index(name=None):
    # return 'Hello World'
    # return '
welcome to my webpage!

输出语句测试
'
    # return render_template("index.html")  # 调用render_template函数，传入html文件参数
    msg = "China up!"
    return render_template("index.html", data=msg)  # 加入变量传递

if __name__ == "__main__":
    app.run(port=2020, host="127.0.0.1", debug=True)


import glob
import nltk
from clean_text import normalize
from collections import Counter
from gensim.models import Phrases
from gensim.models.phrases import Phraser
from nltk.corpus import stopwords
import pandas as pd
import string
import re
from itertools import chain

briefings = glob.glob("../../data/interim/*.txt")

dates = []
remarks = []
for briefing in briefings:
    date = pd.to_datetime(briefing.split('_')[-1].split('.')[0], infer_datetime_format=True)
    with open(briefing,'r') as f:
        lines = f.readlines()
        for line in lines:
            # keep the longer remarks
            if len(line.split()) >= 16:
                # BERT cannot handle really long lines, so we split up any paragraphs that are too big
                if len(line.split()) > 120:
                    text = line.split('.')
                    num_sent = len(text)
                    len1 = num_sent//2
                    #print('\n',text)
                    #print(text[:len1])
                    #print(text[len1:])
                    remarks.append('.'.join(text[:len1]).rstrip().replace(r'’','\''))
                    dates.append(date)
                    remarks.append('.'.join(text[len1:]).rstrip().replace(r'’','\''))
                    dates.append(date)
                    #print(remarks[-2:])
                else:
                    remarks.append(line.rstrip().replace(r'’','\''))
                    dates.append(date)

df = pd.DataFrame({'date':dates,'text':remarks})

df['normalized_text'] = df['text'].apply(lambda x: normalize(x))

# make bigrams

sentences = df['normalized_text'].values
sentence_stream = [sentence.split() for sentence in sentences]
bigram = Phrases(sentence_stream, min_count=5, threshold=80)
trigram = Phrases(bigram[sentence_stream], threshold=80)
df['clean_text'] = df['normalized_text'].apply(lambda x: ' '.join(bigram[x.split()]))
df['clean_text'] = df['clean_text'].apply(lambda x: ' '.join(trigram[bigram[x.split()]]))

df.to_csv('trump_wh_remarks.csv')


# This code is defining the URL patterns for a Django project.
from django.contrib import admin
from django.urls import path, include

# The `urlpatterns` variable is a list of URL patterns for a Django project. Each `path` function call
# specifies a URL pattern and the corresponding view or URL configuration to be used.
urlpatterns = [
    path('', include('core.urls')),
    path('rooms/', include('room.urls')),
    path('admin/', admin.site.urls),
]


# -*- coding: utf-8 -*-

import csv
from datetime import datetime
import os
import pprint
import sys

if len(sys.argv) < 3:
    print ("Usage: "+sys.argv[0]+" <inputfile> <startdate> <enddate>")
    print ("  e.g. "+sys.argv[0]+" yellow_tripdata_2015-06.csv \"2015-06-01 00:00:00\" \"2015-06-01 23:59:59\"")
    print ("  (all dates are inclusive)")
    sys.exit(1)

INPUT_FILE = sys.argv[1]
START_DATE = datetime.strptime(sys.argv[2], '%Y-%m-%d %H:%M:%S')
END_DATE = datetime.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S')
OUTPUT_FILE = 'visualization/data/rides.csv'
NYC_BOUNDS = [40.477399, -74.25909, 40.917577, -73.7000272] # Source: https://www.maptechnica.com/us-city-boundary-map/city/New%20York/state/NY/cityid/3651000?

rides = []

# Read files from file
with open(INPUT_FILE, 'rb') as f:
    r = csv.reader(f, delimiter=',')
    next(r, None) # remove header
    for VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, passenger_count, trip_distance, pickup_longitude, pickup_latitude, RateCodeID, store_and_fwd_flag, dropoff_longitude, dropoff_latitude, payment_type, fare_amount, extra, mta_tax, tip_amount, tolls_amount, total_amount in r:

        pickup_datetime = datetime.strptime(tpep_pickup_datetime, '%Y-%m-%d %H:%M:%S')
        dropoff_datetime = datetime.strptime(tpep_dropoff_datetime, '%Y-%m-%d %H:%M:%S')

        # Make sure dates are in range
        if pickup_datetime >= START_DATE and dropoff_datetime <= END_DATE:
            pickup_latitude = float(pickup_latitude)
            pickup_longitude = float(pickup_longitude)
            dropoff_latitude = float(dropoff_latitude)
            dropoff_longitude = float(dropoff_longitude)
            # Make sure lat/lng in nyc bounds
            if pickup_latitude > NYC_BOUNDS[0] and pickup_longitude > NYC_BOUNDS[1] and pickup_latitude < NYC_BOUNDS[2] and pickup_longitude < NYC_BOUNDS[3] and dropoff_latitude > NYC_BOUNDS[0] and dropoff_longitude > NYC_BOUNDS[1] and dropoff_latitude < NYC_BOUNDS[2] and dropoff_longitude < NYC_BOUNDS[3]:
                rides.append([pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude])

with open(OUTPUT_FILE, 'wb') as f:
    w = csv.writer(f)
    w.writerow(['lt0', 'ln0', 'lt1', 'ln1'])
    for r in rides:
        w.writerow(r)
    print('Successfully wrote '+str(len(rides))+' rides file: '+OUTPUT_FILE)

print ("Done.")


from django.db import models

# Create your models here.
from django.contrib.auth import get_user_model

User = get_user_model()

class Post(models.Model):
    text = models.TextField(
        verbose_name='Текст поста',
        help_text='Напишите что-нибудь'
    )
    pub_date = models.DateTimeField(
        auto_now_add=True,
        verbose_name='Дата публикации'
    )
    author = models.ForeignKey(
        User,
        on_delete=models.CASCADE,
        related_name='posts',
        verbose_name='Автор поста'
    )
    group = models.ForeignKey(
        'Group',
        on_delete=models.CASCADE,
        related_name='posts',
        verbose_name='Группа',
        blank=True,
        null=True,
        help_text='Здесь можно выбрать группу'
    )

    class Meta:
        ordering = ('-pub_date',)
        default_related_name = 'posts'
        verbose_name = 'Пост'
        verbose_name_plural = 'Посты'

class Group(models.Model):
    title = models.CharField(
        max_length=200,
        verbose_name='Заголовок'
    )
    slug = models.SlugField(
        null=False,
        unique=True
    )
    description = models.TextField(
        verbose_name='Описание группы'
    )

    def __str__(self):
        return self.title


import math

import bpy
import random

import numpy as np
from mathutils import Euler
from .node import Nodes

ROT90_X = Euler([math.radians(90), 0, 0]).to_matrix().to_4x4()
ROTN90_X = Euler([math.radians(-90), 0, 0]).to_matrix().to_4x4()
ROT90_Y = Euler([0, math.radians(90), 0]).to_matrix().to_4x4()
ROTN90_Y = Euler([0, math.radians(-90), 0]).to_matrix().to_4x4()
ROT180_Y = Euler([0, math.radians(180), 0]).to_matrix().to_4x4()
ROT90_Z = Euler([0, 0, math.radians(90)]).to_matrix().to_4x4()
ROTN90_Z = Euler([0, 0, math.radians(-90)]).to_matrix().to_4x4()

def get_material(mat_name, model_ob):
    md = model_ob.data
    mat = bpy.data.materials.get(mat_name, None)
    if mat:
        if md.materials.get(mat.name, None):
            for i, material in enumerate(md.materials):
                if material == mat:
                    return i
        else:
            md.materials.append(mat)
            return len(md.materials) - 1
    else:
        mat = bpy.data.materials.new(mat_name)
        mat.diffuse_color = [random.uniform(.4, 1) for _ in range(3)] + [1.0]
        md.materials.append(mat)
        return len(md.materials) - 1

def get_or_create_collection(name, parent: bpy.types.Collection) -> bpy.types.Collection:
    new_collection = (bpy.data.collections.get(name, None) or
                      bpy.data.collections.new(name))
    if new_collection.name not in parent.children:
        parent.children.link(new_collection)
    new_collection.name = name
    return new_collection

def get_new_unique_collection(model_name, parent_collection):
    copy_count = len([collection for collection in bpy.data.collections if model_name in collection.name])

    master_collection = get_or_create_collection(model_name + (f'_{copy_count}' if copy_count > 0 else ''),
                                                 parent_collection)
    return master_collection

def append_blend(filepath, type_name, link=False):
    with bpy.data.libraries.load(filepath, link=link) as (data_from, data_to):
        setattr(data_to, type_name, [asset for asset in getattr(data_from, type_name)])
    for o in getattr(data_to, type_name):
        o.use_fake_user = True

def transform_vec3_array(vec3, matrix):
    tmp = np.zeros((len(vec3), 4), dtype=vec3.dtype)
    tmp[:, :3] = vec3
    tmp = tmp @ matrix
    return tmp[:, :3]

def transform_vec3(vec3, matrix):
    tmp = np.zeros((4,), dtype=np.float32)
    tmp[:3] = vec3
    tmp = tmp @ matrix
    return tmp[:3]


"""Mixture model for matrix completion"""
from typing import Tuple
import numpy as np
from scipy.special import logsumexp
from common import GaussianMixture
from scipy.stats import multivariate_normal as norm

def estep(X: np.ndarray, mixture: GaussianMixture) -> Tuple[np.array, float]:
    """E-step: Softly assigns each datapoint to a gaussian component

    Args:
        X: (n, d) array holding the data, with incomplete entries (set to 0)
        mixture: the current gaussian mixture

    Returns:
        np.ndarray: (n, K) array holding the soft counts
            for all components for all examples
        float: log-likelihood of the assignment

    """
    # 1. build nn: normal distribution n*K
    mu = mixture.mu
    p = mixture.p
    var = mixture.var
    num = X.shape[0]
    K = len(p)

    # 1. build nn
    norm_table = np.array([norm.pdf(X[0], mean=mu[j], cov=(var[j])) for j in np.arange(K)])
    for i in np.arange(1, num):
        nn = np.array([norm.pdf(X[i], mean=mu[j], cov=(var[j])) for j in np.arange(K)])
        norm_table = np.append(norm_table, nn)
    norm_table = norm_table.reshape((num, K))
    # 2. calculate px
    px = np.matmul(norm_table, np.transpose(p))
    # 3. calculate pj_x: the soft count
    pj_x = norm_table * p
    pj_x = pj_x/px.reshape((num, 1))
    # 4. calculate the log likelihood
    log_px = np.log(px)
    likelihood = np.sum(log_px)

    return pj_x, likelihood

def mstep(X: np.ndarray, post: np.ndarray, mixture: GaussianMixture,
          min_variance: float = .25) -> GaussianMixture:
    """M-step: Updates the gaussian mixture by maximizing the log-likelihood
    of the weighted dataset

    Args:
        X: (n, d) array holding the data, with incomplete entries (set to 0)
        post: (n, K) array holding the soft counts
            for all components for all examples
        mixture: the current gaussian mixture
        min_variance: the minimum variance for each gaussian

    Returns:
        GaussianMixture: the new gaussian mixture
    """
    num, d = X.shape
    K = post.shape[1]
    nj = np.sum(post, axis=0)
    pj = nj/num
    u = np.matmul(np.transpose(post), X)
    mu = u/nj.reshape(len(nj), 1)

    post_sum = np.sum(post, axis=0)
    var = np.zeros(K)

    for j in np.arange(K):
        diff = X - mu[j]
        diff = np.linalg.norm(diff, axis=1) ** 2
        v = np.sum(post[:,j] * diff)
        # v = np.transpose(post[:,j]) * diff
        var[j] = v/(d * nj[j])

    new_mixture = GaussianMixture(mu, var, pj)
    return new_mixture

def run(X: np.ndarray, mixture: GaussianMixture,
        post: np.ndarray) -> Tuple[GaussianMixture, np.ndarray, float]:
    """Runs the mixture model

    Args:
        X: (n, d) array holding the data
        post: (n, K) array holding the soft counts
            for all components for all examples

    Returns:
        GaussianMixture: the new gaussian mixture
        np.ndarray: (n, K) array holding the soft counts
            for all components for all examples
        float: log-likelihood of the current assignment
    """
    l_old = 0
    not_converge = True
    while (not_converge):
        post, loglikehood = estep(X, mixture)
        mixture = mstep(X, post, mixture)
        ans = 'logLikelihood = {}'.format(loglikehood)
        # print(ans)
        not_converge = np.abs(loglikehood - l_old) >= (10**-6 * np.abs(loglikehood))
        l_old = loglikehood
    return mixture, post, loglikehood

def fill_matrix(X: np.ndarray, mixture: GaussianMixture) -> np.ndarray:
    """Fills an incomplete matrix according to a mixture model

    Args:
        X: (n, d) array of incomplete data (incomplete entries =0)
        mixture: a mixture of gaussians

    Returns
        np.ndarray: a (n, d) array with completed data
    """
    raise NotImplementedError


import esphome.codegen as cg
import esphome.config_validation as cv
from esphome.components import binary_sensor
from esphome.const import CONF_ID
from . import EmptySensorHub, CONF_HUB_ID

DEPENDENCIES = ['empty_sensor_hub']

binary_sensor_ns = cg.esphome_ns.namespace('binary_sensor')
BinarySensor = binary_sensor_ns.class_('BinarySensor', binary_sensor.BinarySensor, cg.Nameable)

CONFIG_SCHEMA = binary_sensor.BINARY_SENSOR_SCHEMA.extend({
    cv.GenerateID(): cv.declare_id(BinarySensor),
    cv.GenerateID(CONF_HUB_ID): cv.use_id(EmptySensorHub)
}).extend(cv.COMPONENT_SCHEMA)

def to_code(config):
    paren = yield cg.get_variable(config[CONF_HUB_ID])
    var = cg.new_Pvariable(config[CONF_ID])

    yield binary_sensor.register_binary_sensor(var, config)

    cg.add(paren.register_binary_sensor(var))


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import psycopg2
from config import config

conn = psycopg2.connect(config["pg_dsn"])
cursor = conn.cursor()

print("Creating tables")
with open('sql/init.sql') as f:
    cursor.execute(f.read())
    conn.commit()
print("Success (%s)" % (cursor.rowcount,))

print("Filling the database with values")
with open('sql/fill.sql') as f:
    cursor.execute(f.read())
    conn.commit()
    print("Success (%s)" % (cursor.rowcount,))

cursor.close()
conn.close()

print("Database initialized successfully")


from django.forms import ModelForm

from core.forms import SkipFieldValidationMixin
from core.models import LegalBasis

class LegalBasisForm(SkipFieldValidationMixin, ModelForm):
    class Meta:
        model = LegalBasis
        fields = "__all__"
        exclude = []
        heading = "Add Legal Basis"
        heading_help = "Capture the legal grounds for processing of this dataset under GDPR (personal data only). This can require support from your data stewards and data protection officer (DPO)."

    def __init__(self, *args, **kwargs):
        dataset = kwargs.pop("dataset", None)
        super().__init__(*args, **kwargs)
        # we don't allow editing dataset
        self.fields.pop("dataset")
        self.fields["data_declarations"].choices = [
            (d.id, d.title) for d in dataset.data_declarations.all()
        ]

    field_order = [
        "data_declarations",
        "legal_basis_types",
        "personal_data_types",
        "remarks",
    ]

class LegalBasisEditForm(ModelForm):
    class Meta:
        model = LegalBasis
        fields = "__all__"
        exclude = []

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # we don't allow editing dataset
        self.fields.pop("dataset")
        self.fields["data_declarations"].choices = [
            (d.id, d.title) for d in kwargs["instance"].dataset.data_declarations.all()
        ]


import datetime
import os
import general
# create project
def create_project():
    id = input("Enter the id of your project: ")
    while not general.validate_id(id):
        print("Re-enter id  ")
        id = input("nter the id of your project: ")

    user_id = input("Enter your id: ")
    while not general.validate_id(user_id):
        print("Re-enter id  ")
        user_id = input("Enter your id: ")
    title = input("Enter the title of your project: ")

    # check if  project is unique
    if check_project(id,user_id,title):
         print(f"Project with id {id} and title {title} Already exists..")
         return

    details = input("Enter the details of your project: ")
    total_target = float(input("Enter the total target amount for your project: "))
    start_date_str = input("Enter the start date for your project (YYYY-MM-DD): ")
    end_date_str = input("Enter the end date for your project (YYYY-MM-DD): ")
    # Validate start/end date
    try:
        start_date = datetime.datetime.strptime(start_date_str,"%Y-%m-%d").date()
        end_date = datetime.datetime.strptime(end_date_str,"%Y-%m-%d").date()
    except ValueError:
        print("Invalid date format. Please enter dates in the format YYYY-MM-DD .")
        return
    if start_date >= end_date:
        print("End date must be after start date.")
        return

    data=f"{id}:{user_id}:{title}:{details}:{total_target}:{start_date}:{end_date}"
    record="id:user_id:title:details:total_target:start_date:end_date"+"\n"
    general.save('projects.txt',data,record)
    print("Project Scssfully Created")

def check_project(id,user_id,title):
    if os.path.exists('projects.txt'):
      with open('projects.txt', 'r') as file:
            for line in file:
                fields = line.strip().split(':')
                if  fields[0] == id and fields[1] == user_id and fields[2]== title:
                    return True
      file.close()
    else:
        with open('projects.txt',"a") as file:
            pass
        file.close()

#    edit
def edit_project():
     id = input("Enter the id of your project: ")
     if search_project_id(id) :
        title = input("Enter the title of your project: ")
        details = input("Enter the details of your project: ")
        total_target = float(input("Enter the total target amount for your project: "))
        start_date_str = input("Enter the start date for your project (YYYY-MM-DD): ")
        end_date_str = input("Enter the end date for your project (YYYY-MM-DD): ")
        # Validate start/end date
        try:
          start_date = datetime.datetime.strptime(start_date_str,"%Y-%m-%d").date()
          end_date = datetime.datetime.strptime(end_date_str,"%Y-%m-%d").date()
        except ValueError:
         print("Invalid date format. Please enter dates in the format YYYY-MM-DD .")
         return
        if start_date >= end_date:
         print("End date must be after start date.")
         return

        with open('projects.txt', 'r') as file:
            lines=file.readlines()
            for i,line in enumerate(lines):
                fields = line.strip().split(':')
                if fields[0]==id:
                    fields[2]=title
                    fields[3]=details
                    fields[4]=str(total_target)
                    fields[5]=str(start_date)
                    fields[6]=str(end_date)
                    lines[i] = ':'.join(fields) + '\n'

        file.close()
        with open("projects.txt",'w') as file:
            file.writelines(lines)
        file.close()

        print("Project Updated Succssfully")

     else:
          print("Project not found")
          return

def search_project_id(id):
     with open("projects.txt",'r') as file:
         for line in file:
              fields=line.strip().split(":")
              stored_id=fields[0]
              if stored_id == id :
                   return True
     file.close()

# view all projects
def view_projects():
      with open('projects.txt', 'r') as file:
            for line in file:
                print(line,end="")
      file.close()

#   delete project
def delete_project():
    id =  input("Enter the ID of the project to delete: ")

    # Check if project exists
    status=False
    with open('projects.txt', 'r') as file:
            lines=file.readlines()
            for i,line in enumerate(lines):
                fields = line.strip().split(':')
                if  fields[0] == id :
                    del lines[i]
                    status=True
    with open("projects.txt",'w') as file:
            file.writelines(lines)
    file.close()

    # Delete project
    if status:

     print("Project deleted successfully.")
    else:
         print("Project not found")

# serach project by date
def search_project():
      # Get start/end date to search for
    start_date_str = input("Enter the start date to search for (YYYY-MM-DD): ")
    end_date_str = input("Enter the end date to search for (YYYY-MM-DD): ")

    # Validate start/end date
    try:
        start_date = datetime.datetime.strptime(start_date_str,"%Y-%m-%d").date()
        end_date   = datetime.datetime.strptime(end_date_str,"%Y-%m-%d").date()
    except ValueError:
        print("Invalid date format. Please enter dates in the format YYYY-MM-DD.")
        return
    if start_date >= end_date:
        print("End date must be after start date.")
        return
    print (type(start_date))
    with open("projects.txt",'r') as file:
         l=file.readline()
         for line in file:

              fields=line.strip("\n").split(":")
              start=datetime.datetime.strptime(fields[5],"%Y-%m-%d").date()
              end=datetime.datetime.strptime(fields[6],"%Y-%m-%d").date()
              if start_date >= start and end <= end_date:
                   print(line)

    file.close()


#!/usr/bin/env python

import os
import tempfile

from ..model.summarystatistics import FormattedSummaryStatistics
from ..model.ld import LDScores
from ..model.frequencies import Frequencies
from ..model.weights import Weights

class PartitionedHeritability:

    @staticmethod
    def partitionHeritability(summaryStatistics, ld, weights, frequencies, output, print_coefficients = False):
        command = """
            ldsc.py \
                --h2 {summaryStatistics} \
                --ref-ld-chr {ld} \
                --w-ld-chr {weights} \
                --frqfile-chr {frequencies} \
                --overlap-annot \
                --out {output} {print_coefficients}
        """.format(
            summaryStatistics = summaryStatistics,
            ld = ld.fileNamePrefix(),
            weights = weights.fileNamePrefix(),
            frequencies = frequencies.fileNamePrefix(),
            output = output,
            print_coefficients = "--print-coefficients" if print_coefficients else ""
        )
        if os.system(command) != 0:
            raise ChildProcessError("failed to perform heritability partitioning with command %s" % command)

    def __init__(self, summaryStatistics, ld, weights, frequencies, prefix = "h2", print_coefficients = False):
        self.summaryStatistics = summaryStatistics
        self.ld = ld
        self.weights = weights
        self.frequencies = frequencies
        self.prefix = prefix
        self.print_coefficients = print_coefficients

    def __enter__(self):
        self.directory = tempfile.TemporaryDirectory()
        PartitionedHeritability.partitionHeritability(
            self.summaryStatistics, self.ld, self.weights, self.frequencies,
            os.path.join(self.directory.name, self.prefix),
            self.print_coefficients
        )
        self.file = open(os.path.join(self.directory.name, self.prefix + ".results"), 'rt')
        return self.file

    def __exit__(self, *args):
        self.directory.cleanup()
        self.file.close()


from odoo import models, fields, api, exceptions, _
from odoo.exceptions import AccessError, UserError, ValidationError
from datetime import datetime

class HrInductionMaster(models.Model):
	_name = 'hr.induction.master'
	_inherit = ['mail.thread']
	_description = 'Induction Master'

	name = fields.Char(string="Particulars")
	session_details = fields.Char(string="Session Details")
	responsibility = fields.Many2one('hr.department',string="Session Taken by/Responsibility")
	active = fields.Boolean('Active', default=True,help="If unchecked, it will allow you to hide the induction master without removing it.")
	induction_master_created_date = fields.Datetime(string="Induction Master Created",default=lambda self: fields.datetime.now(),track_visibility='onchange',readonly=True)

class HrInduction(models.Model):
	_name = 'hr.induction'
	_description = "Induction"
	_inherit = ['mail.thread']
	_order = 'id desc'

	@api.model
	def _default_induction_line(self):
		master = self.env['hr.induction.master']
		vals = []
		master_ids = master.search([])

		for line in master_ids:
			data={}
			data['session_details'] = line.session_details
			data['name'] = line.name
			data['responsibility'] = line.responsibility.id
			vals.append((0,0,data))
		return vals

	@api.model
	def default_get(self, fields):
		res = super(HrInduction, self).default_get(fields)
		emp_user_id = self.env.uid
		created_hr_employee_id = self.env['hr.employee'].search([('user_id', '=', emp_user_id)])
		if created_hr_employee_id:
			res['created_hr_employee_id'] = created_hr_employee_id.id
		return res

	name = fields.Char(string="Seq No.",track_visibility='onchange')
	seq_date = fields.Datetime(string="Seq Date",default=lambda self: fields.datetime.now())
	induction_created_date = fields.Date(string="Doc Date",default=lambda self: fields.date.today(),track_visibility='onchange',readonly=True)
	employee_id = fields.Many2one('hr.employee',string="Employee Name",track_visibility='onchange',required=True)
	employee_code = fields.Char(string="Employee Id",related="employee_id.employee_id",store=True)
	date_of_joining = fields.Date(string="Date of Joining",related="employee_id.joining_date",store=True)
	reporting_manager = fields.Many2one('hr.employee',string="Approver 1",related="employee_id.lone_manager_id",store=True)
	reporting_manager_user_id = fields.Many2one('res.users',string="Approver 1 User Id",related="employee_id.lone_manager_id.user_id",store=True)
	department = fields.Many2one('hr.department',string="Department",related="employee_id.department_id",store=True,track_visibility='onchange')
	location_work_id = fields.Many2one('work.location', string="Work Location",related="employee_id.location_work_id",track_visibility='onchange')
	# state = fields.Selection([('draft','Draft'),('hr_submit','Induction Created'),('acknowledged','Acknowledged by Approver 1'),('reject','Disagreed'),('resubmit','Resubmitted'),('emp_approval','Agreed by Employee'),('done','Completed')],string="Stage",default='draft',track_visibility='onchange')
	state = fields.Selection([('draft','Draft'),('hr_submit','Induction Created'),('emp_approval','Agreed by Employee'),('reject','Disagreed'),('resubmit','Resubmitted'),('acknowledged','Acknowledged by Approver 1'),('done','Completed')],string="Stage",default='draft',track_visibility='onchange')

	induction_line_ids = fields.One2many('hr.induction.line','induction_id',default = _default_induction_line)
	user_id = fields.Many2one('res.users', string='Related User', index=True, track_sequence=2, related="employee_id.user_id",store=True)
	company_id = fields.Many2one('res.company', 'Company', default=lambda self: self.env.user.company_id.id)
	remarks_by_l1_manager = fields.Text(string="Acknowledgement by Approver 1",track_visibility='onchange',readonly=True)
	remarks_by_employee = fields.Text(string="Reason for Rejection / Disagree",track_visibility='onchange',readonly=True)
	document_created_by = fields.Many2one('res.users',default=lambda self: self.env.uid,string="Hr User Id")
	induction_confirm_date = fields.Datetime(string='Induction Confirmed by HR', track_visibility='onchange', track_sequence=2)
	emp_accept_date = fields.Datetime(string='Employee Agreed', track_visibility='onchange', track_sequence=2)
	emp_rejected_date = fields.Datetime(string='Employee Disagreed', track_visibility='onchange', track_sequence=2)
	hr_resubmit_date = fields.Datetime(string='HR ReSubmitted', track_visibility='onchange', track_sequence=2)
	l1_manager_ack_date = fields.Datetime(string='L1 Manager Acknowledge', track_visibility='onchange', track_sequence=2)
	created_hr_employee_id = fields.Many2one('hr.employee', string="HR")

	# get Users for Notification/Reminders
	@api.multi
	@api.depends('employee_id')
	def compute_employee_approver1_id(self):
		employee_rec = self.env['hr.employee'].sudo().search([('id', '=', self.employee_id.id)])
		for rec in employee_rec:
			self.hr_reminder_approver1_ids = rec.lone_manager_id
			self.hr_reminder_approver2_ids = rec.ltwo_manager_id
			self.hr_reminder_hod_ids = rec.hod_id
			self.hr_reminder_director_ids = rec.parent_id
			self.employee_ids = rec
			self.hr_ids = self.created_hr_employee_id

	hr_reminder_approver1_ids = fields.Many2many('hr.employee', string='Noti1 Approver1',
												 compute="compute_employee_approver1_id")
	hr_reminder_approver2_ids = fields.Many2many('hr.employee', string='Noti2 Approver2',
												 compute="compute_employee_approver1_id")
	hr_reminder_hod_ids = fields.Many2many('hr.employee', string='Noti3 Hod',
										   compute="compute_employee_approver1_id")
	hr_reminder_director_ids = fields.Many2many('hr.employee', string='Noti4 Director',
												compute="compute_employee_approver1_id")
	employee_ids = fields.Many2many('hr.employee', string='Noti5 Employee Ids', compute="compute_employee_approver1_id")
	hr_ids = fields.Many2many('hr.employee', string='Noti6 Employee Ids', compute="compute_employee_approver1_id")

	@api.model
	def create(self,vals):
		vals['name'] = self.env['ir.sequence'].next_by_code('hr.induction')
		rec = super(HrInduction,self).create(vals)
		return rec

	@api.multi
	def action_hr_confirm(self):
		is_hr = self.env.user.has_group('hr_employee_kra.group_kra_hr')
		for line in self:
			if is_hr:
				line.write({'state':'hr_submit','induction_confirm_date': datetime.now()})
				template_id = self.env.ref('hr_employee_kra.email_template_induction_to_employee')
				template_id.send_mail(self.id, force_send=True)
				# Reminder Notification
				hr_reminder = line.env['hr.reminder'].sudo().create({
					'name': 'Induction process is completed for you',
					'employee_id': line.employee_id.id, 'model_name': 'hr.induction',
					'approver_ids': [(6, 0, line.employee_ids.ids)],
					'hr_induction_id': line.id
				})
			else:
				raise UserError(_('You are not a authorized user to perform actions in this document.'))

	@api.multi
	def action_emp_reject(self):
		current_employee = self.env.uid
		is_employee = self.env.user.has_group('hr_employee_kra.group_kra_user')
		for line in self:
			if line.user_id.id != current_employee:
				raise UserError(_('You are not a authorized user to perform actions in this document.'))
			if line.user_id.id == current_employee:
				template_id = self.env.ref('hr_employee_kra.email_template_induction_employee_rejection_to_hr')
				template_id.send_mail(self.id, force_send=True)
				# Reminder Notification
				hr_reminder = line.env['hr.reminder'].sudo().create({
					'name': 'Employee disagreed Induction Checklist',
					'employee_id': line.employee_id.id, 'model_name': 'hr.induction',
					'approver_ids': [(6, 0, line.hr_ids.ids)],
					'hr_induction_id': line.id
				})

				# line.write({'state':'reject','emp_rejected_date': datetime.now()})
				form_view = self.env.ref('hr_employee_kra.form_induction_emp_reject_remark_wizard')
				return {
		            'name': "Reason for Rejection / Disagree",
		            'view_mode': 'form',
		            'view_type': 'form',
		            'view_id': form_view.id,
		            'res_model': 'induction.emp.reject.remark',
		            'type': 'ir.actions.act_window',
		            'target': 'new',
		            'context': {
		                'induction_id': self.ids, 'is_reject': True
		            },
		            # 'emp_rejected_date': datetime.now()
		        }

	@api.multi
	def action_emp_accept(self):
		current_employee = self.env.uid
		is_employee = self.env.user.has_group('hr_employee_kra.group_kra_user')
		for line in self:
			if line.user_id.id != current_employee:
				raise UserError(_('You are not a authorized user to perform actions in this document.'))
			if line.user_id.id == current_employee:
				var = line.remarks_by_employee
				# print("BBBB", var)
				# line.remarks_by_employee.replace("Disagree", "")
				line.write({'state':'emp_approval','emp_accept_date': datetime.now()})
				template_id = self.env.ref('hr_employee_kra.email_template_induction_employee_confirms_to_a1')
				template_id.send_mail(self.id, force_send=True)
				# Reminder Notification
				hr_reminder = line.env['hr.reminder'].sudo().create({
					'name': 'Employee had submitted his Induction',
					'employee_id': line.employee_id.id, 'model_name': 'hr.induction',
					'approver_ids': [(6, 0, line.hr_reminder_approver1_ids.ids)],
					'hr_induction_id': line.id
				})

	@api.multi
	def action_hr_resubmit(self):
		is_hr = self.env.user.has_group('hr_employee_kra.group_kra_hr')
		for line in self:
			if is_hr:
				line.write({'state':'resubmit','remarks_by_employee':'','hr_resubmit_date': datetime.now()})
			else:
				raise UserError(_('You are not a authorized user to perform actions in this document.'))

	@api.multi
	def action_l1_manager_acknowledge(self):
		current_employee = self.env.uid
		is_approver_1 = self.env.user.has_group('hr_employee_kra.group_kra_approver_1')
		for line in self:
			if line.reporting_manager_user_id.id != current_employee:
				raise UserError(_('You are not a authorized user to Acknowledge this document.'))
			if line.reporting_manager_user_id.id == current_employee and is_approver_1:
				# line.write({'state':'acknowledged','l1_manager_ack_date' : datetime.now()})
				form_view = self.env.ref('hr_employee_kra.form_induction_acknowledge_remark_wizard')
				return {
		            'name': "Acknowledgement",
		            'view_mode': 'form',
		            'view_type': 'form',
		            'view_id': form_view.id,
		            'res_model': 'induction.acknowledge.remark',
		            'type': 'ir.actions.act_window',
		            'target': 'new',
		            'context': {
		                'induction_id': self.ids, 'is_acknowledge': True
		            },
		            #'l1_manager_ack_date' : datetime.now()
		        }

	@api.multi
	def action_hr_confirm_resubmit(self):
		current_employee = self.env.uid
		is_hr = self.env.user.has_group('hr_employee_kra.group_kra_hr')
		for line in self:
			if is_hr:
				line.write({'state':'hr_submit','hr_resubmit_date': datetime.now()})
			else:
				raise UserError(_('You are not a authorized user to perform actions in this document.'))

	@api.onchange('employee_id')
	def onchange_employee_id(self):
		if self.employee_id:
			if self.env['hr.induction'].search([('employee_id', '=', self.employee_id.id)]):
				raise UserError(_('Already Induction Created for this Employee.'))

class HrInductionLine(models.Model):
	_name = 'hr.induction.line'
	# _inherit = ['mail.thread']

	induction_id = fields.Many2one('hr.induction')

	name = fields.Char(string="Particulars")
	session_details = fields.Char(string="Session Details",track_visibility="onchange")
	responsibility = fields.Many2one('hr.department',string="Session Taken by/Responsibility")
	completed = fields.Selection([('completed','Completed'),('pending', 'Pending'), ('na','N/A')],string="Status",track_visibility='onchange')
	user_id = fields.Many2one('User', default=lambda self: self.env.uid)
	is_hr = fields.Boolean('HR', compute='_compute_hr')

	@api.depends('user_id')
	def _compute_hr(self):
		for rec in self:
			var = self.env.user.has_group('hr_employee_kra.group_kra_hr')
			# print("VAR", var)
			if var:
				rec.is_hr = True


# Initial Python environment setup...
import numpy as np # linear algebra
import pandas as pd # CSV file I/O (e.g. pd.read_csv)
import os # reading the input files we have access to
import datetime as dt
import matplotlib.pyplot as plt
import seaborn as sns
print(os.listdir('../input'))
# df = pd.DataFrame(index=pd.DatetimeIndex(start=dt.datetime(2016,1,1,0,0,1),
#     end=dt.datetime(2016,1,2,0,0,1), freq='H'))\
#     .reset_index().rename(columns={'index':'datetime'})

# df.head()
# df.dtypes
# df['ts'] = df.datetime.values.astype(np.int64) // 10 ** 9
# df.head()
# df.dtypes
# # t = pd.Timestamp(2017, 1, 1, 12)
# # ts = t.astye
train_df =  pd.read_csv('../input/train.csv', nrows = 10_000_000)
train_df.dtypes
#striping the timezone
def convert_to_datetime(df):
    test_time = df['pickup_datetime'].astype(str).str[:-4]
    df['date_time'] =  pd.to_datetime(test_time, format='%Y%m%d %H:%M:%S')
    return df

# converting the object to date time
train_df = convert_to_datetime(train_df)
# train_df.dtypes
train_df.head()
train_df.describe()
def normalize_fare_passenger(df):
    if 'fare_amount' in df.columns:
        print("old lenght: %d" %len(df))
        df = df[df.fare_amount>0]
    print("length after fare_amount normalization: %d" %len(df))
    df = df[df.passenger_count>0]
    print("length after passenger_count normalization: %d" %len(df))
    return df

train_df = normalize_fare_passenger(train_df)
train_df.head()
# Given a dataframe, add two new features 'abs_diff_longitude' and
# 'abs_diff_latitude' reprensenting the "Manhattan vector" from
# the pickup location to the dropoff location.
def add_travel_vector_features(df):
    df['abs_diff_longitude'] = (df.dropoff_longitude - df.pickup_longitude).abs()
    df['abs_diff_latitude'] = (df.dropoff_latitude - df.pickup_latitude).abs()
    return df

train_df = add_travel_vector_features(train_df)
print(train_df.isnull().sum())
print('Old size: %d' % len(train_df))
train_df = train_df.dropna(how = 'any', axis = 'rows')
print('New size: %d' % len(train_df))
train_df.head()
plot = train_df.iloc[1:2000].plot.scatter('abs_diff_longitude', 'abs_diff_latitude')
print('Old size: %d' % len(train_df))
train_df = train_df[(train_df.abs_diff_longitude < 5.0) & (train_df.abs_diff_latitude < 5.0)]
print('New size: %d' % len(train_df))
train_df[train_df.fare_amount<60].fare_amount.hist(bins=200, figsize=(14,3))
plt.xlabel('fare $USD')
plt.title('Histogram');
train_df.dtypes
#this is a kind of haversine formula to calculate the spherical distance
def haversine_distance_calculation(df):
    lon1, lat1, lon2, lat2 = df['pickup_longitude'], df['pickup_latitude'], df['dropoff_longitude'], df['dropoff_latitude']
    lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = np.sin(dlat/2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2.0)**2
    c = 2 * np.arcsin(np.sqrt(a))
    distance = 6367 * c
    df['distance'] = distance
    return df
train_df = haversine_distance_calculation(train_df)
train_df.head()
train_df.plot(x='distance', y='fare_amount', style = 'o')
def distance_fare_normalization(df):
    print("old lenght with distance greated than 50km: %d" %len(df))
    df = df[df.distance<50]
    print("length after distance normalization: %d" %len(df))
    if 'fare_amount' in df.columns:
        df = df[df.fare_amount<100]
        print("length after fare_amount normalization: %d" %len(df))
    return df

train_df = distance_fare_normalization(train_df)
train_df.head()
train_df.plot(x='distance', y='fare_amount', style = 'o')
def day_converter(df):
    day_of_year = df['date_time'].dt.dayofyear
    day_of_week = df['date_time'].dt.dayofweek
    df['day_of_year'] = day_of_year
    df['day_of_week'] = day_of_week
    return df

train_df = day_converter(train_df)
train_df.plot(x='day_of_year', y='fare_amount', style = 'o')
train_df.head()
# Construct and return an Nx3 input matrix for our linear model
# using the travel vector, plus a 1.0 for a constant bias term.
##MH:adding pickup_datetime in train_df
def get_input_matrix(df):
    return np.column_stack((df.abs_diff_longitude, df.abs_diff_latitude, df.distance, df.day_of_year, df.day_of_week,np.ones(len(df))))

train_X = get_input_matrix(train_df)
train_y = np.array(train_df['fare_amount'])

print(train_X.shape)
print(train_y.shape)
# The lstsq function returns several things, and we only care about the actual weight vector w.
(w, _, _, _) = np.linalg.lstsq(train_X, train_y, rcond = None)
print(w)
w_OLS = np.matmul(np.matmul(np.linalg.inv(np.matmul(train_X.T, train_X)), train_X.T), train_y)
print(w_OLS)
test_df = pd.read_csv('../input/test.csv')
test_df.dtypes
# Reuse the above helper functions to add our features and generate the input matrix.
add_travel_vector_features(test_df)
#
##converting to date time
test_df = convert_to_datetime(test_df)
##Normalizing fare_amount>0 & passenger_count>0
test_df = normalize_fare_passenger(test_df)
## dropping null values
test_df = test_df.dropna(how = 'any', axis = 'rows')
test_df =haversine_distance_calculation(test_df)
##normalization distance>50km and fare>100usd
test_df = distance_fare_normalization(test_df)
#converting date time to day vactor
test_df = day_converter(test_df)

test_X = get_input_matrix(test_df)
# Predict fare_amount on the test set using our model (w) trained on the training set.
test_y_predictions = np.matmul(test_X, w).round(decimals = 2)

# Write the predictions to a CSV file which we can submit to the competition.
submission = pd.DataFrame(
    {'key': test_df.key, 'fare_amount': test_y_predictions},
    columns = ['key', 'fare_amount'])
submission.to_csv('submission.csv', index = False)

print(os.listdir('.'))


import matplotlib.pyplot as plt
import numpy as np
import os
import pickle

def get_mean_stds(data):
    return np.mean(data), np.std(data) / np.sqrt(len(data)) * 1.96

if __name__ == '__main__':

    labels = ['OpenTAL', 'EDL', 'SoftMax']
    result_folders = ['edl_oshead_iou', 'edl_15kc', 'default']
    colors = ['k', 'g', 'm']
    split = '0'
    tiou_target = 0.3
    tidx = 0  # 0-4 for [0,3...,0.7]

    items = ['$TP_{u2u}$', '$TP_{k2k}$', '$FP_{u2k}$', '$FP_{k2k}$', '$FP_{k2u}$', '$FP_{bg2u}$', '$FP_{bg2k}$']
    fontsize = 18
    width = 0.25
    fig_path = 'experiments/figs'
    os.makedirs(fig_path, exist_ok=True)

    xrng = np.arange(len(items))
    fig, ax = plt.subplots(1,1, figsize=(8,5))
    plt.rcParams["font.family"] = "Arial"
    for idx, (folder, label, color) in enumerate(zip(result_folders, labels, colors)):
        # load result file
        result_file = os.path.join('output', folder, f'split_{split}', 'open_stats.pkl')
        with open(result_file, 'rb') as f:
            stats = pickle.load(f)
        print(label)
        all_scores = 1 - np.array(stats['ood_scores'])
        mean_scores = np.zeros((7))
        std_scores = np.zeros((7))
        mean_scores[0], std_scores[0] = get_mean_stds(all_scores[stats['tp_u2u'][tidx] > 0])
        mean_scores[1], std_scores[1] = get_mean_stds(all_scores[stats['tp_k2k'][tidx].sum(axis=0) > 0])
        mean_scores[2], std_scores[2] = get_mean_stds(all_scores[stats['fp_u2k'][tidx].sum(axis=0) > 0])
        mean_scores[3], std_scores[3] = get_mean_stds(all_scores[stats['fp_k2k'][tidx].sum(axis=0) > 0])
        mean_scores[4], std_scores[4] = get_mean_stds(all_scores[stats['fp_k2u'][tidx] > 0])
        mean_scores[5], std_scores[5] = get_mean_stds(all_scores[stats['fp_bg2u'][tidx] > 0])
        mean_scores[6], std_scores[6] = get_mean_stds(all_scores[stats['fp_bg2k'][tidx].sum(axis=0) > 0])

        h = ax.bar(xrng + (idx-1) * width, mean_scores, yerr=std_scores, width=width, label=f'{label}', align='center', alpha=0.5, ecolor='black', color=color)

    ax.set_ylim(0, 1.2)
    ax.set_ylabel('OOD Scores', fontsize=fontsize)
    ax.set_xticks(xrng)
    ax.set_xticklabels(items, fontsize=fontsize-3)
    ax.legend(fontsize=fontsize, loc='upper center', ncol=3)
    plt.yticks(fontsize=fontsize)
    plt.tight_layout()
    plt.savefig(os.path.join(fig_path, 'OOD_Score_compare.png'))


import os
import shutil
from tqdm import tqdm

# Specify the directories containing the images and labels
src_dirs = [
    'data/day/img/train',
    'data/day/img/test',
    'data/day/img/val',
    'data/night/img/train',
    'data/night/img/test',
    'data/night/img/val',
]

# Specify the directory to move images without corresponding labels
dst_dir = 'data/without_labels'

# Create the destination directory if it does not exist
os.makedirs(dst_dir, exist_ok=True)

# Initialize a counter for the number of images moved
total_count = 0

# For each source directory
for src_dir in src_dirs:
    # Get a list of all files in the source directory
    files = os.listdir(src_dir)

    # Initialize a counter for the number of images moved
    count = 0

    # For each file in the source directory
    for file in tqdm(files, desc=f'Processing {src_dir}', unit='file'):
        # Get the file name without the extension
        name, ext = os.path.splitext(file)
        # If the file is an image
        if ext in ['.jpg', '.png']:
            # If there is no corresponding .txt file
            if name + '.txt' not in files:
                # Define the destination path
                dst_path = os.path.join(dst_dir, file)
                # If the file already exists in the destination directory
                if os.path.exists(dst_path):
                    # Append a number to the file name to make it unique
                    i = 1
                    while os.path.exists(os.path.join(dst_dir, f'{name}_{i}{ext}')):
                        i += 1
                    dst_path = os.path.join(dst_dir, f'{name}_{i}{ext}')
                # Move the image to the destination directory
                shutil.move(os.path.join(src_dir, file), dst_path)
                # Increment the counter
                count += 1

    print(f'Moved {count} images from {src_dir} without corresponding .txt files')
    total_count += count

print(f'Total moved images: {total_count}')


import argparse
import matplotlib.pyplot as plt
import csv
import numpy as np
from sys import argv

class Data:
    def __init__(self, raw):
        self.foreground = raw[0]
        self.background = raw[1]
        self.desc = raw[2]
        self.value = raw[3]
        self.uninterfered = float(raw[4])
        self.interfered = float(raw[5])
        r = raw[6].replace('%', '')
        self.delta = float(r)
    def fore(self):
        return self.foreground
    def back(self):
        return self.background

parser = argparse.ArgumentParser()
parser.add_argument("-s", '--save', help='save file', default='img.png')
parser.add_argument("-c", '--color', help='define matplotlib colormap', default='RdYlBu')
parser.add_argument('-i', '--input', help='input data file', default='raw_data.csv')
args = parser.parse_args()

svname = args.save
color = args.color
fname = args.input

with open(fname, 'r') as f:
    x = [line.strip().split(',') for line in f]
x = x[1:]
data = [Data(line) for line in x]
labels = list({d.foreground for d in data if d.foreground != 'fio'})
labels.sort()
maps = {l:{k: [] for k in labels} for l in labels}
for d in data:
    if d.fore() == 'fio' or d.back() == 'fio':
        continue
    maps[d.fore()][d.back()].append(d.delta)

for fore in labels:
    for back in labels:
        arr = maps[fore][back]
        k = len(arr)
        if k == 1:
            maps[fore][back] = round(arr[0], 2)
        else:
            maps[fore][back] = round(sum(arr) / k, 2)
arr = []
for fore in labels:
    temp = []
    for back in labels:
        temp.append(maps[fore][back])
    arr.append(temp)

fig, ax = plt.subplots()
im = ax.imshow(arr, cmap=color)
cbar = ax.figure.colorbar(im, ax=ax, cmap=color)
cbar.ax.set_ylabel('% interference', rotation=-90, va='bottom')
ax.set_xticks(np.arange(len(labels)))
ax.set_yticks(np.arange(len(labels)))

ax.set_xticklabels(labels)
ax.set_yticklabels(labels)

plt.xlabel("Foreground")
plt.ylabel("Background")

plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")

for i in range(len(labels)):
    for j in range(len(labels)):
        text = ax.text(j, i, maps[labels[i]][labels[j]], ha="center",
                va='center', color='black')
ax.set_title("VM interference")

fig.tight_layout()
plt.savefig(svname, format='png')


#!/usr/bin/env python3

import glob
import os
import math
import pickle
import argparse
import logging as log
import numpy as np
import tensorflow as tf

from time import time
from datetime import timedelta

def read_dict(file_name):
    with open(file_name, "rb") as fd:
        return pickle.load(fd)

def build_input_pipeline(input_file, batch_size):
    input_files = tf.train.string_input_producer([input_file])
    reader = tf.FixedLengthRecordReader(record_bytes=4+4)
    _, raw_val_t = reader.read(input_files)
    int_val_t = tf.decode_raw(raw_val_t, tf.int32)
    center_t, context_t = int_val_t[0], int_val_t[1]

    center_batch_t, context_batch_t = tf.train.batch([center_t, context_t], batch_size, num_threads=4, capacity=1024)
    return center_batch_t, context_batch_t

def get_num_entries(input_file):
    st = os.stat(input_file)
    return st.st_size / 8

if __name__ == "__main__":
    REPORT_EVERY_STEPS = 100
    SAVE_EMBEDDINGS_EVERY = 1000

    log.basicConfig(level=log.INFO, format="%(asctime)s %(levelname)s %(message)s")

    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--dict", required=True, help="File with dict serialized data")
    parser.add_argument("-t", "--train", required=True, help="Train data file")
    parser.add_argument("-r", "--run", default="w2v", help="Name of this run, default=w2v")
    args = parser.parse_args()

    log.info("Reading dict from %s", args.dict)
    dict_data = read_dict(args.dict)
    log.info("Dict has %d entries", len(dict_data))

    input_entries = get_num_entries(args.train)
    log.info("Input data has %d samples", input_entries)

    dict_size = len(dict_data)
    vec_size = 100
    batch_size = 1024*10
    num_sampled = 10
    log.info("Training params: vec_size=%d, batch=%d, num_neg=%d", vec_size, batch_size, num_sampled)

    embeddings = tf.Variable(tf.random_uniform([dict_size, vec_size], -1.0, 1.0))

    norm_t = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
    normalized_embeddings_t = embeddings / norm_t

    nce_weights = tf.Variable(
        tf.truncated_normal([dict_size, vec_size],
                            stddev=1.0 / math.sqrt(vec_size)))

    center_batch_t, context_batch_t = build_input_pipeline(args.train, batch_size)

    # Placeholders for inputs
    train_inputs_t = tf.placeholder(tf.int32, shape=[batch_size])
    train_labels_t = tf.placeholder(tf.int32, shape=[batch_size, 1])
    embed = tf.nn.embedding_lookup(embeddings, train_inputs_t)
    nce_weights = tf.Variable(
        tf.truncated_normal([dict_size, vec_size],
                            stddev=1.0 / math.sqrt(vec_size)))
    nce_biases = tf.Variable(tf.zeros([dict_size]))

    loss_t = tf.reduce_mean(
        tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels_t,
                       num_sampled, dict_size))

    global_step_t = tf.Variable(0, name='global_step', trainable=False)
    #optimizer = tf.train.AdamOptimizer(learning_rate=0.03).minimize(loss_t, global_step=global_step_t)
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0).minimize(loss_t, global_step=global_step_t)

    # summaries
    tf.scalar_summary('loss_cur', loss_t)
    loss_avg_t = tf.Variable(0.0, name='loss_avg', trainable=False)
    tf.scalar_summary('loss_avg', loss_avg_t)
    speed_t = tf.Variable(0.0, name="speed", trainable=False)
    tf.scalar_summary('speed', speed_t)

    init = tf.initialize_all_variables()
    merged_summaries = tf.merge_all_summaries()
    time_started = time()

    # limit amount of allocated memory to 1/2 of 8GB
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4)

    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as session:
        summary_writer = tf.train.SummaryWriter("logs/" + args.run, session.graph)
        session.run(init)

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(session, coord)

        try:
            sum_loss = 0.0
            while True:
                batch, labels = session.run([center_batch_t, context_batch_t])

                feed_dict = {
                    train_inputs_t: batch,
                    train_labels_t: np.expand_dims(labels, 1)
                }

                _, step, loss = session.run([optimizer, global_step_t, loss_t], feed_dict=feed_dict)
                sum_loss += loss

                if step % REPORT_EVERY_STEPS == 0:
                    epoch = int(step * batch_size / input_entries)
                    sec_passed = time() - time_started
                    speed = step * batch_size / sec_passed

                    summary_res, = session.run([merged_summaries], feed_dict={
                        loss_avg_t: sum_loss / REPORT_EVERY_STEPS,
                        loss_t: loss,
                        speed_t: speed,
                    })
                    summary_writer.add_summary(summary_res, step)

                    print("%s: epoch=%d, step=%d, loss=%f, speed=%.3f samples/s" % (
                        timedelta(seconds=sec_passed), epoch,
                        step, sum_loss / REPORT_EVERY_STEPS, speed))
                    sum_loss = 0.0

                if step % SAVE_EMBEDDINGS_EVERY == 0:
                    norm_embeddings, = session.run([normalized_embeddings_t])
                    existing = glob.glob(args.run + "-*.npy")
                    np.save(args.run + "-%d" % step, norm_embeddings)
                    for prev_save in existing:
                        os.unlink(prev_save)
        finally:
            coord.request_stop()

        coord.join(threads)


from django.db import models
from django.contrib.auth.models import AbstractUser

# Create your models here.

class MyUser(AbstractUser):
    isOperator = models.BooleanField(default=False)

class Guard(models.Model):
    name = models.CharField(max_length=255)
    staff_id = models.CharField(max_length=255, unique=True)
    date_joined = models.DateTimeField(blank=True, null=True)
    date_left = models.DateTimeField(blank=True, null=True)
    is_deleted = models.BooleanField(default=False)

    def __str__(self):
        return "{} : {}".format(self.name, self.staff_id)

class Wristband(models.Model):
    band_id = models.CharField(max_length=255, unique=True)
    guard = models.OneToOneField(Guard, on_delete=models.SET_NULL, null=True, blank=True, default=None,
                                 related_name='band')
    is_deleted = models.BooleanField(default=False)

    def __str__(self):
        return "band {}".format(self.band_id)

class LogInstance(models.Model):
    lat = models.DecimalField(max_digits=10, decimal_places=7)
    lang = models.DecimalField(max_digits=10, decimal_places=7)
    wristband = models.ForeignKey(Wristband, on_delete=models.CASCADE)
    guard = models.ForeignKey(Guard, on_delete=models.SET_NULL, blank=True, null=True)
    time = models.DateTimeField(auto_now_add=True)
    heartbeat = models.PositiveSmallIntegerField(blank=True, null=True)
    emergency_alert = models.BooleanField(default=False)

    def __str__(self):
        return "log for band {} on {}".format(self.wristband.band_id, self.time)

    class Meta:
        unique_together = ('time', 'wristband')