import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import cv2
import numpy as np
import glob
import re

# 'percentage_binary_thresholding.ipynb'
def percentages_to_thresholds(img_gray, lower_percentage, upper_percentage):
    norm_hist = (np.cumsum(np.histogram(img_gray, 255, (0.0, 255.0))[0]) / \
                 img_gray.size).astype(np.float32)
    values = np.arange(0, 255)[((norm_hist >= lower_percentage) & \
                                (norm_hist <= upper_percentage))]

    return np.min(values), np.max(values)

# 'gradient_experiment.ipynb'
def get_binary_gradient(img, low_thresh=0.95, high_thresh=1.0):
    img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2HLS_FULL)[:,:,1]

    gradient_x = np.absolute(cv2.Sobel(img_gray, cv2.CV_64F, 1, 0))
    gradient_x = 255 * gradient_x / np.max(gradient_x)

    binary = np.zeros_like(img_gray).astype(np.bool8)
    low_thresh, high_thresh = percentages_to_thresholds(gradient_x, \
        low_thresh, high_thresh)

    binary[(gradient_x >= low_thresh) & (gradient_x <= high_thresh)] = True
    return binary

# 'color_scheme_experiment.ipynb'
def Sally(img):
    S = cv2.cvtColor(img, cv2.COLOR_RGB2HLS_FULL)[:,:,2]
    A = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)[:,:,1]

    return np.sqrt( \
                S.astype(np.uint32) * \
                A.astype(np.uint32) \
                  ).astype(np.uint8)

# 'color_scheme_experiment.ipynb'
def get_binary_color(img, low_thresh = 0.9, high_thresh = 1.0):
    img_gray = Sally(img)

    low_thresh, high_thresh = percentages_to_thresholds(img_gray, \
        low_thresh, high_thresh)

    img_binary = np.zeros_like(img_gray).astype(np.bool8)
    img_binary[(img_gray > low_thresh) & (img_gray < high_thresh)] = True

    return img_binary

from moviepy.editor import VideoFileClip

temp = None
def process_image(img):
    binary_R = 255 * np.zeros(img.shape[:2]).astype(np.uint8)
    binary_G = 255 * get_binary_color(img).astype(np.uint8)
    binary_B = 255 * get_binary_gradient(img).astype(np.uint8)
    binary = np.dstack((binary_R, binary_G, binary_B))

    global temp
    if temp is None:
        temp = np.copy(binary)

    return binary

# The video processing and storing code
input_video = VideoFileClip("white.mp4")
output_video = input_video.fl_image(process_image)

output_filename = 'project_output_1.mp4'
output_video.write_videofile(output_filename, audio=False)


# -*- coding: utf-8 -*-

from tqdm import tqdm
import pickle

UNK, PAD = '<UNK>', '<PAD>'  # unknown word, padding character

def _build_vocab(file_path, tokenizer, max_vocab_size, min_word_freq, line_sep):
    vocab_dic = {}
    bad_line = 0
    with open(file_path, 'r', encoding='UTF-8') as f:
        for line in tqdm(f):
            lin = line.strip()
            if not lin:
                bad_line += 1
                continue

            seps = lin.split(line_sep)
            if 1 <= len(seps) <= 2:
                content = seps[0]  # default first part separated is content, second is label
            else:
                bad_line += 1
                continue

            for word in tokenizer(content):
                vocab_dic[word] = vocab_dic.get(word, 0) + 1
        vocab_list = sorted([_ for _ in vocab_dic.items() if _[1] >= min_word_freq],
                            key=lambda x: x[1], reverse=True)[:max_vocab_size]
        vocab_dic = {word_count[0]: idx for idx, word_count in enumerate(vocab_list)}
        vocab_dic.update({UNK: len(vocab_dic), PAD: len(vocab_dic) + 1})
    print(f"bad line number is {bad_line}")
    return vocab_dic

def _line_to_number(content, tokenizer, vocab_dic, pad_size):
    token = tokenizer(content)
    seq_len = len(token)
    if seq_len < pad_size:
        token.extend([vocab_dic.get(PAD)]*(pad_size - seq_len))
    else:
        token = token[: pad_size]
        seq_len = pad_size

    words_line = []
    for word in token:
        words_line.append(vocab_dic.get(word, vocab_dic.get(UNK)))
    return words_line, seq_len

def format_content(file_path, train_granularity, build_vocab, vocab_save_path,
                   max_vocab_size, min_word_freq, pad_size, line_sep, if_with_label=True, word_sep=" "):
    if train_granularity == "word":
        tokenizer = lambda x: x.split(word_sep)
    elif train_granularity == "char":
        tokenizer = lambda x: [y for y in x]
    else:
        raise ValueError("param train_granularity must be in ('word', 'char')!")

    if build_vocab:
        vocab_dic = _build_vocab(file_path, tokenizer, max_vocab_size, min_word_freq, line_sep)
        pickle.dump(vocab_dic, open(vocab_save_path, 'wb'))
        print(f"create vocabulary from {file_path}, vocabulary size {len(vocab_dic)}")
    else:
        vocab_dic = pickle.load(open(vocab_save_path, 'rb'))
        print(f"load vocabulary from {vocab_save_path}, vocabulary size {len(vocab_dic)}")

    contents = []
    bad_line = 0
    with open(file_path, 'r', encoding='UTF-8') as f:
        for line in tqdm(f):
            lin = line.strip()
            if not lin:
                continue
            seps = lin.split(line_sep)
            if if_with_label:
                if len(seps) == 2:
                    content, label = seps
                    words_line, seq_len = _line_to_number(content, tokenizer, vocab_dic, pad_size)
                    contents.append((words_line, seq_len, int(label)))
                else:
                    bad_line += 1
                    continue
            else:
                if len(seps) == 1:
                    content = seps
                    words_line, seq_len = _line_to_number(content, tokenizer, vocab_dic, pad_size)
                    contents.append((words_line, seq_len, None))
                else:
                    bad_line += 1
                    continue
    print(f"bad line number is {bad_line}")
    return contents, len(vocab_dic)


"""
 python  main_run.py arg
 example: python main_run.py --iterations 1000 --replicas 1 --nrsteps 100000 --folderName 'Data' --algorithm 0 --diffMapMetric 'rmsd'

pythonw main_run.py --iterations 100 --replicas 5 --nrsteps 10000 --folderName 'Data' --algorithm 4 --diffMapMetric 'euclidean'

  python main_run.py --iterations 1000 --replicas 1 --nrsteps 100000 --folderName 'Data' --algorithm 0
"""

import numpy as np

import sys
import os
sys.path.append( os.getcwd()+'/srcDiffmap')

import mdtraj as md
import sampler
import integrator

from simtk import openmm, unit
from openmmtools.constants import kB

import model

#create model defined in class model

modelName = 'Alanine'
mdl=model.Model(modelName)

# IC = md.load('alanine_start_state_IC.h5')
# mdl.positions = IC.xyz * mdl.x_unit

print(mdl.x_unit)
print(mdl.modelname)
print('System has %d particle(s)' % mdl.system.getNumParticles())
#systemName='Ala2Pept'

# Parse command-line arguments
import argparse

parser = argparse.ArgumentParser(description='Diffusion map accelerated sampling.')
parser.add_argument('--algorithm', dest='algoFlag', type=str, default='0',
                    help='algorithm name or number to use (default: 0)')
parser.add_argument('--iterations', dest='niterations', type=int, default=2,
                    help='number of iterations to run (default: 2)')
parser.add_argument('--replicas', dest='nreplicas', type=int, default=2,
                    help='number of replicas to use per iteration (default: 2)')
parser.add_argument('--nrsteps', dest='nrSteps', type=int, default=1000,
                    help='number of replicas to use per iteration (default: 1000)')
#parser.add_argument('--temporaryFolder', dest='temporaryFile', type=int, default=0,
#                    help='0: save data into Data folder; 1: Save data to temporary folder; 2: another temporary folder ')
parser.add_argument('--folderName', dest='saveToFileName', type=str, default='Data',
                    help='Create folder and save data there')
parser.add_argument('--diffMapMetric', dest='diffMapMetric', type=str, default='euclidean',
                    help='Metric for diffusion maps. Choose between rmsd and euclidean.')
parser.add_argument('--diffusionMap', dest='diffusionMap', type=str, default='Diffmap',
                    help='Diffusion map: choose Diffmap for vanilla or TMDiffmap for target measure diffusion map.')

args = parser.parse_args()

saveToFileName = args.saveToFileName
diffusionMap = args.diffusionMap

#read imput parameters to choose the algorithm
algoFlag=args.algoFlag
diffMapMetric=args.diffMapMetric

if(algoFlag=='std' or algoFlag=='0'):
    iAlgo=0
    algoName = 'std'
elif(algoFlag=='eftad_fixed_cv' or algoFlag=='1'):
    iAlgo=1
    algoName = 'eftad_fixed_cv'
elif(algoFlag=='initial_condition' or algoFlag=='2'):
    iAlgo=2
    algoName ='initial_condition'
elif(algoFlag=='frontier_points_corner_change_temperature' or algoFlag=='3'):
    iAlgo=3
    algoName = 'frontier_points_corner_change_temperature'
elif(algoFlag=='frontier_points_corner_change_temperature_off' or algoFlag=='4'):
    iAlgo=4
    algoName = 'frontier_points_corner_change_temperature_off'
elif(algoFlag=='local_frontier_points_corner_change_temperature_off' or algoFlag=='5'):
    iAlgo=5
    algoName = 'local_frontier_points_corner_change_temperature_off'
elif(algoFlag=='local_frontier_points_corner_change_temperature' or algoFlag=='6'):
    iAlgo=6
    algoName = 'local_frontier_points_corner_change_temperature'
elif(algoFlag=='DiffmapABF' or algoFlag=='7'):
    iAlgo=7
    algoName = 'DiffmapABF'
elif(algoFlag=='local_frontier_points' or algoFlag=='8'):
    iAlgo=8
    algoName = 'local_frontier_points'
    print('Error: ABF not working yet. ')
else:
    print('Error: wrong algorithm flag. ')

# parameters
T=300.0
temperature =  T * unit.kelvin
kT = kB * temperature

gamma = 1.0 / unit.picosecond
dt = 1.0 * unit.femtosecond

TemperatureTAMDFactor=30.0
massScale=50.0

gammaScale=100.0
kappaScale=1000.0

if iAlgo >0:
    print("TemperatureTAMDFactor = " + repr(TemperatureTAMDFactor))
temperatureAlpha= (T*TemperatureTAMDFactor)* unit.kelvin

print('Gamma is '+repr(gamma))
print('Temperature is '+repr(temperature))
if iAlgo >0:
    print('Temperature TAMD '+repr(TemperatureTAMDFactor)+'xTemperature')
    print('Mass alpha is '+repr(massScale)+'x Mass')

#create folders to save the data

dataFileName=saveToFileName+'/'+str(modelName)+'/'+str(algoName)+'/Traj/'
dataFileNameFrontierPoints=saveToFileName+'/'+str(modelName)+'/'+str(algoName)+'/Traj/FrontierPoints/'
dataFileNameEigenVectors=saveToFileName+'/'+str(modelName)+'/'+str(algoName)+'/Traj/Eigenvectors/'
dataFileEnergy=saveToFileName+'/'+str(modelName)+'/'+str(algoName)+'/Traj/Energies/'

newpath = os.path.join(os.getcwd(),dataFileName)
if not os.path.exists(newpath):
        os.makedirs(newpath)

newpath = os.path.join(os.getcwd(),dataFileNameFrontierPoints)
if not os.path.exists(newpath):
        os.makedirs(newpath)

newpath = os.path.join(os.getcwd(),dataFileNameEigenVectors)
if not os.path.exists(newpath):
        os.makedirs(newpath)

newpath = os.path.join(os.getcwd(),dataFileEnergy)
if not os.path.exists(newpath):
        os.makedirs(newpath)

# simulation class sampler takes integrator class with chosen parameters as input
integrator=integrator.Integrator( model=mdl, gamma=gamma, temperature=temperature, temperatureAlpha=temperatureAlpha, dt=dt, massScale=massScale, gammaScale=gammaScale, kappaScale=kappaScale)

## load initial condition from file
# IC = md.load('alanine_start_state_IC.h5')
# # remove first dimension - the intial condition has shape (1,nrParticles, spaceDimension) when taken from trajectory
# InitialPosition = np.squeeze(IC.xyz)
# integrator.x0 = InitialPosition * mdl.x_unit

general_sampler=sampler.Sampler(model=mdl, integrator=integrator, algorithm=iAlgo, diffusionMapMetric=diffMapMetric, dataFileName=dataFileName, dataFrontierPointsName = dataFileNameFrontierPoints, dataEigenVectorsName =dataFileNameEigenVectors, dataEnergyName = dataFileEnergy, diffusionMap=diffusionMap)

# nrSteps is number of steps for each nrRep , and iterate the algo nrIterations times - total simulation time is nrSteps x nrIterations
nrSteps=args.nrSteps
nrEquilSteps = 0 #10000
nrIterations=args.niterations
nrRep=args.nreplicas

print('Simulation time: '+repr(nrSteps*nrIterations*dt.value_in_unit(unit.femtosecond))+' '+str(unit.femtosecond)+'\n ***** \n' )
#
if (nrEquilSteps>0):
    print('Equilibration: '+repr(nrEquilSteps*dt.value_in_unit(unit.picosecond))+' '+str(unit.picosecond)+'\n ***** \n' )
    general_sampler.run_std(nrEquilSteps, 1, 1)
    general_sampler.resetInitialConditions()
# run the simulation
print('\n ****\n Starting simulation\n')
general_sampler.run(nrSteps, nrIterations, nrRep)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @brief:  read and write *.properties files
# @date:   2023.05.10 14:40:50
import os, sys

g_this_file = os.path.realpath(sys.argv[0])
g_this_path = os.path.dirname(g_this_file)
sys.path.append(os.path.dirname(g_this_path))

from utils.utils_cmn import CmnUtils
from utils.utils_logger import LoggerUtils

# --------------------------------------------------------------------------------------------------------------------------
class BasicPropertiesItem:

    def __init__(self, k, v=None):
        assert v is not None or k.startswith('#'), 'Invalid key: ' + k
        self.key, self.val = k, v

    def matchKey(self, k):
        return None != self.val and self.key == k

    def matchValue(self, v):
        return self.val == v

    def getKey(self):
        return self.key

    def getValue(self):
        return self.val

    def setValue(self, v):
        self.val = v

    def println(self):
        if None == self.val:
            LoggerUtils.println(self.key)
        else:
            LoggerUtils.println(self.key + '=' + self.val)

class BasicProperties:
    def __init__(self):
        self.properties = []

    def hasKey(self, key):
        assert not CmnUtils.isEmpty(key), 'Invalid key: ' + key
        for p in self.properties:
            if p.matchKey(key): return True
        return False

    def get(self, key, default_value=None):
        assert not CmnUtils.isEmpty(key), 'Invalid key: ' + key
        for p in self.properties:
            if p.matchKey(key): return p.getValue()
        return default_value

    def put(self, key, value=None):
        assert not CmnUtils.isEmpty(key), 'Invalid key: ' + key
        if value is None: assert key.startswith('#')
        for p in self.properties:
            if p.matchKey(key):
                p.setValue(value)
                return
        self.properties.append(BasicPropertiesItem(key, value))

    def remove(self, key):
        assert not CmnUtils.isEmpty(key), 'Invalid key: ' + key
        for p in self.properties:
            if p.matchKey(key):
                self.properties.remove(p)
                return

    def load(self, f):
        with open(f, 'r') as f:
            lines = f.readlines()
        for line in lines:
            line = line.strip()
            if CmnUtils.isEmpty(line): continue
            if line.startswith('#'):
                self.put(line)
                continue
            pos = line.find('=')
            if pos <= 0: continue
            self.put(line[:pos].strip(), line[pos + 1:])
        return self

    @staticmethod
    def loadFromFile(f):
        items = {}
        with open(f, 'r') as f:
            lines = f.readlines()
        for line in lines:
            line = line.strip()
            if CmnUtils.isEmpty(line): continue
            if line.startswith('#'): continue
            pos = line.find('=')
            if pos <= 0: continue
            items[line[:pos].strip()] = line[pos + 1:]
        return items

    @staticmethod
    def loadFromFileWithKey(f, key, defVal=None):
        with open(f, 'r') as f:
            lines = f.readlines()
        for line in lines:
            line = line.strip()
            if CmnUtils.isEmpty(line): continue
            if line.startswith('#'): continue
            pos = line.find('=')
            if pos <= 0: continue
            if key != line[:pos].strip(): continue
            return line[pos + 1:]
        return defVal

    def save(self, fileName):
        with open(fileName, 'w') as f:
            for p in self.properties:
                v = p.getValue()
                if CmnUtils.isEmpty(v):
                    f.write(p.getKey() + '\n')
                else:
                    f.write(p.getKey() + '=' + v + '\n')

    @staticmethod
    def saveToFile(fileName, dictData):
        with open(fileName, 'w') as f:
            for k, v in dictData.items():
                f.write(k + '=' + v + '\n')

    def println(self):
        for p in self.properties: p.println()

def run():
    pass
    # items = BasicProperties.loadFromFile('/Users/xxx/workspace/test.properties')
    # k = items['x2j-column-src']
    # v = items['x2j-column-val']
    # print(k, v)
    # p = BasicProperties()
    # p.put('a', 'b')
    # p.put('a1', 'b1')
    # p.println()
    #
    # assert p.hasKey('a')
    # assert p.hasKey('a1')
    # assert p.get('a') == 'b'
    # assert p.get('a1') == 'b1'
    # p.put('a', 'c')
    # p.println()
    # assert p.get('a') == 'c'
    # p.remove('a')
    # p.println()
    # assert not p.hasKey('a')
    # assert p.get('a') == None

if __name__ == "__main__":
    run()


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sbn
import argparse
import os
import logging
from utils.common import read_yaml, create_directories

STAGE = "EDA ANALYSIS"

logging.basicConfig(
    filename = os.path.join("logs", 'running_logs.log'),
    level = logging.INFO,
    format = "[%(asctime)s: %(levelname)s: %(module)s]: %(message)s",
    filemode = "a"
)

def main(config_path, params_path):
    ## read config files
    config = read_yaml(config_path)
    params = read_yaml(params_path)

    artifacts = config["artifacts"]
    artifacts_dir = artifacts["ARTIFACTS_DIR"]
    get_local_dir = artifacts["GET_DATA_DIR"]
    get_local_file = artifacts["GET_DATA_FILE"]
    get_local_dir_path = os.path.join(artifacts_dir, get_local_dir)
    create_directories([get_local_dir_path])
    get_local_filepath = os.path.join(get_local_dir_path, get_local_file)
    eda_data_dir = artifacts["EDA_DIR"]
    miss_col_file = artifacts["MISS_COL"]
    miss_row_file = artifacts["MISS_ROW"]
    dist_stat = artifacts["DIS_STAT"]
    dist_image = artifacts["DIST_IMAGE"]
    outliers = artifacts["CHECK_OUTLIERS"]
    correlation = artifacts["CORR_IMAGE"]
    pollutant_period = artifacts["POLL_PERIOD"]
    count_aqib = artifacts["COUNT_AQIB"]
    min_conc = artifacts["MIN_CONC"]
    max_conc = artifacts["MAX_CONC"]
    mean_conc = artifacts["MEAN_CONC"]
    pollutant_impact = artifacts["POLLUTANT_IMPACT"]
    aqi_impact = artifacts["AQI_IMPACT"]
    eda_data_dir_path = os.path.join(artifacts_dir, eda_data_dir)
    create_directories([eda_data_dir_path])
    miss_col_file_path = os.path.join(eda_data_dir_path, miss_col_file)
    miss_row_file_path = os.path.join(eda_data_dir_path, miss_row_file)
    dist_stat_path = os.path.join(eda_data_dir_path, dist_stat)
    dist_image_path = os.path.join(eda_data_dir_path, dist_image)
    outliers_path = os.path.join(eda_data_dir_path, outliers)
    correlation_path = os.path.join(eda_data_dir_path, correlation)
    pollutant_period_path = os.path.join(eda_data_dir_path, pollutant_period)
    count_aqib_path = os.path.join(eda_data_dir_path, count_aqib)
    min_conc_path = os.path.join(eda_data_dir_path, min_conc)
    max_conc_path = os.path.join(eda_data_dir_path, max_conc)
    mean_conc_path = os.path.join(eda_data_dir_path, mean_conc)
    pollutant_impact_path = os.path.join(eda_data_dir_path, pollutant_impact)
    aqi_impact_path = os.path.join(eda_data_dir_path, aqi_impact)

    #Load file
    data = pd.read_csv(get_local_filepath)

    # data types
    logging.info(f"Data Types:\n {data.dtypes}")

    # shape of the data
    logging.info(f"Shape of the Data:\n {data.shape}")

    # missing values in the columns in %
    missing_cols_v = pd.DataFrame(data.isna().sum() / len(data))
    missing_cols_v.columns = ["missing_columns_values"]
    missing_cols_v.to_csv(miss_col_file_path)

    # missing values in the rows in %
    missing_rows_v = pd.DataFrame(data.isna().sum(axis = 1) / len(data.columns))
    missing_rows_v.columns = ["missing_rows_values"]
    missing_rows_v.to_csv(miss_row_file_path)

    # descriptive statistics
    data_des = data.describe()
    data_des.to_csv(dist_stat_path)

    # data distribution
    columns = list(set(data.columns) - {'State', 'City', 'Station',	'From Date', 'To Date', 'AQI_Bucket'})
    q, r = divmod(len(columns), 4)
    fig, ax = plt.subplots(q, 4, figsize = (20, 20))
    for i, col in enumerate(columns):
        q, r = divmod(i, 4)
        sbn.histplot(data[col], ax = ax[q, r], kde = True)
    plt.savefig(dist_image_path)

    # checking outliers
    columns = list(set(data.columns) - {'State', 'City', 'Station',	'From Date', 'To Date', 'AQI_Bucket'})
    q, r = divmod(len(columns), 4)
    fig, ax = plt.subplots(q, 4, figsize = (20, 20))
    for i, col in enumerate(columns):
        q, r = divmod(i, 4)
        sbn.boxplot(data = data, x = col, ax = ax[q, r])
    plt.savefig(outliers_path)

    # checking skewness of columns
    columns = list(set(data.columns) - {'State', 'City', 'Station',	'From Date', 'To Date', 'AQI_Bucket'})
    for col in columns:
        logging.info(f"Checking the Skewness of Columns: {data[col].skew()}")

    # Correlation Analysis
    plt.figure(figsize = (20, 20))
    sbn.heatmap(data.corr(), annot = True)
    plt.savefig(correlation_path)

    # Checking concentration of pollutants over the period
    # Changing Date datatype
    data['To Date'] = pd.to_datetime(data['To Date'])
    columns = list(set(data.columns) - {'State', 'City', 'Station',	'From Date', 'AQI', 'To Date', 'AQI_Bucket'})
    q, r = divmod(len(columns), 1)
    fig, ax = plt.subplots(q, 1, figsize = (15, 40))
    for i, col in enumerate(columns):
        q, r = divmod(i, 1)
        sbn.lineplot(data = data, x = 'To Date', y = col, ax = ax[q])
    plt.savefig(pollutant_period_path)

    # Count of unique values in AQIBucket
    plt.figure(figsize = (10, 10))
    sbn.countplot(data = data, x = 'AQI_Bucket')
    plt.savefig(count_aqib_path)

    # Minimum concentration of each pollutant over the city
    columns = list(set(data.columns) - {'State', 'Station',	'From Date', 'To Date', 'AQI', 'AQI_Bucket'})
    data_min = data.loc[:, columns].groupby('City').min()
    data_min.to_csv(min_conc_path)

    # Maximum concentration of each pollutant over the city
    columns = list(set(data.columns) - {'State', 'Station',	'From Date', 'To Date', 'AQI', 'AQI_Bucket'})
    data_max = data.loc[:, columns].groupby('City').max()
    data_max.to_csv(max_conc_path)

    # Mean concentration of each pollutant over the city
    columns = list(set(data.columns) - {'State', 'Station',	'From Date', 'To Date', 'AQI', 'AQI_Bucket'})
    data_mean = data.loc[:, columns].groupby('City').mean()
    data_mean.to_csv(mean_conc_path)

    # Distribution of pollutants based on city
    columns = list(set(data.columns) - {'State', 'Station', 'City',	'From Date', 'To Date', 'AQI', 'AQI_Bucket'})
    for i, col in enumerate(columns):
        plt.figure(figsize = (10, 10))
        sbn.boxplot(data = data, x = col, y = 'City')
        plt.xticks(rotation = 90)
        plt.savefig(os.path.join(eda_data_dir_path, "city_" + col + ".jpg"))

        #pollutants impact on cities BVHB

        columns = ['PM2.5', 'PM10', 'NO2', 'CO', 'SO2', 'Ozone', 'NH3']
        fig = plt.figure(figsize=(35, 35))
        for i, col in enumerate(columns):
            q, r = divmod(i, 4)
            ax = fig.add_subplot(int(str(q + 1) + "4" + str(r + 1)))
            data_mean = data_mean.fillna(0)
            ax.pie(data_mean[col], labels=data_mean.index, autopct='%.2f%%')
            plt.xlabel(col)
        plt.savefig(pollutant_impact_path)

        #AQI impact on cities over the years
        columns = ['Bengaluru', 'Lucknow', 'Patna', 'Delhi']
        q, r = divmod(len(columns), 1)
        fig, ax = plt.subplots(q, 1, figsize=(20, 40))
        for i, col in enumerate(columns):
            q, r = divmod(i, 1)
            data_col = data[data['City'] == col]
            g = sbn.lineplot(data=data_col, x='To Date', y='AQI', ax=ax[q])
            g.set_xlabel(col)
        plt.savefig(aqi_impact_path)

if __name__ == '__main__':
    args = argparse.ArgumentParser()
    args.add_argument("--config", "-c", default="configs/config.yaml")
    args.add_argument("--params", "-p", default="params.yaml")
    parsed_args = args.parse_args()

    try:
        logging.info("\n********************")
        logging.info(f">>>>> stage {STAGE} started <<<<<")
        main(config_path = parsed_args.config, params_path = parsed_args.params)
        logging.info(f">>>>> stage {STAGE} completed!<<<<<\n")
    except Exception as e:
        logging.exception(e)
        raise e


import logging

import paddle

import paddle.vision.transforms as transforms
import paddle.vision.datasets as datasets

from paddle.io import DataLoader,RandomSampler, SequenceSampler,DistributedBatchSampler,Dataset

from paddle.distributed import fleet

logger = logging.getLogger(__name__)

class MyDataset(Dataset):
    def __init__(self, trainset):
        self.num_samples = len(trainset)
        self.trainset=trainset
    def __getitem__(self,idx):

        if(isinstance(idx,int)):
            print(1)
            image,label=self.trainset[idx]
        else:
            print(2)
            image,label=self.trainset[idx[0]]

        return image,label
    def __len__(self):
        return self.num_samples

def get_loader(args):
    if args.local_rank not in [-1, 0]:
        paddle.distributed.barrier()

    transform_train = transforms.Compose([
        transforms.RandomResizedCrop((args.img_size, args.img_size), scale=(0.05, 1.0)),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ])
    transform_test = transforms.Compose([
        transforms.Resize((args.img_size, args.img_size)),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ])

    if args.dataset == "cifar10":
        trainset = datasets.Cifar10(mode='train',
                                    download=True,
                                    transform=transform_train)
        testset = datasets.Cifar10(mode='test',
                                   download=True,
                                   transform=transform_test) if args.local_rank in [-1, 0] else None

    elif args.dataset == "cifar100":
        trainset = datasets.Cifar100(mode='train',
                                     download=True,
                                     transform=transform_train)
        train_dataset = MyDataset(trainset)

        testset = datasets.Cifar100(mode='test',
                                    download=True,
                                    transform=transform_test) if args.local_rank in [-1, 0] else None
        test_dataset = MyDataset(testset)

    if args.local_rank == 0:
        fleet.barrier_worker()

    tra_sampler = RandomSampler(train_dataset)if args.local_rank == -1 else DistributedBatchSampler(train_dataset,
                                                                                batch_size=1,
                                                                                shuffle=True)

    train_sampler=paddle.io.BatchSampler(sampler=tra_sampler,
                                        batch_size=args.train_batch_size,
                                        drop_last=True)
    tes_sampler =  SequenceSampler(test_dataset)
    test_sampler=paddle.io.BatchSampler(sampler=tes_sampler,
                                        batch_size=args.eval_batch_size,
                                        drop_last=True)

    train_loader = DataLoader(train_dataset,
                              batch_sampler=train_sampler,
                              num_workers=0)
    test_loader = DataLoader(test_dataset,
                             batch_sampler=test_sampler,
                             num_workers=0) if testset is not None else None

    return train_loader, test_loader


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# Hanna Born
# ANLY 590 Fall 2021
# HW6.0

# references:
# code on course github at https://github.com/jh2343/590-CODES/
# https://blog.keras.io/building-autoencoders-in-keras.html

# In[2]:

# import packages
import sys

import numpy as np
import pandas as pd
import seaborn as sns
import tensorflow as tf

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

from sklearn.preprocessing import MinMaxScaler

import keras
from keras import models
from keras import layers
from keras import Model

from keras.datasets import mnist, fashion_mnist #, cifar10

# In[3]:

# REPEAT 6.1 but instead using a convolutional auto-encoder
# write output to log file
old_stdout = sys.stdout
log_file = open("HW6.1_log_file.log","w")
sys.stdout = log_file

# ----------------------------
# USER PARAMETERS
# for MNIST data
# ----------------------------
N_channels=1
PIX=28
EPOCHS          =   10          # 35
NKEEP           =   2500        # DOWNSIZE DATASET
# NKEEP           =   60000       # FULL SIZE DATASET
BATCH_SIZE      =   128

# ----------------------------
# GET MNIST DATASET
# ----------------------------
(X, Y), (test_images, test_labels) = mnist.load_data()
# (x_train, _), (x_test, _) = mnist.load_data()

#NORMALIZE AND RESHAPE
X = X.astype('float32') / 255.
test_images = test_images.astype('float32') / 255.

# ----------------------------
# GET MNIST-FASHION DATA
# ----------------------------
from keras.datasets import fashion_mnist

(X_fashion, Y_fashion), (test_images_fashion, test_labels_fashion) = fashion_mnist.load_data()

# NORMALIZE AND RESHAPE
X_fashion=X_fashion.astype('float32') / 255.
test_images_fashion=test_images_fashion.astype('float32') / 255.

# ----------------------------
# MODEL
# BUILD CNN-AE MODEL
# ----------------------------
input_img = keras.Input(shape=(PIX, PIX, N_channels))

# ENCODER --------------------
x = layers.Conv2D(32, (3, 3), activation='relu', padding='same')(input_img)
x = layers.MaxPooling2D((2, 2), padding='same')(x)
x = layers.Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = layers.MaxPooling2D((2, 2), padding='same')(x)
x = layers.Conv2D(8, (3, 3), activation='relu', padding='same')(x)
encoded = layers.MaxPooling2D((2, 2), padding='same')(x)
# AT THIS POINT THE REPRESENTATION IS (4, 4, 8) I.E. 128-DIMENSIONAL

# DECODER --------------------
x = layers.Conv2D(8, (3, 3), activation='relu', padding='same')(encoded)
x = layers.UpSampling2D((2, 2))(x)
x = layers.Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = layers.UpSampling2D((2, 2))(x)
x = layers.Conv2D(32, (3, 3), activation='relu')(x)
x = layers.UpSampling2D((2, 2))(x)
decoded = layers.Conv2D(N_channels, (3, 3), activation='sigmoid', padding='same')(x)

# ----------------------------
# COMPILE
# ----------------------------
model = keras.Model(input_img, decoded)
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']);
model.summary()

# ----------------------------
# TRAIN
# and track training history
# ----------------------------
history = model.fit(X, X,
                epochs=EPOCHS,
                batch_size=BATCH_SIZE,
                shuffle=True,
                validation_data=(test_images, test_images))

# # # save the trained model for future use
# model.save('HW6.1_trained_model_2500.h5')

# # load and evaluate the model on the test set
# model = tf.keras.models.load_model('HW6.1_trained_model_2500.h5')
# model.evaluate(test_images, test_images)

# ----------------------------
# HISTORY PLOT
# ----------------------------
epochs = range(1, len(history.history['loss']) + 1)
plt.figure()
plt.plot(epochs, history.history['loss'], 'bo', label='Training loss')
plt.plot(epochs, history.history['val_loss'], 'b', label='Validation loss')
plt.legend()

# -------------------------------
# MAKE PREDICTIONS FOR TEST DATA
# -------------------------------
decoded_imgs = model.predict(test_images)
decoded_imgs_fashion = model.predict(test_images_fashion)

# #COMPILE AND FIT
# model.compile(optimizer='rmsprop',
#               loss='mean_squared_error',
#               metrics=['accuracy'])
# model.summary()
# model.fit(X, X, epochs=10, batch_size=1000,validation_split=0.2)

# --------------------------------
# ORIGINAL AND RECONSTRUCTED
# --------------------------------
X1=model.predict(X)
X1_fashion=model.predict(X_fashion)

# ----------------------------
# DEFINE ERROR THRESHOLD
# ----------------------------
# define error threshold for anomaly detection
# if(err > threshold) --> anomaly
threshold = 4*model.evaluate(X,X, batch_size=X.shape[0])
# print('threshold', threshold)

#-------------------------------------
# EVALUATE ON TEST DATA (MNIST)
# and
# EVALUATE ON TEST DATA (MNIST-FASHION)
#-------------------------------------
train_loss, train_acc = model.evaluate(X, X, batch_size=X.shape[0])
test_loss, test_acc = model.evaluate(test_images, test_images, batch_size=test_images.shape[0])
test_loss_fashion, test_acc_fashion = model.evaluate(test_images_fashion,
                                                     test_images_fashion,
                                                     batch_size=test_images_fashion.shape[0])

print('\nthreshold: \t\t\t', threshold)

print('\ntrain_loss: \t\t\t', train_loss)
print('test_loss (MNIST): \t\t', test_loss)
print('test_loss (MNIST-FASHION): \t', test_loss_fashion)
print('\ntrain_acc: \t\t\t', train_acc)
print('test_acc (MNIST): \t\t', test_acc)
print('test_acc (MNIST-FASHION): \t', test_acc_fashion)

# -------------------------------------------
# COMPARE ORIGINAL AND RECONSTRUCTED IMAGES
# -------------------------------------------
fig, axs = plt.subplots(1,6)
axs[0].imshow(test_images[11].reshape(PIX, PIX))
axs[1].imshow(decoded_imgs[11].reshape(PIX, PIX))
axs[2].imshow(decoded_imgs_fashion[11].reshape(PIX, PIX))
axs[3].imshow(test_images[46].reshape(PIX, PIX))
axs[4].imshow(decoded_imgs[46].reshape(PIX, PIX))
axs[5].imshow(decoded_imgs_fashion[46].reshape(PIX, PIX))

# ---------------------------
# ANOMALY DETECTION
# ---------------------------
# reference: https://www.analyticsvidhya.com/blog/2021/05/anomaly-detection-using-autoencoders-a-walk-through-in-python/

def anomaly_predictions(model, test_images, threshold):
    predictions = model.predict(test_images)
    errors = tf.keras.losses.mean_squared_logarithmic_error(predictions,
                                  test_images)
    errors = tf.math.reduce_sum(errors, axis=1, keepdims=False, name=None)
    errors = tf.math.reduce_sum(errors, axis=1, keepdims=False, name=None)/28
    print(errors)
    anomaly_mask = pd.Series(errors > np.mean(threshold))
    preds = anomaly_mask.map(lambda x: 1.0 if x == True else 0.0) # 0 normal, 1 anomaly
    percent_anomalies = sum(preds)/len(preds) * 100
    # return preds # to get predictions
    return print(f'-- {sum(preds):.0f} anomalies / {len(preds)} total \n-- % anomalies:\t {percent_anomalies:.2f}%')

# threshold
print(f"threshold: \t {np.mean(threshold):2f}")

# get anomaly predictions for MNIST (model trained on MNIST)
print("\nanomaly predictions for (MNIST trained on MNIST):")
anomaly_predictions(model, decoded_imgs, threshold)

# get anomaly predictions for MNIST-FASHION (model trained on MNIST)
print("\nanomaly predictions for (MNIST-FASHION trained on MNIST):")
anomaly_predictions(model, decoded_imgs_fashion, threshold)

# finish wirting output to log file and close
sys.stdout = old_stdout
log_file.close()

# In[ ]:


import math
import random
import numpy as np
import pandas as pd
import yfinance as yf

from scipy.stats import norm
from scipy.stats import uniform
from sklearn.neighbors import KNeighborsRegressor
from lightgbm import LGBMRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.svm import SVR
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.model_selection import RandomizedSearchCV
from sklearn.preprocessing import StandardScaler

import warnings
from sklearn.exceptions import ConvergenceWarning
warnings.filterwarnings("ignore", category=ConvergenceWarning, module="sklearn")
warnings.filterwarnings("ignore", category=UserWarning, module="sklearn")

random.seed(0)

tick = yf.Ticker('NQ=F')
data = tick.history(period="max")
data.reset_index(inplace=True)
data.Date = data.Date.dt.date

# functions for option prices
N = norm.cdf
def BS_CALL(S, K, T, r, sigma):
    d1 = (np.log(S/K) + (r + sigma**2/2)*T) / (sigma*np.sqrt(T))
    d2 = d1 - sigma * np.sqrt(T)
    return S * N(d1) - K * np.exp(-r*T)* N(d2)

def BS_PUT(S, K, T, r, sigma):
    d1 = (np.log(S/K) + (r + sigma**2/2)*T) / (sigma*np.sqrt(T))
    d2 = d1 - sigma* np.sqrt(T)
    return K*np.exp(-r*T)*N(-d2) - S*N(-d1)

# add daily HPR and 250 day volatility
data['dailyR'] = data.Close/data.Close.shift(1)-1
data['dailyV'] = data.dailyR.rolling(250).std()
data.dropna(inplace=True)
data.reset_index(inplace=True, drop=True)

# define parameter ranges to simulate inputs
# risk_free_range = np.arange(.01,.06,0.005)
# expiration_range = np.arange(25,150,25)
# moneyness_range = np.arange(-.10,.10,0.004)

risk_free_range = [.03]
expiration_range = [20]
moneyness_range = [0]

# compute option prices
output = []
for i in data.index:
    risk_free = random.choice(risk_free_range)/250
    expiration = random.choice(expiration_range)
    moneyness = 1 + random.choice(moneyness_range)

    volatility = data.dailyV[i]
    close = data.Close[i]
    strike = close*moneyness

    call_price = BS_CALL(close, strike, expiration, risk_free, volatility)
    put_price = BS_PUT(close, close, expiration, risk_free, volatility)

    output.append([risk_free,expiration,moneyness,strike,call_price,put_price])

output_df = pd.DataFrame(output)
output_df.columns = ['risk_free','expiration','moneyness','strike','call_price','put_price']

# create final dataset for training and testing
full_data = pd.concat([data[['Date','Close','dailyV']],output_df], axis=1)

test_size = .05
train_end = int(len(full_data)*(1-test_size))

train_data = full_data.loc[:train_end].copy()
test_data = full_data.loc[train_end+1:].copy()

# define x y data sets
X_train = train_data[['Close','strike','moneyness','expiration','risk_free','dailyV']].to_numpy()
Y_train_call = train_data['call_price'].to_numpy()
Y_train_put = train_data['put_price'].to_numpy()

X_test = test_data[['Close','strike','moneyness','expiration','risk_free','dailyV']].to_numpy()
Y_test_call = test_data['call_price'].to_numpy()
Y_test_put = test_data['put_price'].to_numpy()

# scale data
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

## fit predict with ML models
# K-nearest neighbors
knn_search = RandomizedSearchCV(KNeighborsRegressor(), {'leaf_size':np.arange(5,100,5),'n_neighbors':np.arange(5,100,5)})
knn_search.fit(X_train,Y_train_call)
call_predict = knn_search.predict(X_test)
knn_call_df = pd.DataFrame(call_predict-Y_test_call, columns=['knn_call'])

knn_search.fit(X_train,Y_train_put)
put_predict = knn_search.predict(X_test)
knn_put_df = pd.DataFrame(put_predict-Y_test_put, columns=['knn_put'])

# Multi Layer Perceptron
mlp_search = RandomizedSearchCV(MLPRegressor(random_state=0,max_iter=500,solver='lbfgs',tol=1e-8),
                                {'alpha':[0.01,0.001,0.0001],'hidden_layer_sizes':[(5,5,5,),(5,),(5,5,)]})
mlp_search.fit(X_train,Y_train_call)
call_predict = mlp_search.predict(X_test)
mlp_call_df = pd.DataFrame(call_predict-Y_test_call, columns=['mlp_call'])

mlp_search.fit(X_train,Y_train_put)
put_predict = mlp_search.predict(X_test)
mlp_put_df = pd.DataFrame(put_predict-Y_test_put, columns=['mlp_put'])

# LightGBM
lgb_search = RandomizedSearchCV(LGBMRegressor(boosting_type='goss',subsample=0.8,colsample_bytree=0.8),
                                {'n_estimators':np.arange(50,1000,50),'max_depth':np.arange(3,10,1)})
lgb_search.fit(X_train,Y_train_call)
call_predict = lgb_search.predict(X_test)
lgb_call_df = pd.DataFrame(call_predict-Y_test_call, columns=['lgb_call'])

lgb_search.fit(X_train,Y_train_put)
put_predict = lgb_search.predict(X_test)
lgb_put_df = pd.DataFrame(put_predict-Y_test_put, columns=['lgb_put'])

# Support Vector Machines
svr_search = RandomizedSearchCV(SVR(),{'C':np.arange(.01,1,.01),'gamma':np.arange(.01,1,.01)})
svr_search.fit(X_train,Y_train_call)
call_predict = svr_search.predict(X_test)
svr_call_df = pd.DataFrame(call_predict-Y_test_call, columns=['svr_call'])

svr_search.fit(X_train,Y_train_put)
put_predict = svr_search.predict(X_test)
svr_put_df = pd.DataFrame(put_predict-Y_test_put, columns=['svr_put'])

# save ML model results in excel file
test_df = test_data[['Date','call_price','put_price']].copy()
test_df.reset_index(inplace=True, drop=True)
ml_results = pd.concat([test_df,
                        knn_call_df,mlp_call_df,lgb_call_df,svr_call_df,
                        knn_put_df,mlp_put_df,lgb_put_df,svr_put_df], axis=1)
ml_results.to_excel('ml_results.xlsx', index=False)

# function for option to expand
def option_to_expand(T,S,sig,r,N,expand,cost):

    dt=T/N
    dxu=math.exp(sig*math.sqrt(dt))
    dxd=1/dxu
    pu=((math.exp(r*dt))-dxd)/(dxu-dxd)
    pd=1-pu
    disc=math.exp(-r*dt)

    St = [0] * (N+1)
    V = [0] * (N+1)

    St[0]=S*dxd**N

    for j in range(1, N+1):
        St[j] = St[j-1] * dxu/dxd

    for j in range(1, N+1):
        V[j] = max(St[j]*expand-cost,St[j])

    for i in range(N, 0, -1):
        for j in range(0, i):
            V[j] = disc*(pu*V[j+1]+pd*V[j])

    return V[0]

r=0.05
T=3
N=3
expand=1.5
cost=.5
S = np.arange(10,1000,10)
sig = np.arange(.2,.5,.01)

expand_options = []
for i in range(10000):
    i_S = random.choice(S)
    i_sig = random.choice(sig)
    expansion = option_to_expand(T=T,S=i_S,sig=i_sig,r=r,N=N,expand=expand,cost=i_S*cost)
    expand_options.append([i_S,i_sig,expansion])

expand_df = pd.DataFrame(expand_options)
expand_df.columns = ['S','sig','expand']

# create final dataset for training and testing
test_size = .05
train_end = int(len(expand_df)*(1-test_size))

train_data = expand_df.loc[:train_end].copy()
test_data = expand_df.loc[train_end+1:].copy()

# define x y data sets
X_train = train_data[['S','sig']].to_numpy()
Y_train = train_data['expand'].to_numpy()
X_test = test_data[['S','sig']].to_numpy()
Y_test = test_data['expand'].to_numpy()

# scale data
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

## fit predict with ML models
# K-nearest neighbors
knn_search = RandomizedSearchCV(KNeighborsRegressor(), {'leaf_size':np.arange(5,100,5),'n_neighbors':np.arange(5,100,5)})
knn_search.fit(X_train,Y_train)
predict = knn_search.predict(X_test)
knn_expand_df = pd.DataFrame(predict-Y_test, columns=['knn_expand'])

# Multi Layer Perceptron
mlp_search = RandomizedSearchCV(MLPRegressor(random_state=0,max_iter=500,solver='lbfgs',tol=1e-8),
                                {'alpha':[0.01,0.001,0.0001],'hidden_layer_sizes':[(5,5,5,),(5,),(5,5,)]})
mlp_search.fit(X_train,Y_train)
predict = mlp_search.predict(X_test)
mlp_expand_df = pd.DataFrame(predict-Y_test, columns=['mlp_expand'])

# LightGBM
lgb_search = RandomizedSearchCV(LGBMRegressor(boosting_type='goss',subsample=0.8,colsample_bytree=0.8),
                                {'n_estimators':np.arange(50,1000,50),'max_depth':np.arange(3,10,1)})
lgb_search.fit(X_train,Y_train)
predict = lgb_search.predict(X_test)
lgb_expand_df = pd.DataFrame(predict-Y_test, columns=['lgb_expand'])

# Support Vector Machines
svr_search = RandomizedSearchCV(SVR(),{'C':np.arange(.01,1,.01),'gamma':np.arange(.01,1,.01)})
svr_search.fit(X_train,Y_train)
predict = svr_search.predict(X_test)
svr_expand_df = pd.DataFrame(predict-Y_test, columns=['svr_expand'])

# save ML model results in excel file
test_df = test_data.copy()
test_df.reset_index(inplace=True, drop=True)
test_df.insert(loc=0,value=cost,column='expand cost')
test_df.insert(loc=0,value=expand,column='expand factor')
test_df.insert(loc=0,value=N,column='N steps')
test_df.insert(loc=0,value=T,column='T years')
test_df.insert(loc=0,value=r,column='risk-free')

expand_results = pd.concat([test_df,knn_expand_df,mlp_expand_df,lgb_expand_df,svr_expand_df,], axis=1)
expand_results.to_excel('expansion_results.xlsx', index=False)

print('DONE')
# references for borrowed code:
#     - https://www.codearmo.com/python-tutorial/options-trading-black-scholes-model
#     - https://github.com/joseprupi/pyOptionPricing/blob/master/binomialTree.py


import download_images
import requests

def fetch_spacex_photo_latest_launch(
    spacex_url='https://api.spacexdata.com/v3/launches/latest'
):
    response = requests.get(spacex_url)
    response.raise_for_status()
    response_json = response.json()
    image_urls = [
        {
            'url':image_url,
            'name': '{name}_{index}'.format(name=response_json['mission_name'],
                                            index=index)
        }
        for index, image_url in enumerate(
            response_json['links']['flickr_images'], 1)
    ]
    return image_urls

def main():
    try:
        image_urls = fetch_spacex_photo_latest_launch()
        download_images.download_images_by_urls_and_names(image_urls)
    except requests.exceptions.HTTPError as error:
        exit("Can't get data from server:\n{0}".format(error))

if __name__ == '__main__':
    main()


# Nameof project= Merak
# made by Sahar Nikkhah
#This is to find the time lag of Merak light to reach the telescopes relative to each other/and this code use the formula in MerakTimeLag note.
from subprocess import call
from astropy.coordinates import EarthLocation
from astropy.time import Time
from astropy.coordinates import AltAz
from astropy.coordinates import SkyCoord
from datetime import datetime, timedelta
import math
#===============finding time lag formula is from slides==========
def Time_lag(T1X,T1Y,T1H,T2X,T2Y,T2H,az,alt):
    theta12=math.atan((T1X-T2X)/(T1Y-T2Y))
    beta12=math.acos(math.sqrt((math.pow((T1X-T2X),2))+(math.pow((T1Y-T2Y),2)))/math.sqrt((math.pow((T1X-T2X),2))+(math.pow((T1Y-T2Y),2))+(math.pow((T1H-T2H),2))))
    alpha12=math.acos(math.cos(az-theta12)*math.cos(alt))
    delta12=math.sqrt((math.pow((T1X-T2X),2))+(math.pow((T1Y-T2Y),2))+(math.pow((T1H-T2H),2)))
    t= delta12*math.cos(alpha12+beta12)/3e8
    return t

#==================some infos====================
observing_time = Time('2021-03-31 3:00:00')# observing starts
observing_time2= Time('2021-03-31 3:00:07')#observing ends
Merak= SkyCoord.from_name('Merak')#finding Merak in sky
Veritas= EarthLocation(lat='31.675', lon='-110.952')#loaction of orgin in slides

#============telescopes locations(slides)========
T1X=135.48
T1Y=-8.61
T1H= 12.23
T2X=44.1
T2Y=-47.7
T2H=4.4
T3X=29.4
T3Y=60.1
T3H=9.8
T4X=-35.9
T4Y=11.3
T4H= 7.0

 #======start of loop over time============
while observing_time < observing_time2:
    #taking star data for the time
    aa = AltAz(location=Veritas, obstime=observing_time)
    altdeg=Merak.transform_to(aa).alt.deg
    azdeg=Merak.transform_to(aa).az.deg
    alt= math.radians(altdeg)
    az=math.radians(azdeg)

    #time lag

    print('=================',observing_time,'===================')
    t12=Time_lag(T1X,T1Y,T1H,T2X,T2Y,T2H,az,alt)
    t13=Time_lag(T1X,T1Y,T1H,T3X,T3Y,T3H,az,alt)
    t14=Time_lag(T1X,T1Y,T1H,T4X,T4Y,T4H,az,alt)
    t23=Time_lag(T2X,T2Y,T2H,T3X,T3Y,T3H,az,alt)
    t24=Time_lag(T2X,T2Y,T2H,T4X,T4Y,T4H,az,alt)
    t34=Time_lag(T3X,T3Y,T3H,T4X,T4Y,T4H,az,alt)

    print('t12=',t12,'\n','t13=',t13, '\n','t14=',t14, '\n','t24=',t24, '\n','t23=',t23, '\n','t34=',t34 )

    observing_time += timedelta(0,1)


from distutils.log import error
from tkinter.tix import Tree
from tracemalloc import start
import sys

from numpy import nbytes
from hpc.job import Job, Workloads
from hpc.cluster import ClusterWithTopology
from config import Machine_State, config
import model.modelhpc as model

class HPCInstance:
    def __init__(self, workload_file = ''):  # do nothing and return. A workaround for passing parameters to the environment
        print("Initialize Simple HPC Env")
        print ("loading workloads from dataset:", workload_file)
        self.loads = Workloads(workload_file)
        self.cluster = ClusterWithTopology()
        self.job_queue = []
        self.need_schedule_by_model_jobs = []
        self.request_one_node_jobs = []
        self.running_jobs = []
        self.start_index = 0
        self.next_arriving_job_idx = self.start_index + 1
        self.current_timestamp = self.loads[self.start_index].submit_time
        self.job_number = self.loads.size()
        self.idle_node_ids_in_line = []

        # 0: Average bounded slowdown, 1: Average waiting time
        # 2: Average turnaround time, 3: Resource utilization 4: Average slowdown
        # 5: Average Bounded Slowdown + Resource Utilization (0 + 3)
        self.job_score_type = config.job_score_type
        self.scheduled_logs = {}

        print("number of nodes:"+str(self.cluster.nb_machines) + ", number of jobs:"+str(self.job_number))

    def reset(self, start_time_step):
        self.job_queue = []
        self.need_schedule_by_model_jobs = []
        self.request_one_node_jobs = []
        self.running_jobs = []
        self.start_index = start_time_step
        self.next_arriving_job_idx = start_time_step + 1
        self.current_timestamp = self.loads[self.start_index].submit_time
        self.scheduled_logs = {}
        self.cluster.reset()

    def assign_job_machines(self):
        # 先移除已完成
        should_keep_run_jobs = []
        has_remove = True
        for job in self.running_jobs:
            if job.scheduled_time + job.run_time > self.current_timestamp:
                should_keep_run_jobs.append(job)
            else:
                self.cluster.release(job.allocated_machine_ids)
                # print("release job:"+str(job.job_id))

        if len(self.running_jobs) == len(should_keep_run_jobs):
            has_remove = False
        self.running_jobs = should_keep_run_jobs

        # 添加上个时段到达的作业
        has_new_arrive = False
        next_stage_arrive_job_index = 0
        time_span = [self.current_timestamp - config.time_slot, self.current_timestamp]
        assert self.loads[self.start_index].submit_time >= self.current_timestamp - config.time_slot
        for i in range(self.start_index, self.loads.size()):
            job_scheduling = self.loads[i]
            if job_scheduling.submit_time >= time_span[0] and job_scheduling.submit_time < time_span[1]:
                has_new_arrive = True
                self.job_queue.append(job_scheduling)
            else:
                next_stage_arrive_job_index = i
                break
        # assert next_stage_arrive_job_index > self.start_index
        # 如果没有移除的作业且没有新到达的作业，就没有必要求解
        if next_stage_arrive_job_index == 0: # has scheduled at the last node
            return self.job_number

        if len(self.job_queue) == 0 or self.cluster.idle_node_num == 0 or (not has_remove and not has_new_arrive):
            return next_stage_arrive_job_index

        used_nodes_num = 0
        for job in should_keep_run_jobs:
            used_nodes_num += job.request_number_of_nodes

        self.select_jobs_can_be_allocated()

        if not self.need_schedule_by_model_jobs:
            if self.request_one_node_jobs:
                for job in self.request_one_node_jobs:
                    self.allocate_for_jobs_without_RL(job)
            return next_stage_arrive_job_index

        assign_result, nb_can_use_machines, solve_time, cancel_job_ids = model.build_and_solve(jobs = self.need_schedule_by_model_jobs, cluster = self.cluster, temp_can_not_use_node_ids=[],
        idel_node_ids_in_line=[])

        assert len(cancel_job_ids) == 0

        solve_time = int(solve_time)

        assert used_nodes_num + nb_can_use_machines == self.cluster.nb_machines

        nb_jobs = len(self.need_schedule_by_model_jobs)
        unschedule_jobs = []
        for jobindex in range(nb_jobs):
            job = self.need_schedule_by_model_jobs[jobindex]
            assign_node_ids_this_job = assign_result[jobindex]
            assert len(assign_node_ids_this_job) == job.request_number_of_nodes

            assert job.scheduled_time == -1
            job.scheduled_time = self.current_timestamp + solve_time
            job.allocated_machine_ids = assign_node_ids_this_job
            score = self.job_score(job)  # calculated reward
            self.scheduled_logs[job] = score # scheduled_logs uses job as the key
            self.running_jobs.append(job)
            self.cluster.allocate_in_Batch(assign_node_ids_this_job)

        nb_nodes_request = 0
        nb_nodes_request_for_each = []
        for job in self.need_schedule_by_model_jobs:
            nb_nodes_request += job.request_number_of_nodes
            nb_nodes_request_for_each.append(job.request_number_of_nodes)

        print("running job:", len(should_keep_run_jobs), ", can use nodes:", nb_can_use_machines, ", request nodes:", nb_nodes_request, ", request for each job:", nb_nodes_request_for_each)
        print("jobs need to assign:" + str(nb_jobs) + ", not assigned jobs:" + str(len(self.job_queue)))

        return next_stage_arrive_job_index

        # 尽可能选择越多的作业数量完成分配
    def select_jobs_can_be_allocated(self):
        self.need_schedule_by_model_jobs = []
        self.request_one_node_jobs = []
        self.job_queue.sort(key = lambda job: [-job.wait_stage_num, job.request_number_of_nodes])  # 升序排序
        has_add_node_num = 0
        for job in self.job_queue:
            if has_add_node_num + job.request_number_of_nodes <= self.cluster.idle_node_num:
                if job.request_number_of_nodes != 1:
                    if len(self.need_schedule_by_model_jobs) < config.max_queue_size:
                        self.need_schedule_by_model_jobs.append(job)
                    else:
                        break
                else:
                    self.request_one_node_jobs.append(job)
                has_add_node_num += job.request_number_of_nodes
            else:
                break

        # update job queue
        need_schedule_job_num = len(self.request_one_node_jobs) + len(self.need_schedule_by_model_jobs)
        if len(self.job_queue) > need_schedule_job_num:
            self.job_queue = self.job_queue[need_schedule_job_num:]
            # 对于没有被调度的作业，放到下一次调度，需要更新wait_stage_num
            for job in self.job_queue:
                job.wait_stage_num += 1

        elif len(self.job_queue) == need_schedule_job_num:
            self.job_queue = []
        else:
            raise error

    def allocate_for_jobs_without_RL(self, job):
        node_ids_temp = []
        success_flag = False
        for m in self.cluster.machines:
            if m.state == Machine_State.idle_state:
                node_ids_temp.append(m.id)
                # update mask , put those lines in update state function
                # assert self.mask_for_invalid_action[m.id] == 1
                # self.mask_for_invalid_action[m.id] = 0
            if len(node_ids_temp) == job.request_number_of_nodes:
                success_flag = True
                break
        assert success_flag
        self.cluster.allocate_in_Batch(node_ids_temp)
        job.allocated_machine_ids = node_ids_temp
        job.scheduled_time = self.current_timestamp
        self.running_jobs.append(job)

    def convert_job_list_to_dic(self, jobs):
        schedule_dic = {}
        for job in jobs:
            assert not schedule_dic.__contains__(job.id)
            assert len(job.allocated_machine_ids) == job.request_number_of_nodes
            schedule_dic[job.id] = job.allocated_machine_ids
        return schedule_dic

    # BF、FCFS等启发式算法调度
    def schedule_curr_sequence_reset(self, score_fn, backfill, start_time_step, max_time_steps):
        self.reset(start_time_step)
        self.job_queue.append(self.loads[self.start_index])
        max_job_number = start_time_step + max_time_steps
        while True:
            # 按照规则排序
            # self.job_queue.sort(key=lambda j: score_fn(j))
            job_for_scheduling = self.job_queue[0]
            # 如果资源不满足要求
            if not self.cluster.can_allocated(job_for_scheduling):
                if backfill:
                    self.moveforward_for_resources_backfill_greedy(job_for_scheduling, max_job_number)
                else:
                    self.skip_for_resources_greedy(job_for_scheduling, max_job_number)
            # 该作业之前没有被调度过
            assert job_for_scheduling.scheduled_time == -1
            job_for_scheduling.scheduled_time = self.current_timestamp
            job_for_scheduling.allocated_machine_ids = self.cluster.allocate(job_for_scheduling.id,
                                                                          job_for_scheduling.request_number_of_nodes)
            # print('Allocating job:' + str(job_for_scheduling.id))

            self.running_jobs.append(job_for_scheduling)
            score = self.job_score(job_for_scheduling)  # calculated reward
            self.scheduled_logs[job_for_scheduling] = score # scheduled_logs uses job as the key
            self.job_queue.remove(job_for_scheduling)

            not_empty = self.moveforward_for_job(max_job_number)
            if not not_empty:
                break

    # @profile
    def moveforward_for_resources_backfill_greedy(self, job, max_job_number):
        # note that this function is only called when current job can not be scheduled.
        assert not self.cluster.can_allocated(job)

        earliest_start_time = self.current_timestamp
        # sort all running jobs by estimated finish time
        self.running_jobs.sort(key=lambda running_job: (running_job.scheduled_time + running_job.run_time))
        free_nodes = self.cluster.idle_node_num
        for running_job in self.running_jobs:
            free_nodes += running_job.request_number_of_nodes
            earliest_start_time = (running_job.scheduled_time + running_job.run_time)
            if free_nodes >= job.request_number_of_nodes:
                break

        while not self.cluster.can_allocated(job):

            # try to backfill as many jobs as possible. Use FCFS
            self.job_queue.sort(key=lambda _j: self.fcfs_score(_j))
            job_queue_iter_copy = list(self.job_queue)
            for _j in job_queue_iter_copy:
                if (self.current_timestamp + _j.run_time) < earliest_start_time:
                    if self.cluster.can_allocated(_j):
                        # we should be OK to schedule the job now
                        assert _j.scheduled_time == -1  # this job should never be scheduled before.
                        _j.scheduled_time = self.current_timestamp
                        _j.allocated_machine_ids = self.cluster.allocate(_j.id, _j.request_number_of_nodes)
                        self.running_jobs.append(_j)
                        score = self.job_score(_j)  # calculated reward
                        self.scheduled_logs[_j] = score  # scheduled_logs uses job as key
                        self.job_queue.remove(_j)  # remove the job from job queue

            # move to the next timestamp
            assert self.running_jobs
            self.running_jobs.sort(key=lambda running_job: (running_job.scheduled_time + running_job.run_time))
            next_resource_release_time = (self.running_jobs[0].scheduled_time + self.running_jobs[0].run_time)
            next_resource_release_machines = self.running_jobs[0].allocated_machine_ids

            if self.next_arriving_job_idx < max_job_number \
                    and self.loads[self.next_arriving_job_idx].submit_time <= next_resource_release_time:
                self.current_timestamp = max(self.current_timestamp, self.loads[self.next_arriving_job_idx].submit_time)
                self.job_queue.append(self.loads[self.next_arriving_job_idx])
                self.next_arriving_job_idx += 1
            else:
                self.current_timestamp = max(self.current_timestamp, next_resource_release_time)
                self.cluster.release(next_resource_release_machines)
                self.running_jobs.pop(0)  # remove the first running job

    def skip_for_resources_greedy(self, job, max_job_number):
        # note that this function is only called when current job can not be scheduled.
        assert not self.cluster.can_allocated(job)

        while not self.cluster.can_allocated(job):
            # schedule nothing, just move forward to next timestamp. It should just add a new job or finish a running job
            assert self.running_jobs
            self.running_jobs.sort(key=lambda running_job: (running_job.scheduled_time + running_job.run_time))
            next_resource_release_time = (self.running_jobs[0].scheduled_time + self.running_jobs[0].run_time)
            next_resource_release_machines = self.running_jobs[0].allocated_machine_ids

            if self.next_arriving_job_idx < max_job_number and self.loads[
                self.next_arriving_job_idx].submit_time <= next_resource_release_time:
                self.current_timestamp = max(self.current_timestamp, self.loads[self.next_arriving_job_idx].submit_time)
                self.job_queue.append(self.loads[self.next_arriving_job_idx])
                self.next_arriving_job_idx += 1
            else:
                self.current_timestamp = max(self.current_timestamp, next_resource_release_time)
                self.cluster.release(next_resource_release_machines)
                self.running_jobs.pop(0)  # remove the first running job.

    # @profile
    def moveforward_for_job(self, max_job_number):
        if self.job_queue:
            return True

        # if we need to add job, but can not add any more, return False indicating the job_queue is for sure empty now.
        if self.next_arriving_job_idx >= max_job_number:
            # assert not self.job_queue
            return False

        # move forward to add jobs into job queue.
        while not self.job_queue:
            if not self.running_jobs:  # 如果没有运行作业
                next_resource_release_time = sys.maxsize  # always add jobs if no resource can be released.
                next_resource_release_machines = []
            else:
                self.running_jobs.sort(key=lambda running_job: (running_job.scheduled_time + running_job.run_time))
                next_resource_release_time = (self.running_jobs[0].scheduled_time + self.running_jobs[0].run_time)
                next_resource_release_machines = self.running_jobs[0].allocated_machine_ids

            if self.loads[self.next_arriving_job_idx].submit_time <= next_resource_release_time and self.next_arriving_job_idx < max_job_number:
                self.current_timestamp = max(self.current_timestamp, self.loads[self.next_arriving_job_idx].submit_time)
                self.job_queue.append(self.loads[self.next_arriving_job_idx])
                self.next_arriving_job_idx += 1
                return True  # job added
            else:
                self.current_timestamp = max(self.current_timestamp, next_resource_release_time)
                self.cluster.release(next_resource_release_machines)
                self.running_jobs.pop(0)  # remove the first running job.

    def fcfs_score(self, job):
        submit_time = job.submit_time
        return submit_time

    def job_score(self, job_for_scheduling):
        # 0: Average bounded slowdown, 1: Average waiting time
        # 2: Average turnaround time, 3: Resource utilization 4: Average slowdown 5: communication cost
        if self.job_score_type == 0:
            # bsld
            _tmp = max(1.0, (float(
                job_for_scheduling.scheduled_time - job_for_scheduling.submit_time + job_for_scheduling.run_time)
                             /
                             max(job_for_scheduling.run_time, 10)))
        elif self.job_score_type == 1:
            # wait time
            _tmp = float(job_for_scheduling.scheduled_time - job_for_scheduling.submit_time)
        elif self.job_score_type == 2:
            # turnaround time
            _tmp = float(
                job_for_scheduling.scheduled_time - job_for_scheduling.submit_time + job_for_scheduling.run_time)
        elif self.job_score_type == 3:
            # utilization
            _tmp = -float(job_for_scheduling.run_time * job_for_scheduling.request_number_of_processors)
        elif self.job_score_type == 4:
            # sld
            _tmp = float(
                job_for_scheduling.scheduled_time - job_for_scheduling.submit_time + job_for_scheduling.run_time) \
                   / job_for_scheduling.run_time
        elif self.job_score_type == 5:
            _tmp_1 = self.cluster.calculate_total_distance_nodes(node_ids=job_for_scheduling.allocated_machine_ids)
            _tmp_2 = float(job_for_scheduling.scheduled_time - job_for_scheduling.submit_time)
            return [_tmp_1, _tmp_2]
        else:
            raise NotImplementedError

            # Weight larger jobs.
        # _tmp = _tmp * (job_for_scheduling.run_time * job_for_scheduling.request_number_of_processors)
        return _tmp

    def deal_logs(self):
        sum_comm = 0
        sum_wait = 0
        length = len(self.scheduled_logs.values())
        for v in self.scheduled_logs.values():
            sum_comm += v[0]
            sum_wait += v[1]
        self.scheduled_logs = {}
        assert not self.job_queue
        return sum_comm/length, sum_wait/length


import open3d as o3d
import open3d.visualization.gui as gui
import numpy as np
import time
import sys
import json
import pandas as pd
sys.path.append("../")
from testingStageViz import jsonToDf
from positionAnalysis import jointsNames
from realTimeHandRotationCompute import jointsNames as handJointsNames

# 全身骨架的結構
fullBodyBoneStrcuture = [
    [jointsNames.Hip, jointsNames.LeftUpperLeg], [jointsNames.Hip, jointsNames.RightUpperLeg],
    [jointsNames.LeftUpperLeg, jointsNames.LeftLowerLeg], [jointsNames.RightUpperLeg, jointsNames.RightLowerLeg],
    [jointsNames.LeftLowerLeg, jointsNames.LeftFoot], [jointsNames.RightLowerLeg, jointsNames.RightFoot],
    [jointsNames.Hip, jointsNames.Spine], [jointsNames.Spine, jointsNames.Chest], [jointsNames.Chest, jointsNames.UpperChest],
    [jointsNames.UpperChest, jointsNames.LeftUpperArm], [jointsNames.UpperChest, jointsNames.RightUpperArm], [jointsNames.UpperChest, jointsNames.Head],
    [jointsNames.LeftUpperArm, jointsNames.LeftLowerArm], [jointsNames.RightUpperArm, jointsNames.RightLowerArm],
    [jointsNames.LeftLowerArm, jointsNames.LeftHand], [jointsNames.RightLowerArm, jointsNames.RightHand],
]

# 手部骨架結構
handBoneStructure = [
    [handJointsNames.wrist, handJointsNames.thunmbCMC], [handJointsNames.wrist, handJointsNames.indexMCP],
    [handJointsNames.wrist, handJointsNames.middleMCP], [handJointsNames.wrist, handJointsNames.ringMCP],
    [handJointsNames.wrist, handJointsNames.pinkyMCP],
    [handJointsNames.thunmbCMC, handJointsNames.thunmbMCP], [handJointsNames.thunmbMCP, handJointsNames.thunmbIP],
    [handJointsNames.thunmbIP, handJointsNames.thunmbTIP],
    [handJointsNames.indexMCP, handJointsNames.indexPIP], [handJointsNames.indexPIP, handJointsNames.indexDIP],
    [handJointsNames.indexDIP, handJointsNames.indexTIP],
    [handJointsNames.middleMCP, handJointsNames.middlePIP], [handJointsNames.middlePIP, handJointsNames.middleDIP],
    [handJointsNames.middleDIP, handJointsNames.middleTIP],
    [handJointsNames.ringMCP, handJointsNames.ringPIP], [handJointsNames.ringPIP, handJointsNames.ringDIP],
    [handJointsNames.ringDIP, handJointsNames.ringTIP],
    [handJointsNames.pinkyMCP, handJointsNames.pinkyPIP], [handJointsNames.pinkyPIP, handJointsNames.pinkyDIP],
    [handJointsNames.pinkyDIP, handJointsNames.pinkyTIP],
]

# 目前顯示的frame index
curTimeInd = 0

def readAppliedRotPos(rotAppliedFilePath='../positionData/leftFrontKick_quat_directMapping.json', TPoseFilePath = ''):
    # 可以參考testingStageViz.py
    with open(rotAppliedFilePath, 'r') as WFile:
        lowerBodyPosition=json.load(WFile)
    timeCount = len(lowerBodyPosition)
    jointsIndices = list(lowerBodyPosition[0]['data'].keys())
    lowerBodyPosition = [{'data': {int(i): lowerBodyPosition[t]['data'][i] for i in jointsIndices}} for t in range(timeCount)]
    lowerBodyDf = jsonToDf(lowerBodyPosition)
    concatDfs = pd.concat(lowerBodyDf.values(), axis=1)
    concatArr = concatDfs.values
    concatArr = concatArr.reshape((concatArr.shape[0], -1, 3))
    print('lower body joint indices:', jointsIndices)
    print('timeCount: ', timeCount)
    print('lower body array shape', concatArr.shape)
    # Hip校正為原點
    concatArr = concatArr - concatArr[:, -1:, :]

    # read T pose
    # Ref: realTimeRotToAvatarPos.py
    saveDirPath = '../positionData/fromDB/genericAvatar/'
    TPoseJson = None
    with open(saveDirPath+'TPose.json', 'r') as fileIn:
        TPoseJson = json.load(fileIn)['results']
    jointCount = len(TPoseJson[0]['data'])
    TPoseDfs = jsonToDf(TPoseJson)
    TPoseConcatDfs = pd.concat(TPoseDfs.values(), axis=1)
    TPoseArr = TPoseConcatDfs.values
    TPoseArr = TPoseArr.reshape((TPoseArr.shape[0], -1, 3))
    TPoseArr = TPoseArr[3:4, :, :]  # Just dont use the first frame, since it might be wrong
    TPoseArr = np.repeat(TPoseArr, concatArr.shape[0], axis=0)
    print('T Pose joint count: ', jointCount)
    print('T Pose time count: ', TPoseArr.shape[0])
    print('T Pose array shape: ', TPoseArr.shape)
    # Hip校正回原點
    TPoseArr = TPoseArr - TPoseArr[:, 6:7, :]

    # combine T pose to lower body motions
    retArr = np.concatenate((concatArr, TPoseArr[:, 7:, :]), axis=1)
    print('return array shape: ', retArr.shape)

    return retArr

def readSynthesisPos(synthesisFilePath = '../positionData/afterSynthesis/NoVelAccOverlap/leftFrontKick_quat_direct_075_EWMA.json'):
    # TODO
    # read synthesis result
    # TODO: 好像會少一個點, 應該就是少hip, 應為預設他就是0, 需要把hip是0補回來
    synthesisPos = None
    with open(synthesisFilePath, 'r') as WFile:
        synthesisPos=json.load(WFile)
    timeCount = len(synthesisPos)
    jointCount = len(synthesisPos[0]['data'])
    synthesisDfs = jsonToDf(synthesisPos)
    synthesisConcatDfs = pd.concat(synthesisDfs.values(), axis=1)
    synthesisArr = synthesisConcatDfs.values
    synthesisArr = synthesisArr.reshape((synthesisArr.shape[0], -1, 3))
    print('time count: ', timeCount)
    print('joint count: ', jointCount)

    # 補Hip是0回array當中
    _hip = np.zeros((synthesisArr.shape[0], 3))
    synthesisArr = np.insert(synthesisArr, 6, _hip, axis=1)
    print('output array shape: ', synthesisArr.shape)
    return synthesisArr

def readHandPos(
    handLandmarkFilePath = '../complexModel/newRecord/twoLegJump_rgb.json',
    scale = [1, 1, 1], negate = [False, False, False]
):
    handPos = None
    with open(handLandmarkFilePath, 'r') as RFile:
        handPos = json.load(RFile)
    timeCount = len(handPos)
    jointCount = len(handPos[0]['data'])
    handDfs = jsonToDf(handPos)
    handConcatDfs = pd.concat(handDfs.values(), axis=1)
    handArr = handConcatDfs.values
    handArr = handArr.reshape((handArr.shape[0], -1, 3))
    print('time count: ', timeCount)
    print('joint count: ', jointCount)
    print('output array shape: ', handArr.shape)

    # 轉換成wrist在原點
    handArr = handArr - handArr[:, handJointsNames.wrist:handJointsNames.wrist+1, :]
    # 放大數值範圍
    for _axis, _scale in enumerate(scale):
        handArr[:, :, _axis] = handArr[:, :, _axis] * _scale
    # 反轉坐標軸
    for _axis, _neg in enumerate(negate):
        if _neg:
            handArr[:, :, _axis] = handArr[:, :, _axis] * -1
    return handArr

def readExampleAnimPos(filePath = ''):
    print('read example animation')
    # read example animation positions
    synthesisPos = None
    with open(filePath, 'r') as WFile:
        synthesisPos=json.load(WFile)['results']
    timeCount = len(synthesisPos)
    jointCount = len(synthesisPos[0]['data'])
    synthesisDfs = jsonToDf(synthesisPos)
    synthesisConcatDfs = pd.concat(synthesisDfs.values(), axis=1)
    synthesisArr = synthesisConcatDfs.values
    synthesisArr = synthesisArr.reshape((synthesisArr.shape[0], -1, 3))

    # Hip校正回原點
    synthesisArr = synthesisArr - synthesisArr[:, 6:7, :]
    print('time count: ', timeCount)
    print('joint count: ', jointCount)
    print('output array shape: ', synthesisArr.shape)
    return synthesisArr

def vizMotions(jointData, jointHeirarchy, hipPos, axisJointInd, frameRate=0.05, return_imgs=False, cameraParam=None):
    '''
    Objective
        輸入joint data, 使用open 3d顯示骨架運動資訊
    Input
        - jointData: list of 3 dimention array. 多組關節點的3d數值. time * number of joints * 3
        - jointHeirarchy: list of lists. 關節點連結資訊. each element is a list with two numbers in it.
            First number indicates parent joint's index, the other indicates child joint's index.
        - hipPos: list of 1 dimention array. 每一組關節點的hip位置. XYZ values in the array.
        - axisJointInd: list of list. 哪一些點需要在它上面繪製座標軸
        - frameRate: 更新率
    '''
    # 輸入joint data, 使用open 3d顯示骨架運動資訊

    # create visualizer and window.
    # vis = o3d.visualization.Visualizer()
    vis = o3d.visualization.VisualizerWithKeyCallback()
    # vis.create_window(height=480, width=640)
    vis.create_window(height=768, width=1024)
    opt = vis.get_render_option()
    opt.point_size = 10
    opt.line_width = 5

    # 註冊callback function, 讓按鍵觸發可以被偵測
    ## 印出當前顯示的frame index -> 按下空格鍵
    vis.register_key_callback(ord(' '), printCurFrameIdxCallback)

    # create axis at origin
    # origin_axis = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.2)
    # vis.add_geometry(origin_axis)

    # 根據hipPos改變每一組關節點的3d positions資訊
    for i, _hipPos in enumerate(hipPos):
        jointData[i] = jointData[i] + _hipPos[np.newaxis, np.newaxis, :]

    # 在部分節點繪製坐標軸標示 (e.g. 下半身的upper leg and lower leg)
    axisFrameList = []
    axisFrameOriginPosList = []
    for i, _indList in enumerate(axisJointInd):
        _axis_frame_list = []
        _axis_frame_pos_list = []
        for _ind in _indList:
            _axis_frame = o3d.geometry.TriangleMesh.create_coordinate_frame(
                size=0.1, origin=jointData[i][0, _ind, :]
            )
            vis.add_geometry(_axis_frame)
            _axis_frame_list.append(_axis_frame)
            _axis_frame_pos_list.append(np.array(_axis_frame.vertices))
        axisFrameList.append(_axis_frame_list)
        axisFrameOriginPosList.append(_axis_frame_pos_list)

    # initial point clouds list
    # initial line set list
    pcdList = []
    lineSetList = []
    for _jointData, _jointHeirarchy in zip(jointData, jointHeirarchy):
        _pcd = o3d.geometry.PointCloud()
        _pcd.points = o3d.utility.Vector3dVector(_jointData[0, :, :])
        pcdList.append(_pcd)

        colors = [[1, 0, 0] for i in range(len(_jointHeirarchy))]
        _line_set = o3d.geometry.LineSet()
        _line_set.points = o3d.utility.Vector3dVector(_jointData[0, :, :])
        _line_set.lines = o3d.utility.Vector2iVector(_jointHeirarchy)
        _line_set.colors = o3d.utility.Vector3dVector(colors)
        lineSetList.append(_line_set)
        # print(_jointData[0, :, :])
        # print(np.asarray(_line_set.points).shape)
        # exit()

        vis.add_geometry(_pcd)
        vis.add_geometry(_line_set)

    # 將右腿的顏色改成與其他點不同, 並且把線段/骨頭顏色改為黑色
    # 右腿的三個點的index = {0, 1, 2}, 三個骨頭index = {0, 2, 4}
    # 假設前面兩個輸入的point cloud是full body point cloud
    # bodyPointCloudInd = [0, 1]
    bodyPointCloudInd = [0]
    bodyBoneInd = [0, 2, 4]
    bodyJointInd = [0, 1, 2]
    for _pcdInd in bodyPointCloudInd:
        _colors = np.array([[0, 0, 0] for i in range(17)])
        _boneColors = np.array([[0, 0, 0] for i in range(16)])
        for i in bodyBoneInd:
            _boneColors[i, :] = [1, 0, 0]
        lineSetList[_pcdInd].colors = o3d.utility.Vector3dVector(_boneColors)
        for i in bodyJointInd:
            _colors[i, :] = [1, 0, 0]
        pcdList[_pcdInd].colors = o3d.utility.Vector3dVector(_colors)

    # 手的食指的點與線段也要修改顏色
    # 食指四個點index = {5, 6, 7, 8}, 三個骨頭的index = {8, 9, 10}
    # handPointCloudInd = [2]
    # handPointCloudInd = [0]
    # handBoneInd = [8, 9, 10]
    # handJointInd = [5, 6, 7, 8]
    # _colors = np.array([[0, 0, 0] for i in range(21)])
    # _boneColors = np.array([[0, 0, 0] for i in range(20)])
    # for i in handBoneInd:
    #     _boneColors[i, :] = [1, 0, 0]
    # for i in handJointInd:
    #     _colors[i, :] = [1, 0, 0]
    # for i in handPointCloudInd:
    #     pcdList[i].colors = o3d.utility.Vector3dVector(_colors)
    #     lineSetList[i].colors = o3d.utility.Vector3dVector(_boneColors)

    # Trajectory改變顏色, 改成藍色
    trajectoryInd = [1]
    # trajectoryInd = [0]
    for _pcdInd in trajectoryInd:
        _colors = np.array([[0, 0, 1] for i in range(jointData[_pcdInd].shape[1])])
        pcdList[_pcdInd].colors = o3d.utility.Vector3dVector(_colors)

    # 修改camera參數, 包含內參與外參
    if cameraParam is not None:
        res=vis.get_view_control().convert_from_pinhole_camera_parameters(cameraParam, allow_arbitrary=True)
        print('Is camera parameter adjust? ', res)

    # 紀錄visualize的影像畫面
    imgs = []

    # to add new points each dt secs.
    dt = frameRate
    previous_t = time.time()
    global curTimeInd
    curTimeInd = 0

    # run non-blocking visualization.
    # To exit, press 'q' or click the 'x' of the window.
    keep_running = True
    while keep_running:

        if time.time() - previous_t > dt:

            # update joints
            # update bones
            # TODO update coordinate frame
            for i, _jointData in enumerate(jointData):
                pcdList[i].points = o3d.utility.Vector3dVector(_jointData[curTimeInd, :, :])
                vis.update_geometry(pcdList[i])

                lineSetList[i].points = o3d.utility.Vector3dVector(_jointData[curTimeInd, :, :])
                vis.update_geometry(lineSetList[i])

                for _j, _axisJointInd in enumerate(axisJointInd[i]):
                    axisFrameList[i][_j].vertices = \
                        o3d.utility.Vector3dVector(
                            axisFrameOriginPosList[i][_j] - _jointData[0, _axisJointInd:_axisJointInd+1, :] + _jointData[curTimeInd, _axisJointInd:_axisJointInd+1, :]
                        )
                    vis.update_geometry(axisFrameList[i][_j])

            previous_t = time.time()
            curTimeInd += 1
            if curTimeInd >= jointData[0].shape[0]:
                break
            if return_imgs:
                imgs.append(vis.capture_screen_float_buffer())

        keep_running = vis.poll_events()
        vis.update_renderer()

    vis.destroy_window()

    # return 擷取的圖片
    if return_imgs:
        imgs = [np.asarray(i) for i in imgs]
        return imgs
    pass

def printCurFrameIdxCallback(vis):
    global curTimeInd
    print('cur frame index: ', curTimeInd)
    # print(o3d.camera.PinholeCameraParameters.extrinsic)
    print('Intrinsic matrix: ', vis.get_view_control().convert_to_pinhole_camera_parameters().intrinsic.intrinsic_matrix)
    print('Extrinsic matrix: ', vis.get_view_control().convert_to_pinhole_camera_parameters().extrinsic)
    return True

def main():
    # create visualizer and window.
    vis = o3d.visualization.Visualizer()
    vis.create_window(height=480, width=640)

    # create axis at origin
    origin_axis = o3d.geometry.TriangleMesh.create_coordinate_frame()
    vis.add_geometry(origin_axis)

    # create plane by box mesh
    # Ref: https://github.com/isl-org/Open3D/issues/3618
    plane = o3d.geometry.TriangleMesh.create_box(width=1.0, height=1.0, depth=0.01)
    vis.add_geometry(plane)

    # initialize pointcloud instance.
    pcd = o3d.geometry.PointCloud()
    # *optionally* add initial points
    points = np.random.rand(10, 3)
    pcd.points = o3d.utility.Vector3dVector(points)

    # initialize line set
    points = np.array([[0, 0, 0], [1, 0, 0], [0, 1, 0], [1, 1, 0], [0, 0, 1], [1, 0, 1],
              [0, 1, 1], [1, 1, 1]]).astype(float)
    lines = [[0, 1], [0, 2], [1, 3], [2, 3], [4, 5], [4, 6], [5, 7], [6, 7],
             [0, 4], [1, 5], [2, 6], [3, 7]]
    colors = [[1, 0, 0] for i in range(len(lines))]
    line_set = o3d.geometry.LineSet()
    line_set.points = o3d.utility.Vector3dVector(points)
    line_set.lines = o3d.utility.Vector2iVector(lines)
    line_set.colors = o3d.utility.Vector3dVector(colors)
    vis.add_geometry(line_set)

    # include it in the visualizer before non-blocking visualization.
    vis.add_geometry(pcd)

    # to add new points each dt secs.
    dt = 0.05
    # number of points that will be added
    n_new = 10

    previous_t = time.time()

    # run non-blocking visualization.
    # To exit, press 'q' or click the 'x' of the window.
    keep_running = True
    while keep_running:

        if time.time() - previous_t > dt:
            s = time.time()

            # Options (uncomment each to try them out):
            # 1) extend with ndarrays.
            pcd.points.extend(np.random.rand(n_new, 3))

            # 取代舊的points
            # print(np.asarray(pcd.points).shape[0])
            if np.asarray(pcd.points).shape[0] > 10:
                pcd.points = o3d.utility.Vector3dVector(np.random.rand(n_new, 3))

            # 線段當中某個點的x, y, z軸不斷增加
            points[0,0]+=0.001
            points[0,1]+=0.001
            points[0,2]+=0.001
            line_set.points = o3d.utility.Vector3dVector(points)
            # print(np.asarray(line_set.points).shape)

            # 2) extend with Vector3dVector instances.
            # pcd.points.extend(
            #     o3d.utility.Vector3dVector(np.random.rand(n_new, 3)))

            # 3) other iterables, e.g
            # pcd.points.extend(np.random.rand(n_new, 3).tolist())

            vis.update_geometry(pcd)
            vis.update_geometry(line_set)
            previous_t = time.time()

        keep_running = vis.poll_events()
        vis.update_renderer()

    vis.destroy_window()

    pass

# 目標是展示全身動作, 並且也展示腳的trajectory
# 正在考慮要不要實作
if __name__=='__main01__':
    # example animation joints position隨時間變動資料, 透過重複相同資料延長播放時間
    exampleAnimMotion = readExampleAnimPos('../positionData/fromDB/genericAvatar/leftFrontKickPositionFullJointsWithHead_withoutHip_075_quat_direct_normalized.json')
    exampleAnimMotion = np.tile(exampleAnimMotion, (3, 1, 1))
    # 只有腳的部分隨時間變動資料, 重複複製原始frame數量的次數
    frameCount = exampleAnimMotion.shape[0]
    leftFootTrajectory = exampleAnimMotion[:, 2, :]
    leftFootTrajectory = np.tile(leftFootTrajectory, (frameCount, 1, 1))

    # 修改成顯示單一frame
    # spFrameInd = 80
    # # spFrameInd = 105
    # repeatShowingTime = 10000
    # exampleAnimMotion = np.repeat(
    #     exampleAnimMotion[spFrameInd:spFrameInd+1, :, :],
    #     repeatShowingTime,
    #     axis=0
    # )
    # leftFootTrajectory = np.repeat(
    #     leftFootTrajectory[spFrameInd:spFrameInd+1, :, :],
    #     repeatShowingTime,
    #     axis=0
    # )

    # 讀取camera參數 (目的: 預先設定錄製mp4的拍攝角度)
    cameraParamFilePath = 'ScreenCamera_2023-05-02-13-22-03.json'
    cameraParam = o3d.io.read_pinhole_camera_parameters(cameraParamFilePath)

    capture_imgs = vizMotions(
        [exampleAnimMotion, leftFootTrajectory],
        [fullBodyBoneStrcuture, [[5, 10]]],
        [np.array([0, 0, 0.5]), np.array([0, 0, 0.5])],
        [[], [], []],
        0.05,
        True,
        cameraParam
    )

    # 將animation記錄成mp4
    # import imageio
    # outputVideoFilePath = 'trajectoryAndPose.mp4'
    # imageio.mimwrite(outputVideoFilePath, capture_imgs, fps=30, quality=8)
    # pass

# 最初始的展示. 展示手的作與全身的動作
if __name__=='__main__':
    # main()
    appliedRotMotion = readAppliedRotPos('../positionData/fromAfterMappingHand/leftSideKickStreamLinearMapping_FTTFFF.json')
    # appliedRotMotion = readAppliedRotPos('../positionData/leftSideKick_quat_directMapping.json')
    # 因為synthesis motion會少前面10個frame, 所以applied rotation版本需要捨去前面10個frame
    appliedRotMotion = appliedRotMotion[10:, :, :]

    # synthesisMotion = readSynthesisPos('../positionData/afterSynthesis/leftFrontKickStreamLinearMapping_TFFTTT_075_EWMA.json')
    exampleAnimMotion = readExampleAnimPos('../positionData/fromDB/genericAvatar/leftSideKickPositionFullJointsWithHead_withoutHip.json')
    synthesisMotion = exampleAnimMotion

    fingerMotion = readHandPos('../complexModel/leftSideKick.json', scale=[3.5, 1.5, 7], negate=[True, True, True])
    fingerMotion = fingerMotion[2600:3500, :, :]    # 側踢只有選部分區間的資料
    fingerMotion = fingerMotion[10:, :, :]
    # capture_imgs = vizMotions(
    #     [appliedRotMotion, synthesisMotion, fingerMotion],
    #     [fullBodyBoneStrcuture, fullBodyBoneStrcuture, handBoneStructure],
    #     [np.array([0, 0, 0.5]), np.array([-1, 0, 0.5]), np.array([1, 0, 0.5])],
    #     # [[], [jointsNames.LeftUpperLeg, jointsNames.LeftLowerLeg], [handJointsNames.indexMCP, handJointsNames.indexPIP]],
    #     [[], [], []],
    #     0.03333,
    #     True
    # )

    # 只展示單一個骨架的動作
    ## 讀取特定的camera設定檔案
    cameraParamFilePath = 'ScreenCamera_2023-03-30-14-17-05.json'   # For body motion
    # cameraParamFilePath = 'ScreenCamera_2023-03-30-15-44-09.json'   # For finger motion
    # cameraParamFilePath = 'ScreenCamera_2023-04-26-14-46-59.json'   # For finger motion (side kick)
    cameraParam = o3d.io.read_pinhole_camera_parameters(cameraParamFilePath)
    capture_imgs = vizMotions(
        [synthesisMotion],
        [fullBodyBoneStrcuture],
        # [fingerMotion],
        # [handBoneStructure],
        [np.array([0, 0, 0.5])],
        [[], [], []],
        0.03333,
        True,
        cameraParam
    )

    print(len(capture_imgs))
    print(capture_imgs[0].shape)
    # 輸出骨架運動過程的影片
    # import imageio
    # outputVideoFilePath = 'sideKickExampleAnimWithoutHip.mp4'
    # imageio.mimwrite(outputVideoFilePath, capture_imgs, fps=30, quality=8)

    # 顯示單一個frame的資訊, 方便拍攝論文的展示圖片
    # specificFrameInd = 57
    # # specificFrameInd = 67
    # repeatShowingTime = 10000
    # appliedRotMotion = np.repeat(
    #     appliedRotMotion[specificFrameInd:specificFrameInd+1, :, :], repeatShowingTime, axis=0
    # )
    # synthesisMotion = np.repeat(
    #     synthesisMotion[specificFrameInd:specificFrameInd+1, :, :], repeatShowingTime, axis=0
    # )
    # fingerMotion = np.repeat(
    #     fingerMotion[specificFrameInd:specificFrameInd+1, :, :], repeatShowingTime, axis=0
    # )
    # vizMotions(
    #     [appliedRotMotion, synthesisMotion, fingerMotion],
    #     [fullBodyBoneStrcuture, fullBodyBoneStrcuture, handBoneStructure],
    #     [np.array([-5, 0, 0.5]), np.array([-8, 0, 0.5]), np.array([5, 0, 0.5])],
    #     # [[], [jointsNames.LeftUpperLeg, jointsNames.LeftLowerLeg], [handJointsNames.indexMCP, handJointsNames.indexPIP]],
    #     [[], [], []],
    #     0.05
    # )

    # 錄製只有全身與手的立正狀態對比, 使用front kick放鬆姿態作為例子.
    # 需要繪製座標軸在特定的joint上方, 兩者需要距離近一些
    # vizMotions(
    #     [synthesisMotion, fingerMotion],
    #     [fullBodyBoneStrcuture, handBoneStructure],
    #     [np.array([-0.5, 0, 0.5]), np.array([0.5, 0, 0.5])],
    #     [[jointsNames.LeftUpperLeg, jointsNames.LeftLowerLeg], [handJointsNames.indexMCP, handJointsNames.indexPIP]],
    #     0.05
    # )


from src.model import KYC
import cv2

model_type = ""
device = "gpu"
kyc = KYC(model_type,device)
#frame = cv2.imread("test1.jpg")
frame = cv2.imread("test2.png")

# Module Retinaface (Face detection)
faces = kyc.retinaface_model(frame) #without batch
print(faces)

# Module Antiface (Face anti-spoofing)
labels, values = kyc.face_anti_spoofing(faces,frame) #without batch
print(labels, values)

# Module Insightface (Face embedding)
vectors = kyc.insightface_model(faces,frame) #512-D without batch
print(vectors)

# Module EQface (Face quality)
qualities = kyc.eqface_model(faces,frame) #without batch
print(qualities)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Sep 16 17:32:20 2018

@author:
Chung Yang 31732286
Yi-Pei Chen 31739156
Hao Cheng Cheam 31749564
Wenting Wang 31930946

# Coding question 3.27

"""

from scipy import misc
import matplotlib.pyplot as plt
import numpy as np
from numpy.linalg import norm
from numpy.linalg import svd
from q3_17_powersvd import powersvd # own svd function

# Show input image
def showImg(image):
    print('Input image:')
    plt.figure(1)
    plt.imshow(image, cmap='gray')
    plt.title("The input image")
    plt.draw()
    print(' ')

# Perform SVD, then reconstruct image in several levels and show it
def reconImg(image, pct, figure_n):
    plt.figure(figure_n)
    # SVD
    n, m = image.shape
    u, s, vh = svd(image, full_matrices=True)
#    u, s, vh = powersvd(image, m) # very slow

    # Reconstruct
    idx = int(min(n,m) * pct)
    img_recon = u[:, :idx].dot(np.diag(s[:idx])).dot(vh[:idx,:])
    print('Use Top {:.0%} of the singular values.'.format(pct), '\n')

    # Percentage of Frobenius norm
    fb_norm_pct = norm(s[:idx])/norm(s)
    print('{:.5%} percent of the Forbenius norm is captured.'.format(fb_norm_pct))

    # Show image
    plt.imshow(img_recon, cmap='gray')
    plt.title('Top {:.0%} of the singular values.'.format(pct) + '\n' + \
              '{:.5%} percent of the Forbenius norm is captured.'.format(fb_norm_pct))
    plt.draw()

if __name__ == "__main__":
    # Read Image
    # (put the image file to the folder os.getcwd())
    img = misc.imread('image.jpg', 1)
    showImg(img) # show the input image

    percentages = [0.05, 0.10, 0.25, 0.50]
    figure_n = 2
    for pct in percentages:
        reconImg(img, pct, figure_n) # show the reconstructed image
        figure_n += 1
    plt.show()


from smcpy.model.base_model import BaseModel
import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint

#------------------------------------------------------
#Helper function to use scipy integrator in model class
def mass_spring(state, t, K, g):
    '''
    Return velocity/acceleration given velocity/position and values for
    stiffness and mass
    '''

    # unpack the state vector
    x = state[0]
    xd = state[1]

    # compute acceleration xdd
    xdd = -K*x + g

    # return the two state derivatives
    return [xd, xdd]

#------------------------------------------------------

class SpringMassModel(BaseModel):
    '''
    Defines Spring Mass model with 2 free params (spring stiffness, k & mass, m)
    '''
    def __init__(self, state0=None, time_grid=None):

        #Give default initial conditions & time grid if not specified
        if state0 is None:
            state0 = [0.0, 0.0]
        if time_grid is None:
            time_grid = np.arange(0.0, 10.0, 0.1)

        self._state0 = state0
        self._t = time_grid

    def evaluate(self, *args, **kwargs):
        '''
        Simulate spring mass system for given spring constant. Returns state
        (position, velocity) at all points in time grid
        '''
        params = self.process_args(args, kwargs)
        K = params['K']
        g = params['g']
        results = odeint(mass_spring, self._state0, self._t, args=(K, g))
        #acc = -K*results[:, 0]+g
        #return acc
        return results[:, 0]

if __name__ == '__main__':

    k = 2.5 # Newtons per metre
    m = 1.5 # Kilograms
    g = 9.8
    state0 = [0.0, 0.0]  #initial conditions
    t = np.arange(0.0, 10.0, 0.1)  #time grid for simulation

    #Initialize model & simulate
    model = SpringMassModel(state0, t)
    state = model.evaluate(K=k/m, g=g)

    #plot results
    plt.figure()
    plt.plot(t, state)
    plt.xlabel('TIME (sec)')
    plt.ylabel('States')
    plt.title('Mass-Spring System')
    plt.legend(('$x$ (m)', '$\dot{x}$ (m/sec)'))
    plt.show()


"""Integration tests for WeMo Switch devices."""

import pytest

from pywemo import Switch

class Base:
    """Tests that run for each Switch model."""

    @pytest.mark.vcr()
    def test_turn_on(self, switch):
        switch.on()
        assert switch.get_state(force_update=True) == 1

    @pytest.mark.vcr()
    def test_turn_off(self, switch):
        switch.off()
        assert switch.get_state(force_update=True) == 0

class Test_F7C027(Base):
    """Tests for the WeMo F7C027 model switch."""

    @pytest.fixture
    def switch(self, vcr):
        with vcr.use_cassette("WeMo_US_2.00.2769.PVT.yaml"):
            return Switch("http://192.168.1.100:49153/setup.xml")

    @pytest.mark.vcr()
    def test_config_any(self, switch):
        assert switch._config_any == {
            "binaryState": "0",
            "firmwareVersion": "WeMo_US_2.00.2769.PVT",
            "iconVersion": "0|49153",
        }

class Test_F7C063(Base):
    """Tests for the WeMo F7C063 model switch."""

    @pytest.fixture
    def switch(self, vcr):
        with vcr.use_cassette("WeMo_WW_2.00.11420.PVT-OWRT-SNSV2.yaml"):
            return Switch("http://192.168.1.100:49153/setup.xml")

    @pytest.mark.vcr()
    def test_config_any(self, switch):
        assert switch._config_any == {
            "binaryState": "0",
            "firmwareVersion": "WeMo_WW_2.00.11420.PVT-OWRT-SNSV2",
            "hkSetupCode": "012-34-567",
            "hwVersion": "v3",
            "iconVersion": "2|49153",
            "new_algo": "1",
        }

class Test_WSP080(Base):
    """Tests for the WeMo WSP080 model switch."""

    @pytest.fixture
    def switch(self, vcr):
        with vcr.use_cassette("WEMO_WW_4.00.20101902.PVT-RTOS-SNSV4.yaml"):
            return Switch("http://192.168.1.100:49153/setup.xml")

    @pytest.mark.vcr()
    def test_config_any(self, switch):
        assert switch._config_any == {
            "binaryState": "0",
            "firmwareVersion": "WEMO_WW_4.00.20101902.PVT-RTOS-SNSV4",
            "hkSetupCode": "012-34-567",
            "hwVersion": "v4",
            "iconVersion": "1|49152",
            "new_algo": "1",
            "rtos": "1",
        }


import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import scipy.stats as stats
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import statsmodels.api as sm

os.chdir(r"D:\Data Mining\Sample files")
df = pd.read_csv('Life Expectancy Data.csv')
dataset=df.copy()
dataset.head(10)

# REMOVING UNNECESSARY FEATURES
dataset.drop(['GDP','Population','Income composition of resources', 'Schooling','Total expenditure'],axis=1,inplace=True)

# missing values handling
dataset.isnull().sum()
dataset = dataset[(dataset.Country != 'Monaco')&(dataset.Country != 'San Marino')& (dataset.Country != 'South Sudan') & (dataset.Country != 'Sudan') ]#,'San Marino','Sudan')] # South sudan has alot of missing data.
dataset['Alcohol'].fillna(method='ffill',inplace=True)
dataset['Life expectancy '].fillna(method='ffill',inplace=True)
dataset['Adult Mortality'].fillna(method='ffill',inplace=True)

#Canada and France are mislabeled as Developing
dataset.loc[(dataset.Country == 'Canada'), 'Status'] = "Developed"
dataset.loc[(dataset.Country == 'France'), 'Status'] = "Developed"

dataset.fillna({'Hepatitis_B': df.groupby('Country')['Hepatitis_B'].transform(lambda grp: grp.fillna(np.mean(grp))),
                'Polio': df.groupby('Country')['Polio'].transform(lambda grp: grp.fillna(np.mean(grp))),
                'Diphtheria ': df.groupby('Country')['Diphtheria '].transform(lambda grp: grp.fillna(np.mean(grp)))}
               ,inplace=True)
dataset.fillna({'Hepatitis_B':dataset['Hepatitis_B'].mode().iloc[0]},inplace = True) # some countries dont have this value at all
dataset.isnull().sum()

# ENCODING

dataset.Status = dataset.Status.replace({'Developing':0,
                             'Developed':1})

#QQ plot
stats.probplot(df_out['Life expectancy '], dist="norm", plot=plt)
plt.title('Life Expectancy QQ Plot')
plt.show()
print(stats.shapiro(dataset['Life expectancy ']))

# Outliers identification
before = dataset.describe()
Q1 = dataset.quantile(0.25)
Q3 = dataset.quantile(0.75)
IQR = Q3 - Q1
outliers = ((dataset < (Q1 - 1.5 * IQR)) |(dataset > (Q3 + 1.5 * IQR)))
df_out = dataset[~((dataset < (Q1 - 1.5 * IQR)) |(dataset > (Q3 + 1.5 * IQR))).any(axis=1)]
after = df_out.describe()

X= dataset.drop(['Country' ,'Year','Status'],axis=1)

X = sm.add_constant(X)

y = dataset['Life expectancy ']

X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)

ols = sm.OLS(y_train,X_train) # target, list of independent variable
lr = ols.fit()
print(lr.summary())

while (lr.pvalues.max()>0.05):
    X_train.drop(lr.pvalues.idxmax(),axis=1,inplace=True)
    X_test.drop(lr.pvalues.idxmax(),axis=1, inplace = True)
    ols= sm.OLS(y_train,X_train)
    lr = ols.fit()

X_train.columns # remove the constant column

X_train = X2.drop('const', axis=1, inplace=True)

from sklearn.linear_model import LinearRegression

model = LinearRegression()

#train the model on training set
model.fit(X_train,y_train)

#cross validation
y_pred = model.predict(X_test)

from sklearn.metrics import mean_absolute_error, mean_squared_error
mean_abs_error = mean_absolute_error(y_test,y_pred)
mean_sq_error = mean_squared_error(y_test, y_pred)

import math
rmse = math.sqrt(mean_sq_error)
print (rmse,mean_abs_error)

incorrect results after feature elimination.py