#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu May  6 20:14:53 2021

@author: sebastiengorgoni
"""

import pandas as pd
import numpy as np
import os
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from matplotlib import style
import seaborn as sns
from numpy.matlib import repmat
import random

random.seed(10)

sns.set_theme(style="whitegrid")

#Set the working directory
#os.chdir("/Users/sebastiengorgoni/Documents/HEC Master/Semester 4.2/Quantitative Asset & Risk Management/Assignments/Assignment 2")
print("Current working directory: {0}".format(os.getcwd()))

#Create files in the working directory
if not os.path.isdir('Plot'):
    os.makedirs('Plot')

if not os.path.isdir('Output'):
    os.makedirs('Output')

#Download the data of prices
df_price = pd.read_excel('Data_QAM2.xlsx', 'Prices')
df_price.set_index('Dates', inplace=True)

#Create the client's benchmark (50% World Equities, 50% World Bonds)
benchmark = pd.DataFrame(data=(0.5*df_price['World Equities'] + 0.5*df_price['World Bonds']))

#In/Out Sample Prices
in_sample_price = df_price.loc[(df_price.index <= pd.to_datetime('2010-12-31'))].iloc[:,:]
out_sample_price = df_price.loc[(df_price.index > pd.to_datetime('2010-12-31'))].iloc[:,:]

#Compute the simple returns (IS: In-sample, OS: Out-Sample)
returns_price = np.log(df_price/df_price.shift(1)).replace(np.nan, 0)
returns_IS_price = np.log(in_sample_price/in_sample_price.shift(1)).replace(np.nan, 0)
returns_OS_price = returns_price.loc[(returns_price.index > pd.to_datetime('2010-12-31'))].iloc[:,:]

#Plot the overall cumulative returns of our assets
plt.figure(figsize=(12,7))
plt.plot((returns_price + 1).cumprod()*100)
plt.legend(returns_price.columns, loc='upper left', frameon=False)
plt.title('Cumulative Returns All Assets')
plt.savefig('Plot/returns_asset.png')
plt.close()

#Compute the IS and OS returns of the benchmark
benchmark_return_IS = pd.DataFrame({'IS Benchmark': 0.5*returns_IS_price['World Equities'] + 0.5*returns_IS_price['World Bonds']})
benchmark_return_OS = pd.DataFrame({'OS Benchmark': 0.5*returns_OS_price['World Equities'] + 0.5*returns_OS_price['World Bonds']})

#Compute the IS and OS weights of the benchmark (50% World Equities, 50% World Bonds)
benchmark_weights_IS = pd.DataFrame({'World Equities': np.ones(returns_IS_price.shape[0])*0.5,
                                     'World Bonds': np.ones(returns_IS_price.shape[0])*0.5,
                                     'US Investment Grade': np.zeros(returns_IS_price.shape[0]),
                                     'US High Yield': np.zeros(returns_IS_price.shape[0]),
                                     'Gold': np.zeros(returns_IS_price.shape[0]),
                                     'Energy': np.zeros(returns_IS_price.shape[0]),
                                     'Copper': np.zeros(returns_IS_price.shape[0])}, index = returns_IS_price.index)

benchmark_weights_OS = pd.DataFrame({'World Equities': np.ones(returns_OS_price.shape[0])*0.5,
                                     'World Bonds': np.ones(returns_OS_price.shape[0])*0.5,
                                     'US Investment Grade': np.zeros(returns_OS_price.shape[0]),
                                     'US High Yield': np.zeros(returns_OS_price.shape[0]),
                                     'Gold': np.zeros(returns_OS_price.shape[0]),
                                     'Energy': np.zeros(returns_OS_price.shape[0]),
                                     'Copper': np.zeros(returns_OS_price.shape[0])}, index = returns_OS_price.index)

# =============================================================================
# =============================================================================
# #Part 1: SAA
# =============================================================================
# =============================================================================

# =============================================================================
# 1.1
# =============================================================================

def MCR_calc(alloc, Returns):
    """
    This function computes the marginal contribution to risk (MCR), which
    determine how much the portfolio volatility would change if we increase
    the weight of a particular asset.

    Parameters
    ----------
    alloc : TYPE
        Weights in the investor's portfolio
    Returns : TYPE
        The returns of the portfolio's assets
    Returns
    -------
    MCR : Object
        Marginal contribution to risk (MCR)
    """
    ptf=np.multiply(Returns,alloc);
    ptfReturns=np.sum(ptf,1); # Summing across columns
    vol_ptf=np.std(ptfReturns);
    Sigma=np.cov(np.transpose(Returns))
    MCR=np.matmul(Sigma,np.transpose(alloc))/vol_ptf;
    return MCR

###ERC Allocation###
def ERC(alloc,Returns):
    """
    This function computes the Equally-Weighted Risk Contribution Portfolio (ERC),
    which attributes the same risk contribution to all the assets.

    Parameters
    ----------
    alloc : TYPE
        Weights in the investor's portfolio
    Returns : TYPE
        The returns of the portfolio's assets
    Returns
    -------
    criterions : Object
        Optimal weights of assets in the portfolio.
    """
    ptf=np.multiply(Returns.iloc[:,:],alloc);
    ptfReturns=np.sum(ptf,1); # Summing across columns
    vol_ptf=np.std(ptfReturns);
    indiv_ERC=alloc*MCR_calc(alloc,Returns);
    criterion=np.power(indiv_ERC-vol_ptf/len(alloc),2)
    criterion=np.sum(criterion)*1000000000
    return criterion

x0 = np.array([0, 0, 0, 0, 0, 0, 0])+0.00001 #Set the first weights of the Gradient Descent

cons=({'type':'eq', 'fun': lambda x:sum(x)-1}, #Sum of weights is equal to 1
      {'type':'ineq', 'fun': lambda x: x[1]-0.01}, #Minimum of 1% in World Bonds
      {'type':'ineq', 'fun': lambda x: x[2]-0.01}) #Minimum of 1% in US investment grades

Bounds= [(0 , 1) for i in range(0,7)] #Long only positions

#Optimisation
res_ERC = minimize(ERC, x0, method='SLSQP', args=(returns_IS_price),bounds=Bounds,constraints=cons,options={'disp': True})

labels=list(returns_IS_price)

plt.figure(figsize=(15,7))

#Plot the optimal weights using ERC
weight_to_chart=np.array(res_ERC.x)
plt.subplot(131)
plt.plot(labels,weight_to_chart,'ro',labels,weight_to_chart*0,'b-')
plt.xticks(rotation=90)
plt.title('Optimal Allocation ERC')
plt.ylabel('Ptf Weight')
plt.tight_layout()

#MCR for Max ERC
MCR_chart=MCR_calc(res_ERC.x, returns_IS_price)
MCR_chart=np.array(MCR_chart)
plt.subplot(132)
plt.plot(labels,MCR_chart,'ro',labels,MCR_chart*0,'b-')
plt.xticks(rotation=90)
plt.title('MCR of ERC')
plt.ylabel('MCR')
plt.tight_layout()

#Test for the alpha x MCR ratio
ratio=res_ERC.x*MCR_chart
plt.subplot(133)
plt.plot(labels,ratio,'ro',labels,ratio*0,'b-')
plt.xticks(rotation=90)
plt.title('Alpha x MCR')
plt.ylabel('Ratio')
plt.tight_layout()
plt.savefig('Plot/ERC_output.png')
plt.close()

###Sharp Ratio Allocation###
def Max_SR(alloc, Returns, Rf=0):
    """
    This function computes the Maximum Sharpe Ratio Portfolio (SR),
    which attributes the weights that maximize the Sharpe Ratio.

    Parameters
    ----------
    alloc : TYPE
        Weights in the investor's portfolio
    Returns : TYPE
        The returns of the portfolio's assets
    Rf : TYPE
        The risk-free rate, assumed to be 0.

    Returns
    -------
    SR : Object
        Optimal weights of assets in the portfolio.
    """
    ptf=np.multiply(Returns.iloc[:,:],alloc);
    ptfReturns=np.sum(ptf,1); # Summing across columns
    mu_bar=np.mean(ptfReturns)-Rf;
    vol_ptf=np.std(ptfReturns);
    SR=-mu_bar/vol_ptf;
    return SR

x0 = np.array([0, 0, 0, 0, 0, 0, 0])+0.00001 #Set the first weights of the Gradient Descent

cons=({'type':'eq', 'fun': lambda x:sum(x)-1}, #Sum of weights is equal to 1
      {'type':'ineq', 'fun': lambda x: x[1]-0.01}, #Minimum of 1% in World Bonds
      {'type':'ineq', 'fun': lambda x: x[2]-0.01}) #Minimum of 1% in US investment grades

Bounds= [(0 , 1) for i in range(0,7)] #Long only positions

#Optimisation
res_SR = minimize(Max_SR, x0, method='SLSQP', args=(returns_IS_price),bounds=Bounds,constraints=cons,options={'disp': True})

plt.figure(figsize=(15,7))

#Plot the optimal weights using Sharpe Ratio
weight_to_chart=np.array(res_SR.x)
plt.subplot(131)
plt.plot(labels,weight_to_chart,'ro',labels,weight_to_chart*0,'b-')
plt.xticks(rotation=90)
plt.title('Optimal Allocation SR')
plt.ylabel('Ptf Weight')
plt.tight_layout()

#MCR for Max SR
MCR_chart=MCR_calc(res_SR.x, returns_IS_price)
MCR_chart=np.array(MCR_chart)
plt.subplot(132)
plt.plot(labels,MCR_chart,'ro',labels,MCR_chart*0,'b-')
plt.xticks(rotation=90)
plt.title('MCR of SR')
plt.ylabel('MCR')
plt.tight_layout()

#Test for excess return to MCR Ratio
reward=np.mean(returns_IS_price, 0)
ratio=MCR_chart/reward
plt.subplot(133)
plt.plot(labels,ratio,'ro',labels,ratio*0,'b-')
plt.xticks(rotation=90)
plt.title('Excess Return to MCR Ratio')
plt.ylabel('Ratio')
plt.tight_layout()
plt.savefig('Plot/SR_output.png')
plt.close()

###Most-Diversified Portfolio###
def Max_DP(alloc, Returns):
    """
    This function computes the Most-Diversified Portfolio (MDP),
    which attributes the same relative marginal volatility to all the assets.

    Parameters
    ----------
    alloc : TYPE
        Weights in the investor's portfolio
    Returns : TYPE
        The returns of the portfolio's assets
    Returns
    -------
    Div_ratio : Object
        Optimal weights of assets in the portfolio.
    """
    ptf=np.multiply(Returns.iloc[:,:],alloc);
    ptfReturns=np.sum(ptf,1); # Summing across columns
    vol_ptf=np.std(ptfReturns);

    numerator=np.multiply(np.std(Returns),alloc);
    numerator=np.sum(numerator);

    Div_Ratio=-numerator/vol_ptf;
    return Div_Ratio

x0 = np.array([0, 0, 0, 0, 0, 0, 0])+0.00001 #Set the first weights of the Gradient Descent

cons=({'type':'eq', 'fun': lambda x:sum(x)-1}, #Sum of weights is equal to 1
      {'type':'ineq', 'fun': lambda x: x[1]-0.01}, #Minimum of 1% in World Bonds
      {'type':'ineq', 'fun': lambda x: x[2]-0.01}) #Minimum of 1% in US investment grades

Bounds= [(0 , 1) for i in range(0,7)] #Long only positions

#Optimisation
res_MDP = minimize(Max_DP, x0, method='SLSQP', args=(returns_IS_price), bounds=Bounds,constraints=cons, options={'disp': True}) #options={'disp': True} means I want feedbacks in my optimization

plt.figure(figsize=(15,7))

#Plot the optimal weights using Most-Diversified Portfolio (MDP)
weight_to_chart=np.array(res_MDP.x)
plt.subplot(131)
plt.plot(labels,weight_to_chart,'ro',labels,weight_to_chart*0,'b-')
plt.xticks(rotation=90)
plt.title('Optimal Allocation MDP')
plt.ylabel('Ptf Weight')
plt.tight_layout()

# MCR for Max SR
MCR_chart=MCR_calc(res_MDP.x, returns_IS_price);
MCR_chart=np.array(MCR_chart)
plt.subplot(132)
plt.plot(labels,MCR_chart,'ro',labels,MCR_chart*0,'b-')
plt.xticks(rotation=90)
plt.title('MCR of MDP')
plt.ylabel('MCR')
plt.tight_layout()

#Test for MCR/Risk ratio
MCR_risk = MCR_chart/np.std(returns_IS_price)
plt.subplot(133)
plt.plot(labels, MCR_risk, 'ro', labels, MCR_chart*0,'b-')
plt.xticks(rotation=90)
plt.title('MCR-per-Risk Ratio')
plt.ylabel('Ratio')
plt.tight_layout()
plt.savefig('Plot/MDP_output.png')
plt.close()

#Compute the returns of the ERC, SR, MDP portfolio IS
portfolio_IS_ERC = pd.DataFrame({'ERC': np.sum(np.multiply(returns_IS_price,np.transpose(res_ERC.x)), 1)})
portfolio_IS_SR = pd.DataFrame({'SR': np.sum(np.multiply(returns_IS_price,np.transpose(res_SR.x)), 1)})
portfolio_IS_MDP = pd.DataFrame({'MDP': np.sum(np.multiply(returns_IS_price,np.transpose(res_MDP.x)), 1)})

def hit_ratio(return_dataset):
    """
    This function determine the hit ratio of any time series returns

    Parameters
    ----------
    return_dataset : TYPE
        The returns of the asset.

    Returns
    -------
    TYPE
        It returns the hit ratio.

    """
    return len(return_dataset[return_dataset >= 0]) / len(return_dataset)

def max_drawdown(cum_returns):
    """
    It determines the maximum drawdown over the cumulative returns
    of a time series.

    Parameters
    ----------
    cum_returns : TYPE
        Cumulative Return.

    Returns
    -------
    max_monthly_drawdown : TYPE
        Evolution of the max drawdown (negative output).

    """
    roll_max = cum_returns.cummax()
    monthly_drawdown = cum_returns/roll_max - 1
    max_monthly_drawdown = monthly_drawdown.cummin()
    return max_monthly_drawdown

#Determine the maximum drawdown (minimum as the output is negative)
max_drawdown((portfolio_IS_ERC['ERC']+1).cumprod()).min()

def perf(data, benchmark, name, name_plt):
    """
    This function compute all the required performances of a time series.
    It also plot the monthly returns, the evolution of the mayx drawdown and
    the cumulative return of the portfolio vs. benchmark

    Parameters
    ----------
    data : TYPE
        Returns of a given portfolio.
    benchmark : TYPE
        Returns of the benchmark.
    name : TYPE
        Name of the dataframe.
    name_plt : TYPE
        Name given to the plot.

    Returns
    -------
    df : TYPE
        Return a dataframe that contains the annualized returns, volatility,
        Sharpe ratio, max drawdown and hit ratio.

    """
    plt.figure(figsize=(20,7))
    plt.subplot(131)
    plt.plot(data, 'b')
    plt.title("Monthly Returns", fontsize=15)
    exp = np.mean(data,0)*12
    vol = np.std(data,0)*np.power(12,0.5)
    sharpe = exp/vol
    max_dd = max_drawdown((data+1).cumprod())
    plt.subplot(132)
    plt.plot(max_dd, 'g')
    plt.title("Evolution of Max Drawdown", fontsize=15)
    hit = hit_ratio(data)
    df = pd.DataFrame({name: [exp, vol, sharpe, max_dd.min(), hit]}, index = ['Mean', 'Volatility', 'SharpeRatio', 'MaxDrawdown', 'HitRatio'])
    plt.subplot(133)
    plt.plot((data + 1).cumprod()*100, 'b', label='Portfolio')
    plt.plot((benchmark + 1).cumprod()*100, 'r', label='Benchmark')
    plt.title(name_plt, fontsize=15)
    plt.legend(loc='upper left', frameon=True)
    plt.savefig('Plot/'+name_plt+'.png')
    plt.close()
    return df

#Determine the perfomances of the ERC, SR, MDP portfolio IS
ERC_IS_results = perf(portfolio_IS_ERC['ERC'], benchmark_return_IS['IS Benchmark'], 'ERC', 'ERC Cumulative Returns In-Sample')
SR_IS_results = perf(portfolio_IS_SR['SR'], benchmark_return_IS['IS Benchmark'], 'SR', 'SR Cumulative Returns In-Sample')
MDP_IS_results = perf(portfolio_IS_MDP['MDP'], benchmark_return_IS['IS Benchmark'], 'MDP', 'MDP Cumulative Returns In-Sample')

#Determine the perfomances of thebenchmark portfolio IS
benchmark_IS_results = perf(benchmark_return_IS['IS Benchmark'], benchmark_return_IS['IS Benchmark'], 'Benchmark IS', 'Benchmark Cumulative Returns In-Sample')

#Merge all results
SAA_IS_results = pd.concat([benchmark_IS_results, ERC_IS_results, SR_IS_results, MDP_IS_results], axis=1)
SAA_IS_results.to_latex('Output/SAA_IS_results.tex')

###Tracking Error###
""" Check the end of code. """

###Information Ratio###
""" Check the end of code. """

# =============================================================================
# 1.2
# =============================================================================

#Compute the returns of the ERC, SR, MDP portfolio OS
portfolio_OS_ERC = pd.DataFrame({'ERC': np.sum(np.multiply(returns_OS_price,np.transpose(res_ERC.x)), 1)})
portfolio_OS_SR = pd.DataFrame({'SR': np.sum(np.multiply(returns_OS_price,np.transpose(res_SR.x)), 1)})
portfolio_OS_MDP = pd.DataFrame({'MDP': np.sum(np.multiply(returns_OS_price,np.transpose(res_MDP.x)), 1)})

#Determine the perfomances of the ERC, SR, MDP portfolio OS
ERC_OS_results = perf(portfolio_OS_ERC['ERC'], benchmark_return_OS['OS Benchmark'], 'ERC', 'ERC Cumulative Returns Out-Sample')
SR_OS_results = perf(portfolio_OS_SR['SR'], benchmark_return_OS['OS Benchmark'], 'SR', 'SR Cumulative Returns Out-Sample')
MDP_OS_results = perf(portfolio_OS_MDP['MDP'], benchmark_return_OS['OS Benchmark'], 'MDP', 'MDP Cumulative Returns Out-Sample')

#Determine the perfomances of thebenchmark portfolio OS
benchmark_OS_results = perf(benchmark_return_OS['OS Benchmark'], benchmark_return_OS['OS Benchmark'], 'Benchmark OS', 'Benchmark Cumulative Returns Out-Sample')

#Merge all results
SAA_OS_results = pd.concat([benchmark_OS_results, ERC_OS_results, SR_OS_results, MDP_OS_results], axis=1)
SAA_OS_results.to_latex('Output/SAA_OS_results.tex')

###Tracking Error###
""" Check the end of code. """

###Information Ratio###
""" Check the end of code. """

# =============================================================================
# =============================================================================
# Part 2: TAA
# =============================================================================
# =============================================================================

# =============================================================================
# 2.1
# =============================================================================

#Download the data of carry
df_carry = pd.read_excel("Data_QAM2.xlsx",sheet_name='Carry')
df_carry.index = df_carry['Unnamed: 0']
df_carry.index.name = 'date'
del df_carry['Unnamed: 0']

###Value In-Sample###

#Take the IS carry
df_carry_insample = df_carry[df_carry.index <= pd.to_datetime('2010-12-31')]

#Standardize the dataframe
df_carry_insample_Z = (df_carry_insample - np.mean(df_carry_insample))/np.std(df_carry_insample)
df_carry_insample_Z['median'] = df_carry_insample_Z.median(axis=1)
df_carry_insample_Z

#The value factor will oppose the assets with a z-score higher than the median of that month to the assets with a score lower than that median value.
df_carry_insample_Z_pos = df_carry_insample_Z.copy()
for value in df_carry_insample_Z.iloc[:,0:7].columns:
    list_value = []
    for i in range(len(df_carry_insample_Z)):
        if df_carry_insample_Z[value][i] > df_carry_insample_Z['median'][i]:
            list_value.append(1)
        else:
            list_value.append(-1)
    df_carry_insample_Z_pos[f'{value}'] = list_value
#df_carry_insample_Z_pos.to_csv('Carry_postion.csv')
df_carry_insample_Z_pos

position = df_carry_insample_Z_pos

position[position[position.columns] > 0]

del position['median']

#Attribue the same weights to all long positions (short position respectively).
weight_final = {}
for i in range(len(position)):
    num_pos = np.sum(np.array(position.values[i]) > 0)
    num_neg = np.sum(np.array(position.values[i]) < 0)
    array = []
    for j in range(len(position.values[i])):
        #print(position.values[i][j]/num_neg)
        if position.iloc[i,j] > 0:
           position.iloc[i,j] = position.iloc[i,j]/num_pos
        else:
            position.iloc[i,j] = position.iloc[i,j]/num_neg
    #print(array)
    #position.loc[i] = array
    #print(position.values[i])
#position.to_csv('value_position.csv')

position_IS = position.copy()

#Compute the returns of the value factor IS
portofolio_value = position.mul(returns_IS_price).sum(axis=1)

#Allocate a 2% ex ante volatility budget
vol_budget = 0.02
vol_value_scaled_IS = vol_budget/portofolio_value.std()

value_IS_scaled = pd.DataFrame({'Value': vol_value_scaled_IS * portofolio_value})

###Momentum In-Sample###

#Source: https://www.youtube.com/watch?v=dnrJ4zwCADM

#Calculate the returns over the past 11 months
returns_IS_past11 = (returns_IS_price+1).rolling(11).apply(np.prod) - 1
returns_IS_past11 = returns_IS_past11.dropna()

#Compute the quintile  of each asset each periods
returns_IS_quantile = returns_IS_past11.T.apply(lambda x: pd.qcut(x, 5, labels=False, duplicates="drop"), axis=0).T

#Long asset with the highest quantile (winners), short asset with the lowest quantile (losers)
for i in returns_IS_quantile.columns:
    returns_IS_quantile.loc[returns_IS_quantile[i] == 4, i] = 0.5
    returns_IS_quantile.loc[returns_IS_quantile[i] == 0, i] = -0.5
    returns_IS_quantile.loc[returns_IS_quantile[i] == 1, i] = 0
    returns_IS_quantile.loc[returns_IS_quantile[i] == 2, i] = 0
    returns_IS_quantile.loc[returns_IS_quantile[i] == 3, i] = 0

#Check if the wum of weights is equal to zero
np.sum(returns_IS_quantile, axis=1) == 0

#Shift the weights as we ignore the last month
weights_IS_mom = returns_IS_quantile.shift(1, axis = 0).dropna()

#Compute the returns of the momentum factor OS
portofolio_mom = weights_IS_mom.multiply(returns_IS_price).sum(axis=1)

#Allocate a 2% ex ante volatility budget
vol_budget = 0.02
vol_mom_scaled_IS = vol_budget/portofolio_mom.std()

mom_IS_scaled = pd.DataFrame({'Momentum': vol_mom_scaled_IS * portofolio_mom})

###Collect the Results###
#Compute the performances of value and momentum
value_IS_results = perf(value_IS_scaled['Value'], benchmark_return_IS['IS Benchmark'], 'Value', 'Value Cumulative Returns In-Sample')
mom_IS_results = perf(mom_IS_scaled['Momentum'], benchmark_return_IS['IS Benchmark'], 'Momentum', 'Momentum Cumulative Returns In-Sample')

#Merge Value and Momentum IS
ValMom_IS_results = pd.concat([benchmark_IS_results, value_IS_results, mom_IS_results], axis=1)
ValMom_IS_results.to_latex('Output/ValMom_IS_results.tex')

#Merge Value and Benchmark IS
ValBen_IS_results = pd.concat([benchmark_IS_results, value_IS_results], axis=1)
ValBen_IS_results.to_latex('Output/ValBen_IS_results.tex')

#Merge Momentum and Benchmark IS
MomBen_IS_results = pd.concat([benchmark_IS_results, mom_IS_results], axis=1)
MomBen_IS_results.to_latex('Output/MomBen_IS_results.tex')

# =============================================================================
# 2.2
# =============================================================================

#Download the VIX Index
df_vix = pd.read_excel("Data_QAM2.xlsx",sheet_name='VIX')
df_vix.set_index('Dates', inplace=True)

#Plot the VIX
"""
plt.figure(figsize=(10,7))
plt.plot(df_vix)
plt.title('Level of VIX Index')
plt.savefig('Plot/VIX.png')
"""

#Standardize the VIX
df_vix = (df_vix - df_vix.mean())/df_vix.std()

#Compute the percentage change of VIX
df_vix['Percentage Change'] = np.log(df_vix['VIX']/df_vix['VIX'].shift(1))
df_vix['Percentage Change'] = df_vix['Percentage Change'].replace(np.nan, 0)

#Set the IS and OS VIX
df_vix_IS = df_vix.loc[(df_vix.index <= pd.to_datetime('2010-12-31'))]
df_vix_OS = df_vix.loc[(df_vix.index > pd.to_datetime('2010-12-31'))]

#Compute the 90% quantile of VIX
quantile = df_vix_IS.quantile(q=0.90)
df_vix_IS['Quantile'] = np.ones(df_vix_IS.shape[0])*quantile.VIX

#Plot the IS VIX level and its percentage change
plt.figure(figsize=(15,7))
plt.subplot(121)
plt.plot(df_vix_IS['Percentage Change'], 'r')
plt.title('Percentage Change of VIX In-Sample')
plt.subplot(122)
plt.plot(df_vix_IS['VIX'], label = 'Standardized VIX')
plt.plot(df_vix_IS['Quantile'], label = '90% Quantile')
plt.title('Standardized VIX In-Sample')
plt.legend(loc='upper left', frameon=True)
plt.savefig('Plot/VIX_IS.png')
plt.close()

#Plot the covariance matrices and 10-month rolling covariances between VIX and value/momentum factor
plt.figure(figsize=(15,10))
plt.subplot(221)
corr_value_IS = sns.heatmap(pd.concat([df_vix_IS['VIX'], value_IS_scaled], axis=1).corr(), annot=True)
plt.title('Full Period Covarianve In-Sample')
plt.subplot(222)
corr_mom_IS = sns.heatmap(pd.concat([df_vix_IS['VIX'], mom_IS_scaled], axis=1).corr(), annot=True)
plt.title('Full Period Covarianve In-Sample')
plt.subplot(223)
plt.plot(df_vix_IS['VIX'].rolling(10).corr(value_IS_scaled))
plt.title('Rolling Covariance Value-VIX (10 Months) In-sample')
plt.subplot(224)
plt.plot(df_vix_IS['VIX'].rolling(10).corr(mom_IS_scaled))
plt.title('Rolling Covariance Momentum-VIX (10 Months) In-sample')
plt.savefig('Plot/VIX_analysis_IS.png')
plt.close()

###Parametric###
df_vix_IS.loc[df_vix_IS['VIX'] <= df_vix_IS['Quantile'], 'Long/Short'] = 1 #Expansion if VIX is lower than the 90% quantile
df_vix_IS.loc[df_vix_IS['VIX'] > df_vix_IS['Quantile'], 'Long/Short'] = -1 #Recession if VIX is higher than the 90% quantile

ValMom_returns_IS = pd.concat([value_IS_scaled['Value'], mom_IS_scaled['Momentum']], axis=1)

#Take the data from 2001-08-31 as the momentum factor is realized from this date
ValMom_returns_IS_adj = ValMom_returns_IS.loc[ValMom_returns_IS.index >= '2001-09-28']
df_vix_IS_adj = df_vix_IS.loc[df_vix_IS.index >= '2001-08-31', 'Long/Short']

#Compute the parametric weights between value and momentum factor (to determine the TAA)
lambda_ra=3
numerator = []
denominator = []
n = 0
for i in range (0, len(ValMom_returns_IS_adj)):
        temp_num = df_vix_IS['Long/Short'][i] * ValMom_returns_IS_adj.iloc[i]
        temp_den = (df_vix_IS['Long/Short'][i]*df_vix_IS['Long/Short'][i]) * np.multiply(ValMom_returns_IS_adj.iloc[i], ValMom_returns_IS_adj.iloc[i].transpose())
        numerator.append(temp_num)
        denominator.append(temp_den)
        n += 1

alpha = (1/lambda_ra)*(np.sum(numerator, axis=0)/np.sum(denominator))
alpha_para_IS = alpha/(alpha[0] + alpha[1])

#Determine the returns of the portfolio when using only parametrics
TAA_IS_parametrics_returns = value_IS_scaled['Value']*alpha_para_IS[0] + mom_IS_scaled['Momentum']*alpha_para_IS[1]
TAA_IS_parametrics_results = perf(TAA_IS_parametrics_returns, benchmark_return_IS['IS Benchmark'], 'TAA_IS_Parametrics', 'TAA (Parametrics) Cumulative Returns In-Sample')

###Complex Strategy###
"""
- Assign the parametric weights to the value and momentum factor when the standardized VIX is lower than its 90% quantile
- Assign 100% to the Momentum and 0% to value factor standardized VIX is higher than its 90% quantile and increasing.
- Assign -100% to the Momentum and 200% to value factor standardized VIX is higher than its 90% quantile and decreasing.
"""
df_vix_IS.loc[(df_vix_IS['VIX'] <= df_vix_IS['Quantile']), 'Value Position'] = alpha_para_IS[0] #Long Value
df_vix_IS.loc[(df_vix_IS['VIX'] > df_vix_IS['Quantile'])  & (df_vix_IS['Percentage Change'] >= 0), 'Value Position'] = 0 #Short Value
df_vix_IS.loc[(df_vix_IS['VIX'] > df_vix_IS['Quantile'])  & (df_vix_IS['Percentage Change'] < 0), 'Value Position'] = 2 #Long Value

df_vix_IS.loc[df_vix_IS['VIX'] <= df_vix_IS['Quantile'], 'Mom Position'] = alpha_para_IS[1] #Long Mom
df_vix_IS.loc[(df_vix_IS['VIX'] > df_vix_IS['Quantile'])  & (df_vix_IS['Percentage Change'] >= 0), 'Mom Position'] = 1 #Long Value
df_vix_IS.loc[(df_vix_IS['VIX'] > df_vix_IS['Quantile']) & (df_vix_IS['Percentage Change'] < 0), 'Mom Position'] = -1 #Long Value

TAA_IS_VIX = pd.DataFrame({'Returns Strategy': value_IS_scaled['Value']*df_vix_IS['Value Position'] + mom_IS_scaled['Momentum']*df_vix_IS['Mom Position']}).replace(np.nan, 0)

TAA_IS_VIX_results = perf(TAA_IS_VIX['Returns Strategy'], benchmark_return_IS['IS Benchmark'], 'TAA_IS_VIX', 'TAA (Own Strategy) Cumulative Returns In-Sample')

TAA_IS = pd.concat([benchmark_IS_results, value_IS_results, mom_IS_results, TAA_IS_VIX_results, TAA_IS_parametrics_results], axis=1)
TAA_IS.to_latex('Output/TAA_IS.tex')

###Tracking Error###
""" Check the end of code. """

###Information Ratio###
""" Check the end of code. """

# =============================================================================
# 2.3
# =============================================================================

###Value###

#Take the OS carry
df_carry_outsample = df_carry[df_carry.index > pd.to_datetime('2010-12-31')]

#Standardize the carry
df_carry_outsample_Z = (df_carry_outsample - np.mean(df_carry_outsample))/np.std(df_carry_outsample)
df_carry_outsample_Z['median'] = df_carry_outsample_Z.median(axis=1)
df_carry_outsample_Z

#The value factor will oppose the assets with a z-score higher than the median of that month to the assets with a score lower than that median value.
df_carry_outsample_Z_pos = df_carry_outsample_Z.copy()
for value in df_carry_outsample_Z.iloc[:,0:7].columns:
    list_value = []
    for i in range(len(df_carry_outsample_Z)):
        if df_carry_outsample_Z[value][i] > df_carry_outsample_Z['median'][i]:
            list_value.append(1)
        else:
            list_value.append(-1)
    df_carry_outsample_Z_pos[f'{value}'] = list_value
#df_carry_outsample_Z_pos.to_csv('Carry_postion.csv')
df_carry_outsample_Z_pos

position = df_carry_outsample_Z_pos
position[position[position.columns] > 0]

del position['median']

#Attribue the same weights to all long positions (short position respectively).
weight_final = {}
for i in range(len(position)):
    num_pos = np.sum(np.array(position.values[i]) > 0)
    num_neg = np.sum(np.array(position.values[i]) < 0)
    array = []
    for j in range(len(position.values[i])):
        #print(position.values[i][j]/num_neg)
        if position.iloc[i,j] > 0:
           position.iloc[i,j] = position.iloc[i,j]/num_pos
        else:
            position.iloc[i,j] = position.iloc[i,j]/num_neg
    #print(array)
    #position.loc[i] = array
    #print(position.values[i])
position.to_csv('value_position.csv')

position_OS = position.copy()

#Compute the returns of the value factor OS
portofolio_value = position.mul(returns_OS_price).sum(axis=1)

#Allocate a 2% ex ante volatility budget (IS volatility)

value_OS_scaled = pd.DataFrame({'Value': vol_value_scaled_IS * portofolio_value})
#value_OS_scaled = pd.DataFrame({'Value': portofolio_value})

###Momentum###

#Source: https://www.youtube.com/watch?v=dnrJ4zwCADM

#Calculate the returns over the past 11 months
returns_past11 = (returns_price+1).rolling(11).apply(np.prod) - 1
returns_past11 = returns_past11.dropna()

#Compute the quintile  of each asset each periods
returns_quantile = returns_past11.T.apply(lambda x: pd.qcut(x, 5, labels=False, duplicates="drop"), axis=0).T

#Long asset with the highest quantile (winners), short asset with the lowest quantile (losers)
for i in returns_quantile.columns:
    returns_quantile.loc[returns_quantile[i] == 4, i] = 0.5
    returns_quantile.loc[returns_quantile[i] == 0, i] = -0.5
    returns_quantile.loc[returns_quantile[i] == 1, i] = 0
    returns_quantile.loc[returns_quantile[i] == 2, i] = 0
    returns_quantile.loc[returns_quantile[i] == 3, i] = 0

#Check if the wum of weights is equal to zero
np.sum(returns_quantile, axis=1) == 0

#Shift the weights as we ignore the last month
weights_mom = returns_quantile.shift(1, axis = 0).dropna()

#Compute the returns of the momentum factor OS
portofolio_mom_OS = weights_mom.multiply(returns_OS_price).sum(axis=1)
portofolio_mom_OS = pd.DataFrame({'Momentum': portofolio_mom_OS})
portofolio_mom_OS =  portofolio_mom_OS.loc[(portofolio_mom_OS != 0).any(1)]

#Allocate a 2% ex ante volatility budget (IS volatility)

mom_OS_scaled = vol_mom_scaled_IS * portofolio_mom_OS
#mom_OS_scaled = portofolio_mom_OS

###Collect the Results###
#Compute the performances of value and momentum
value_OS_results = perf(value_OS_scaled['Value'], benchmark_return_OS['OS Benchmark'], 'Value', 'Value Cumulative Returns Out-Sample')
mom_OS_results = perf(mom_OS_scaled['Momentum'], benchmark_return_OS['OS Benchmark'], 'Momentum', 'Momentum Cumulative Returns Out-Sample')

#Merge Value and Momentum OS
ValMom_OS_results = pd.concat([benchmark_OS_results, value_OS_results, mom_OS_results], axis=1)
ValMom_OS_results.to_latex('Output/ValMom_OS_results.tex')

#Merge Value and Benchmark OS
ValBen_OS_results = pd.concat([benchmark_OS_results, value_OS_results], axis=1)
ValBen_OS_results.to_latex('Output/ValBen_OS_results.tex')

#Merge Momentum and Benchmark OS
MomBen_OS_results = pd.concat([benchmark_OS_results, mom_OS_results], axis=1)
MomBen_OS_results.to_latex('Output/MomBen_OS_results.tex')

###Out-Sample VIX###

#Compute the quantile
quantile = df_vix_IS.quantile(q=0.90)
df_vix_OS['Quantile'] = np.ones(df_vix_OS.shape[0])*quantile.VIX

"""
plt.figure(figsize=(15,7))
plt.subplot(121)
plt.plot(df_vix_OS['Percentage Change'], 'r')
plt.title('Percentage Change of VIX Out-Sample')
plt.subplot(122)
plt.plot(df_vix_OS['VIX'], 'b', label = 'Standardized VIX')
#plt.plot(df_vix_IS['Quantile'], label = '90% Quantile')
plt.title('Standardized VIX Out-Sample')
#plt.legend(loc='upper left', frameon=True)
plt.savefig('Plot/VIX_OS.png')
plt.close()
"""
#Plot the covariance matrices and 10-month rolling covariances between VIX and value/momentum factor
plt.figure(figsize=(15,10))
plt.subplot(221)
corr_value_OS = sns.heatmap(pd.concat([df_vix_OS['VIX'], value_OS_scaled], axis=1).corr(), annot=True)
plt.title('Full Period Covarianve Out-Sample')
plt.subplot(222)
corr_mom_OS = sns.heatmap(pd.concat([df_vix_OS['VIX'], mom_OS_scaled], axis=1).corr(), annot=True)
plt.title('Full Period Covarianve Out-Sample')
plt.subplot(223)
plt.plot(df_vix_OS['VIX'].rolling(10).corr(value_OS_scaled))
plt.title('Rolling Covariance Value-VIX (10 Months) Out-sample')
plt.subplot(224)
plt.plot(df_vix_OS['VIX'].rolling(10).corr(mom_OS_scaled))
plt.title('Rolling Covariance Momentum-VIX (10 Months) Out-sample')
plt.savefig('Plot/VIX_analysis_OS.png')
plt.close()

###Parametric
TAA_OS_parametrics_returns = value_OS_scaled['Value']*alpha_para_IS[0] + mom_OS_scaled['Momentum']*alpha_para_IS[1]
TAA_OS_parametrics_results = perf(TAA_OS_parametrics_returns, benchmark_return_OS['OS Benchmark'], 'TAA_OS_Parametrics', 'TAA (Parametrics) Cumulative Returns Out-Sample')

###Complex Strategy (Adaptive Quantile)###
"""
- Assign the parametric weights to the value and momentum factor when the standardized VIX is lower than its 90% quantile
- Assign 100% to the Momentum and 0% to value factor standardized VIX is higher than its 90% quantile and increasing.
- Assign -100% to the Momentum and 200% to value factor standardized VIX is higher than its 90% quantile and decreasing.
- The quantile is determined from the first period of the IS up to the last data observed (updated each months)
"""
df_vix_OS['Value Position'] = 0
df_vix_OS['Mom Position'] = 0
quantile_tot = []
for i in range(0, df_vix_OS.shape[0]):
    quantile = df_vix.iloc[:df_vix_IS.shape[0]+1+i, 0].quantile(q=0.90)
    quantile_tot.append(quantile)
    if (df_vix_OS.iloc[i, 0] <= quantile):
        df_vix_OS.iloc[i, 3] = alpha_para_IS[0]
    elif (df_vix_OS.iloc[i, 0] > quantile) & (df_vix_OS.iloc[i, 1] >= 0):
        df_vix_OS.iloc[i, 3] = 0
    elif (df_vix_OS.iloc[i, 0] > quantile) & (df_vix_OS.iloc[i, 1] < 0):
        df_vix_OS.iloc[i, 3] = 2
    else:
        pass

    if (df_vix_OS.iloc[i, 0] <= quantile):
        df_vix_OS.iloc[i, 4] = alpha_para_IS[1]
    elif (df_vix_OS.iloc[i, 0] > quantile) & (df_vix_OS.iloc[i, 1] >= 0):
        df_vix_OS.iloc[i, 4] = 1
    elif (df_vix_OS.iloc[i, 0] > quantile) & (df_vix_OS.iloc[i, 1] < 0):
        df_vix_OS.iloc[i, 4] = -1
    else:
        pass

quantile_OS = pd.DataFrame({'Quantile': quantile_tot}, index = df_vix_OS.index)

#Plot the OS VIX level and its percentage change
plt.figure(figsize=(15,7))
plt.subplot(121)
plt.plot(df_vix_OS['Percentage Change'], 'r')
plt.title('Percentage Change of VIX Out-Sample')
plt.subplot(122)
plt.plot(df_vix_OS['VIX'], 'b', label = 'Standardized VIX')
plt.plot(quantile_OS, 'orange', label = '90% Quantile (In-Sample & Out-Sample)')
plt.title('Standardized VIX Out-Sample')
plt.legend(loc='upper left', frameon=True)
plt.savefig('Plot/VIX_OS.png')
plt.close()

#Compute the returns of the TAA
TAA_OS_VIX = pd.DataFrame({'Returns Strategy': value_OS_scaled['Value']*df_vix_OS['Value Position'] + mom_OS_scaled['Momentum']*df_vix_OS['Mom Position']}).replace(np.nan, 0)

#Compute the performances of the TAA
TAA_OS_VIX_results = perf(TAA_OS_VIX['Returns Strategy'], benchmark_return_OS['OS Benchmark'], 'TAA_OS_VIX', 'TAA (Own Strategy) Cumulative Returns Out-Sample')

#Merge all performances
TAA_OS = pd.concat([benchmark_OS_results, value_OS_results, mom_OS_results, TAA_OS_VIX_results, TAA_OS_parametrics_results], axis=1)
TAA_OS.to_latex('Output/TAA_OS.tex')

###Tracking Error###
""" Check the end of code. """

###Information Ratio###
""" Check the end of code. """

# =============================================================================
# =============================================================================
# Part 3
# =============================================================================
# =============================================================================

###Collecting All Weights###
SAA_weights_IS = pd.DataFrame({'World Equities': np.ones(returns_IS_price.shape[0])*res_SR.x[0],
                                   'World Bonds': np.ones(returns_IS_price.shape[0])*res_SR.x[1],
                                   'US Investment Grade': np.ones(returns_IS_price.shape[0])*res_SR.x[2],
                                   'US High Yield': np.ones(returns_IS_price.shape[0])*res_SR.x[3],
                                   'Gold': np.ones(returns_IS_price.shape[0])*res_SR.x[4],
                                   'Energy': np.ones(returns_IS_price.shape[0])*res_SR.x[5],
                                   'Copper': np.ones(returns_IS_price.shape[0])*res_SR.x[6]
                                   }, index = returns_IS_price.index)

SAA_weights_OS = pd.DataFrame({'World Equities': np.ones(returns_OS_price.shape[0])*res_SR.x[0],
                                   'World Bonds': np.ones(returns_OS_price.shape[0])*res_SR.x[1],
                                   'US Investment Grade': np.ones(returns_OS_price.shape[0])*res_SR.x[2],
                                   'US High Yield': np.ones(returns_OS_price.shape[0])*res_SR.x[3],
                                   'Gold': np.ones(returns_OS_price.shape[0])*res_SR.x[4],
                                   'Energy': np.ones(returns_OS_price.shape[0])*res_SR.x[5],
                                   'Copper': np.ones(returns_OS_price.shape[0])*res_SR.x[6]
                                   }, index = returns_OS_price.index)

value_weights_IS = position_IS

value_weights_OS = position_OS

mom_weights_IS = weights_IS_mom

mom_weights_OS = weights_mom.iloc[(weights_mom.index > pd.to_datetime('2010-12-31'))].iloc[:,:]

VIX_weights_IS = pd.DataFrame({'Value Position': df_vix_IS['Value Position'],
                               'Mom Position': df_vix_IS['Mom Position']})

VIX_weights_OS = pd.DataFrame({'Value Position': df_vix_OS['Value Position'],
                               'Mom Position': df_vix_OS['Mom Position']})

value_weights_IS = value_weights_IS.multiply(VIX_weights_IS['Value Position'], axis='index').replace(np.nan, 0)

mom_weight_IS = mom_weights_IS.multiply(VIX_weights_IS['Mom Position'], axis='index').replace(np.nan, 0)

TAA_weights_IS = value_weights_IS.multiply(VIX_weights_IS['Value Position'], axis='index') +  mom_weights_IS.multiply(VIX_weights_IS['Mom Position'], axis='index').replace(np.nan, 0)

TAA_weights_OS = (value_weights_OS.multiply(VIX_weights_OS['Value Position'], axis='index') +  mom_weights_OS.multiply(VIX_weights_OS['Mom Position'], axis='index').replace(np.nan, 0))

weight_target_IS = SAA_weights_IS + TAA_weights_IS

weight_target_OS = SAA_weights_OS + TAA_weights_OS

#Determine the covariance matrix
sigma_IS = returns_IS_price.cov().values
sigma_OS = returns_OS_price.cov().values

###Tracking Error Formula###
def TE(weight_ptf, weight_target, sigma=sigma_IS):
    """
    This function computes the tracking error between
    a portfolio and a benchmark.

    Parameters
    ----------
    weight_ptf : TYPE
        Weight of our portfolio.

    weight_target : TYPE
        Weight the benchmark portfolio.

    sigma : TYPE
        Covariance matrix.

    Returns
    -------
    vol_month : TYPE
        Monthly Tracking Error.

    """

    diff_alloc = weight_ptf - weight_target
    temp =  np.matmul(diff_alloc.T, sigma)
    var_month = np.matmul(temp, diff_alloc)
    vol_month = np.power(var_month, 0.5)
    return vol_month

###Ex Ante Tracking Error & Replication Portfolion In-Sample###

#The sum of weights is equal to 1, and no position to US investment grade
cons = ({'type':'eq', 'fun': lambda x: sum(x)-1},
        {'type':'eq', 'fun': lambda x: x[2]})

#We allow long/short positions (we put high bounds as we sometimes have position of 110-120%)
Bounds = [(-10 , 10) for i in range(0,7)]

#Run the TE in each periods IS
weight_opt_IS = []
TE_ReplicationvsTarget_IS = []
for i in range(0, weight_target_IS.shape[0]):
    x0 = weight_target_IS.copy()*0 + (1/weight_target_IS.shape[1]) #Initial weights for gradient descent (equal weights)
    x0 = x0.iloc[0].values
    weight_target = weight_target_IS.iloc[i, :].values
    sigma = sigma_IS
    opt_TE_IS = minimize(TE, x0, method='SLSQP', bounds=Bounds, args=(weight_target), constraints=cons, options={'disp': True}) #bounds=Bounds,
    weight_opt_IS.append(opt_TE_IS.x)
    TE_ReplicationvsTarget_IS.append(opt_TE_IS.fun)

#Store the TE of Replication vs Target IS
TE_ReplicationvsTarget_IS = pd.DataFrame({'TE_IS': TE_ReplicationvsTarget_IS}, index = weight_target_IS.index)

#Store the weights of the replication ptf IS
weight_rep_IS = np.array(weight_opt_IS)
weight_rep_IS = pd.DataFrame({'World Equities': weight_rep_IS[:, 0],
                              'World Bonds': weight_rep_IS[:, 1],
                              'US Investment Grade': weight_rep_IS[:, 2],
                              'US High Yield': weight_rep_IS[:, 3],
                              'Gold': weight_rep_IS[:, 4],
                              'Energy': weight_rep_IS[:, 5],
                              'Copper': weight_rep_IS[:, 6]}, index = weight_target_IS.index)

#Determine the returns of the target and replication portolio
return_target_IS  = np.sum(np.multiply(returns_IS_price, weight_target_IS), axis=1)

return_rep_IS  = np.sum(np.multiply(returns_IS_price, weight_rep_IS), axis=1)

#Determine the performances of the target and replication portolio
perf_target_IS = perf(return_target_IS, benchmark_return_IS['IS Benchmark'], 'Model', 'Target Portfolio In-Sample')
perf_rep_IS = perf(return_rep_IS, benchmark_return_IS['IS Benchmark'], 'Replication', 'Replication Portfolio In-Sample')

#Merge the performances of target and replication ptf
perf_target_rep_IS = pd.concat([perf_target_IS, perf_rep_IS], axis=1)
perf_target_rep_IS.to_latex('Output/perf_target_rep_IS.tex')

#Plot the IS cumulative Returns and ex-ante TE between target and replication ptf
plt.figure(figsize=(14,7))
plt.subplot(121)
plt.plot((return_target_IS+1).cumprod()*100, 'b', label='Model Portfolio')
plt.plot((return_rep_IS+1).cumprod()*100, 'r', label='Replication Portfolio')
plt.title('In-Sample Cumulative Returns')
plt.legend(loc='upper left', frameon=True)
plt.subplot(122)
plt.plot(TE_ReplicationvsTarget_IS)
plt.title('Ex-Ante Tracking Error between Real Ptf vs. SAA+TAA')
plt.savefig('Plot/target_replication_is.png')
plt.close()

###Ex Post Tracking Error & Replication Portfolio Out-Sample###

#The sum of weights is equal to 1, and no position to US investment grade
cons = ({'type':'eq', 'fun': lambda x: sum(x)-1},
        {'type':'eq', 'fun': lambda x: x[2]})

#We allow long/short positions
Bounds = [(-10 , 10) for i in range(0,7)] #Long/short positions

#Run the TE in each periods OS
weight_opt_OS = []
TE_ReplicationvsTarget_OS = []
for i in range(0, weight_target_OS.shape[0]):
    x0 = weight_target_IS.copy()*0 + (1/weight_target_IS.shape[1]) #Initial weights for gradient descent (equal weights)
    x0 = x0.iloc[0].values
    weight_target = weight_target_OS.iloc[i, :].values
    sigma = sigma_IS
    opt_TE_OS = minimize(TE, x0, method='SLSQP', bounds=Bounds, args=(weight_target), constraints=cons, options={'disp': True}) #Powell works bounds=Bounds,
    weight_opt_OS.append(opt_TE_OS.x)
    TE_ReplicationvsTarget_OS.append(opt_TE_OS.fun)

#Store the TE of Replication vs Target OS
TE_ReplicationvsTarget_OS = pd.DataFrame({'TE_OS': TE_ReplicationvsTarget_OS}, index = weight_target_OS.index)

#Store the weights of the replication ptf OS
weight_rep_OS = np.array(weight_opt_OS)
weight_rep_OS = pd.DataFrame({'World Equities': weight_rep_OS[:, 0],
                              'World Bonds': weight_rep_OS[:, 1],
                              'US Investment Grade': weight_rep_OS[:, 2],
                              'US High Yield': weight_rep_OS[:, 3],
                              'Gold': weight_rep_OS[:, 4],
                              'Energy': weight_rep_OS[:, 5],
                              'Copper': weight_rep_OS[:, 6]}, index = weight_target_OS.index)

#Determine the returns of the target and replication portolio
return_target_OS  = np.sum(np.multiply(returns_OS_price, weight_target_OS), axis=1)
return_rep_OS  = np.sum(np.multiply(returns_OS_price, weight_rep_OS), axis=1)

#Determine the performances of the target and replication portolio
perf_target_OS = perf(return_target_OS, benchmark_return_OS['OS Benchmark'], 'Model', 'Target Portfolio Out-Sample')
perf_rep_OS = perf(return_rep_OS, benchmark_return_OS['OS Benchmark'], 'Replication', 'Replication Portfolio Out-Sample')

#Merge the performances of target and replication ptf
perf_target_rep_OS = pd.concat([perf_target_OS, perf_rep_OS], axis=1)
perf_target_rep_OS.to_latex('Output/perf_target_rep_OS.tex')

#Plot the OS cumulative Returns and ex-ante TE between target and replication ptf
plt.figure(figsize=(14,7))
plt.subplot(121)
plt.plot((return_target_OS+1).cumprod()*100, 'b', label='Model Portfolio')
plt.plot((return_rep_OS+1).cumprod()*100, 'r', label='Replication Portfolio')
plt.title('Out-of-Sample Cumulative Returns')
plt.legend(loc='upper left', frameon=True)
plt.subplot(122)
plt.plot(TE_ReplicationvsTarget_OS)
plt.title('Ex-Post Tracking Error between Real Ptf vs. SAA+TAA')
plt.savefig('Plot/target_replication_os.png')
plt.close()

###Tracking Error###
"""Between SAA and Benchmark"""
#In-Sample
Tot_TE_SAAvsBenchmark_IS = []
for i in range(0, SAA_weights_IS.shape[0]):
    weight_target = benchmark_weights_IS.iloc[i, :].values
    sigma = sigma_IS
    TE_SAAvsBenchmark_IS = TE(SAA_weights_IS.iloc[i, :].values, weight_target)
    Tot_TE_SAAvsBenchmark_IS.append(TE_SAAvsBenchmark_IS)

#Out-Sample
Tot_TE_SAAvsBenchmark_OS = []
for i in range(0, SAA_weights_OS.shape[0]):
    weight_target = benchmark_weights_OS.iloc[i, :].values
    sigma = sigma_IS
    TE_SAAvsBenchmark_OS = TE(SAA_weights_OS.iloc[i, :].values, weight_target)
    Tot_TE_SAAvsBenchmark_OS.append(TE_SAAvsBenchmark_OS)

TE_SAAvsBenchmark_IS = pd.DataFrame({'TE_IS': Tot_TE_SAAvsBenchmark_IS}, index = benchmark_return_IS.index)
TE_SAAvsBenchmark_OS = pd.DataFrame({'TE_OS': Tot_TE_SAAvsBenchmark_OS}, index = benchmark_return_OS.index)

plt.figure(figsize=(13,7))
plt.subplot(121)
plt.plot(TE_SAAvsBenchmark_IS, 'b')
plt.title('Ex-Ante Tracking Error between SAA vs. Benchmark')
plt.subplot(122)
plt.plot(TE_SAAvsBenchmark_OS, 'r')
plt.title('Ex-Post Tracking Error between SAA vs. Benchmark')
plt.savefig('Plot/TE_SAAvsBenchmark.png')
plt.close()

"""Between TAA and Benchmark"""
#In-Sample
Tot_TE_TAAvsBenchmark_IS = []
for i in range(0, TAA_weights_IS.shape[0]):
    weight_target = benchmark_weights_IS.iloc[i, :].values
    sigma = sigma_IS
    TE_TAAvsBenchmark_IS = TE(TAA_weights_IS.iloc[i, :].values, weight_target)
    Tot_TE_TAAvsBenchmark_IS.append(TE_TAAvsBenchmark_IS)

#Out-Sample
Tot_TE_TAAvsBenchmark_OS = []
for i in range(0, TAA_weights_OS.shape[0]):
    weight_target = benchmark_weights_OS.iloc[i, :].values
    sigma = sigma_IS
    TE_TAAvsBenchmark_OS = TE(TAA_weights_OS.iloc[i, :].values, weight_target)
    Tot_TE_TAAvsBenchmark_OS.append(TE_TAAvsBenchmark_OS)

TE_TAAvsBenchmark_IS = pd.DataFrame({'TE_IS': Tot_TE_TAAvsBenchmark_IS}, index = TAA_weights_IS.index)
TE_TAAvsBenchmark_OS = pd.DataFrame({'TE_OS': Tot_TE_TAAvsBenchmark_OS}, index = TAA_weights_OS.index)

plt.figure(figsize=(13,7))
plt.subplot(121)
plt.plot(TE_TAAvsBenchmark_IS , 'b')
plt.title('Ex-Ante Tracking Error between TAA vs. Benchmark')
plt.subplot(122)
plt.plot(TE_TAAvsBenchmark_OS , 'r')
plt.title('Ex-Post Tracking Error between TAA vs. Benchmark')
plt.savefig('Plot/TE_TAAvsBenchmark.png')
plt.close()

"""Between TAA and SAA"""
#In-Sample
Tot_TE_SAAvsTAA_IS = []
for i in range(0, weight_target_IS.shape[0]):
    weight_target = SAA_weights_IS.iloc[i, :].values
    sigma = sigma_IS
    TE_SAAvsTAA_IS = TE(TAA_weights_IS.iloc[i, :].values, weight_target)
    Tot_TE_SAAvsTAA_IS.append(TE_SAAvsTAA_IS)

#Out-Sample
Tot_TE_SAAvsTAA_OS = []
for i in range(0, weight_target_OS.shape[0]):
    weight_target = SAA_weights_OS.iloc[i, :].values
    sigma = sigma_IS
    TE_SAAvsTAA_OS = TE(TAA_weights_OS.iloc[i, :].values, weight_target)
    Tot_TE_SAAvsTAA_OS.append(TE_SAAvsTAA_OS)

TE_SAAvsTAA_IS = pd.DataFrame({'TE_IS': Tot_TE_SAAvsTAA_IS}, index = TAA_weights_IS.index)
TE_SAAvsTAA_OS = pd.DataFrame({'TE_OS': Tot_TE_SAAvsTAA_OS}, index = TAA_weights_OS.index)

plt.figure(figsize=(13,7))
plt.subplot(121)
plt.plot(TE_SAAvsTAA_IS, 'b')
plt.title('Ex-Ante Tracking Error between TAA vs. SAA')
plt.subplot(122)
plt.plot(TE_SAAvsTAA_OS, 'r')
plt.title('Ex-Post Tracking Error between TAA vs. SAA')
plt.savefig('Plot/TE_SAAvsTAA.png')
plt.close()

"""Between SAA+TAA and SAA"""
#In-Sample
Tot_TE_SAAvsTarget_IS = []
for i in range(0, weight_target_IS.shape[0]):
    weight_target = SAA_weights_IS.iloc[i, :].values
    sigma = sigma_IS
    TE_SAAvsTarget_IS = TE(weight_target_IS.iloc[i, :].values, weight_target)
    Tot_TE_SAAvsTarget_IS.append(TE_SAAvsTarget_IS)

#Out-Sample
Tot_TE_SAAvsTarget_OS = []
for i in range(0, weight_target_OS.shape[0]):
    weight_target = SAA_weights_OS.iloc[i, :].values
    sigma = sigma_IS
    TE_SAAvsTarget_OS = TE(weight_target_OS.iloc[i, :].values, weight_target)
    Tot_TE_SAAvsTarget_OS.append(TE_SAAvsTarget_OS)

TE_SAAvsTarget_IS = pd.DataFrame({'TE_IS': Tot_TE_SAAvsTarget_IS}, index = weight_target_IS.index)
TE_SAAvsTarget_OS = pd.DataFrame({'TE_OS': Tot_TE_SAAvsTarget_OS}, index = weight_target_OS.index)

plt.figure(figsize=(13,7))
plt.subplot(121)
plt.plot(TE_SAAvsTarget_IS, 'b')
plt.title('Ex-Ante Tracking Error between SAA+TAA vs. SAA')
plt.subplot(122)
plt.plot(TE_SAAvsTarget_OS, 'r')
plt.title('Ex-Post Tracking Error between SAA+TAA vs. SAA')
plt.savefig('Plot/TE_SAAvsTarget.png')
plt.close()

"""Between Replication and SAA+TAA"""
plt.figure(figsize=(13,7))
plt.subplot(121)
plt.plot(TE_ReplicationvsTarget_IS, 'b')
plt.title('Ex-Ante Tracking Error between Replication vs. SAA+TAA')
plt.subplot(122)
plt.plot(TE_ReplicationvsTarget_OS, 'r')
plt.title('Ex-Post Tracking Error between Replication vs. SAA+TAA')
plt.savefig('Plot/TE_ReplicationvsTarget.png')
plt.close()

"""Collect All Output SAA+TAA Gradually"""
TE_annualized_output_1 =  pd.DataFrame({'In-Sample': [TE_SAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5)],
                          'Out-Sample': [TE_SAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5)]},
                            index=['SAA vs. Benchmark'])
TE_annualized_output_1.to_latex('Output/TE_annualized_output_1.tex')

TE_annualized_output_2 =  pd.DataFrame({'In-Sample': [TE_SAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5)],
                          'Out-Sample': [TE_SAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5)]},
                            index=['SAA vs. Benchmark', 'TAA vs. Benchmark'])
TE_annualized_output_2.to_latex('Output/TE_annualized_output_2.tex')

TE_annualized_output_3 =  pd.DataFrame({'In-Sample': [TE_SAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_SAAvsTAA_IS['TE_IS'].mean(axis=0)*(12**0.5)],
                          'Out-Sample': [TE_SAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_SAAvsTAA_OS['TE_OS'].mean(axis=0)*(12**0.5)]},
                            index=['SAA vs. Benchmark', 'TAA vs. Benchmark', 'TAA vs. SAA'])
TE_annualized_output_3.to_latex('Output/TE_annualized_output_3.tex')

TE_annualized_output_4 =  pd.DataFrame({'In-Sample': [TE_SAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_SAAvsTAA_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_SAAvsTarget_IS['TE_IS'].mean(axis=0)*(12**0.5)],
                          'Out-Sample': [TE_SAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_SAAvsTAA_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_SAAvsTarget_OS['TE_OS'].mean(axis=0)*(12**0.5)]},
                            index=['SAA vs. Benchmark', 'TAA vs. Benchmark', 'TAA vs. SAA', 'TAA+SAA vs. SAA'])
TE_annualized_output_4.to_latex('Output/TE_annualized_output_4.tex')

TE_annualized_output_5 =  pd.DataFrame({'In-Sample': [TE_SAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_SAAvsTAA_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_SAAvsTarget_IS['TE_IS'].mean(axis=0)*(12**0.5), TE_ReplicationvsTarget_IS['TE_IS'].mean(axis=0)*(12**0.5)],
                          'Out-Sample': [TE_SAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_TAAvsBenchmark_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_SAAvsTAA_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_SAAvsTarget_OS['TE_OS'].mean(axis=0)*(12**0.5), TE_ReplicationvsTarget_OS['TE_OS'].mean(axis=0)*(12**0.5)]},
                            index=['SAA vs. Benchmark', 'TAA vs. Benchmark', 'TAA vs. SAA', 'TAA+SAA vs. SAA', 'Replication vs. TAA+SAA'])
TE_annualized_output_5.to_latex('Output/TE_annualized_output_5.tex')

###Information Ratio###

def info_ratio(return_p, return_b):
    """
    This function determine the information ratio of an investment.
    Source: https://en.wikipedia.org/wiki/Information_ratio

    Parameters
    ----------
    return_p : TYPE
        Returns of the actual portfolio.
    return_b : TYPE
        Returns of the benchmark.

    Returns
    -------
    TYPE
        It returns the annualized info. ratio.

    """
    excess = return_p - return_b
    return (excess.mean(axis=0)*12)/(excess.std(axis=0)*(12**0.5))

"""Between SAA and Benchmark"""
IR_SAAvsBenchmark_IS = info_ratio(portfolio_IS_SR['SR'], benchmark_return_IS['IS Benchmark'])
IR_SAAvsBenchmark_OS = info_ratio(portfolio_OS_SR['SR'], benchmark_return_OS['OS Benchmark'])

"""Between TAA and Benchmark"""
IR_TAAvsBenchmark_IS = info_ratio(TAA_IS_VIX['Returns Strategy'], benchmark_return_IS['IS Benchmark'])
IR_TAAvsBenchmark_OS = info_ratio(TAA_OS_VIX['Returns Strategy'], benchmark_return_OS['OS Benchmark'])

"""Between TAA and SAA"""
IR_TAAvsSAA_IS = info_ratio(TAA_IS_VIX['Returns Strategy'], portfolio_IS_SR['SR'])
IR_TAAvsSAA_OS = info_ratio(TAA_OS_VIX['Returns Strategy'], portfolio_OS_SR['SR'])

"""Between SAA+TAA and SAA"""
IR_TargetvsSAA_IS = info_ratio(return_target_IS, portfolio_IS_SR['SR'])
IR_TargetvsSAA_OS = info_ratio(return_target_OS, portfolio_OS_SR['SR'])

"""Between Replication Portfolion and SAA+TAA (i.e. Target Portfolio)"""
IR_ReplicationvsTarget_IS = info_ratio(return_rep_IS, return_target_IS)
IR_ReplicationvsTarget_OS = info_ratio(return_rep_OS, return_target_OS)

"""Collect All Output Gradually"""
IR_output_1 = pd.DataFrame({'In-Sample': [IR_SAAvsBenchmark_IS],
                            'Out-Sample': [IR_SAAvsBenchmark_OS]},
                         index=['SAA vs. Benchmark'])
IR_output_1.to_latex('Output/IR_output_1.tex')

IR_output_2 = pd.DataFrame({'In-Sample': [IR_SAAvsBenchmark_IS, IR_TAAvsBenchmark_IS],
                            'Out-Sample': [IR_SAAvsBenchmark_OS, IR_TAAvsBenchmark_OS]},
                         index=['SAA vs. Benchmark', 'TAA vs. Benchmark'])
IR_output_2.to_latex('Output/IR_output_2.tex')

IR_output_3 = pd.DataFrame({'In-Sample': [IR_SAAvsBenchmark_IS, IR_TAAvsBenchmark_IS, IR_TAAvsSAA_IS],
                          'Out-Sample': [IR_SAAvsBenchmark_OS, IR_TAAvsBenchmark_OS, IR_TAAvsSAA_OS]},
                         index=['SAA vs. Benchmark', 'TAA vs. Benchmark', 'TAA vs. SAA'])
IR_output_3.to_latex('Output/IR_output_3.tex')

IR_output_4 = pd.DataFrame({'In-Sample': [IR_SAAvsBenchmark_IS, IR_TAAvsBenchmark_IS, IR_TAAvsSAA_IS, IR_TargetvsSAA_IS],
                          'Out-Sample': [IR_SAAvsBenchmark_OS, IR_TAAvsBenchmark_OS, IR_TAAvsSAA_OS, IR_TargetvsSAA_OS]},
                         index=['SAA vs. Benchmark', 'TAA & SAA vs. SAA', 'TAA vs. SAA', 'TAA+SAA vs. SAA'])
IR_output_4.to_latex('Output/IR_output_4.tex')

IR_output_5 = pd.DataFrame({'In-Sample': [IR_SAAvsBenchmark_IS, IR_TAAvsBenchmark_IS, IR_TAAvsSAA_IS, IR_TargetvsSAA_IS, IR_ReplicationvsTarget_IS],
                          'Out-Sample': [IR_SAAvsBenchmark_OS, IR_TAAvsBenchmark_OS, IR_TAAvsSAA_OS, IR_TargetvsSAA_OS, IR_ReplicationvsTarget_OS]},
                         index=['SAA vs. Benchmark', 'TAA & SAA vs. SAA', 'TAA vs. SAA', 'TAA+SAA vs. SAA', 'Replication vs. TAA & SAA'])
IR_output_5.to_latex('Output/IR_output_5.tex')

###Allocation/Selection Performance Attribution In-Sample###

#Weight for each type of asset (target)
weights_bonds_target_IS = weight_target_IS['World Bonds'] + weight_target_IS['US Investment Grade'] + weight_target_IS['US High Yield']
weights_equities_target_IS = weight_target_IS['World Equities']
weights_commodities_target_IS = weight_target_IS['Gold'] +  weight_target_IS['Energy'] + weight_target_IS['Copper']

#Weight for each type of asset (replication portfolio)
weights_bonds_rep_IS = weight_rep_IS['World Bonds'] + weight_rep_IS['US Investment Grade'] + weight_rep_IS['US High Yield']
weights_equities_rep_IS = weight_rep_IS['World Equities']
weights_commodities_rep_IS = weight_rep_IS['Gold'] +  weight_rep_IS['Energy'] + weight_rep_IS['Copper']

#Performance for each type of asset (target)
perf_bonds_target_IS = (np.multiply(returns_IS_price['World Bonds'], weight_target_IS['World Bonds']) + np.multiply(returns_IS_price['US Investment Grade'], weight_target_IS['US Investment Grade']) + np.multiply(returns_IS_price['US High Yield'], weight_target_IS['US High Yield']))/weights_bonds_target_IS
perf_equities_target_IS = (np.multiply(returns_IS_price['World Equities'], weight_target_IS['World Equities']))/weights_equities_target_IS
perf_commodities_target_IS = (np.multiply(returns_IS_price['Gold'], weight_target_IS['Gold']) + np.multiply(returns_IS_price['Energy'], weight_target_IS['Energy']) + np.multiply(returns_IS_price['Copper'], weight_target_IS['Copper']))/weights_commodities_target_IS

#Performance for each type of asset (replication portfolio)
perf_bonds_rep_IS = (np.multiply(returns_IS_price['World Bonds'], weight_rep_IS['World Bonds']) + np.multiply(returns_IS_price['US Investment Grade'], weight_rep_IS['US Investment Grade']) + np.multiply(returns_IS_price['US High Yield'], weight_rep_IS['US High Yield']))/weights_bonds_rep_IS
perf_equities_rep_IS = (np.multiply(returns_IS_price['World Equities'], weight_rep_IS['World Equities']))/weights_equities_rep_IS
perf_commodities_rep_IS = (np.multiply(returns_IS_price['Gold'], weight_rep_IS['Gold']) + np.multiply(returns_IS_price['Energy'], weight_rep_IS['Energy']) + np.multiply(returns_IS_price['Copper'], weight_rep_IS['Copper']))/weights_commodities_rep_IS

#The portfolio return
R_IS = weights_bonds_rep_IS*perf_bonds_rep_IS +  weights_equities_rep_IS*perf_equities_rep_IS + weights_commodities_rep_IS*perf_commodities_rep_IS

#The target portfolio return
B_IS = weights_bonds_target_IS*perf_bonds_target_IS + weights_equities_target_IS*perf_equities_target_IS + weights_commodities_target_IS*perf_commodities_target_IS

#The Selection Notional Fund
R_S_IS = weights_bonds_target_IS*perf_bonds_rep_IS + weights_equities_target_IS*perf_equities_rep_IS + weights_commodities_target_IS*perf_commodities_rep_IS

#The Allocation Notional Fund
B_S_IS = weights_bonds_rep_IS*perf_bonds_target_IS + weights_equities_rep_IS*perf_equities_target_IS + weights_commodities_rep_IS*perf_commodities_target_IS

plt.figure(figsize=(16,7))
plt.subplot(131)
#Interaction effect
interaction_IS = R_IS - R_S_IS - B_S_IS + B_IS
plt.plot(interaction_IS, 'r')
plt.title('Monthly Interaction Effect In-Sample')

plt.subplot(132)
#Allocation effect
allocation_IS = B_S_IS - B_IS
plt.plot(allocation_IS, 'b')
plt.title('Monthly Allocation Effect In-Sample')

plt.subplot(133)
#Selection effect
selection_IS = R_S_IS - B_IS
plt.plot(selection_IS, 'g')
plt.title('Monthly Selection Effect In-Sample')
plt.savefig('Plot/attribution_IS.png')
plt.close()

###Allocation/Selection Performance Attribution Out-Of-Sample###

#Weight for each type of asset (target)
weights_bonds_target_OS = weight_target_OS['World Bonds'] + weight_target_OS['US Investment Grade'] + weight_target_OS['US High Yield']
weights_equities_target_OS = weight_target_OS['World Equities']
weights_commodities_target_OS = weight_target_OS['Gold'] +  weight_target_OS['Energy'] + weight_target_OS['Copper']

#Weight for each type of asset (replication portfolio)
weights_bonds_rep_OS = weight_rep_OS['World Bonds'] + weight_rep_OS['US Investment Grade'] + weight_rep_OS['US High Yield']
weights_equities_rep_OS = weight_rep_OS['World Equities']
weights_commodities_rep_OS = weight_rep_OS['Gold'] +  weight_rep_OS['Energy'] + weight_rep_OS['Copper']

#Performance for each type of asset (target)
perf_bonds_target_OS = (np.multiply(returns_OS_price['World Bonds'], weight_target_OS['World Bonds']) + np.multiply(returns_OS_price['US Investment Grade'], weight_target_OS['US Investment Grade']) + np.multiply(returns_OS_price['US High Yield'], weight_target_OS['US High Yield']))/weights_bonds_target_OS
perf_equities_target_OS = (np.multiply(returns_OS_price['World Equities'], weight_target_OS['World Equities']))/weights_equities_target_OS
perf_commodities_target_OS = (np.multiply(returns_OS_price['Gold'], weight_target_OS['Gold']) + np.multiply(returns_OS_price['Energy'], weight_target_OS['Energy']) + np.multiply(returns_OS_price['Copper'], weight_target_OS['Copper']))/weights_commodities_target_OS

#Performance for each type of asset (replication portfolio)
perf_bonds_rep_OS = (np.multiply(returns_OS_price['World Bonds'], weight_rep_OS['World Bonds']) + np.multiply(returns_OS_price['US Investment Grade'], weight_rep_OS['US Investment Grade']) + np.multiply(returns_OS_price['US High Yield'], weight_rep_OS['US High Yield']))/weights_bonds_rep_OS
perf_equities_rep_OS = (np.multiply(returns_OS_price['World Equities'], weight_rep_OS['World Equities']))/weights_equities_rep_OS
perf_commodities_rep_OS = (np.multiply(returns_OS_price['Gold'], weight_rep_OS['Gold']) + np.multiply(returns_OS_price['Energy'], weight_rep_OS['Energy']) + np.multiply(returns_OS_price['Copper'], weight_rep_OS['Copper']))/weights_commodities_rep_OS

#The portfolio return
R_OS = weights_bonds_rep_OS*perf_bonds_rep_OS +  weights_equities_rep_OS*perf_equities_rep_OS + weights_commodities_rep_OS*perf_commodities_rep_OS

#The target portfolio return
B_OS = weights_bonds_target_OS*perf_bonds_target_OS + weights_equities_target_OS*perf_equities_target_OS + weights_commodities_target_OS*perf_commodities_target_OS

#The Selection Notional Fund
R_S_OS = weights_bonds_target_OS*perf_bonds_rep_OS + weights_equities_target_OS*perf_equities_rep_OS + weights_commodities_target_OS*perf_commodities_rep_OS

#The Allocation Notional Fund
B_S_OS = weights_bonds_rep_OS*perf_bonds_target_OS + weights_equities_rep_OS*perf_equities_target_OS + weights_commodities_rep_OS*perf_commodities_target_OS

plt.figure(figsize=(16,7))
plt.subplot(131)
#Interaction effect
interaction_OS = R_OS - R_S_OS - B_S_OS + B_OS
plt.plot(interaction_OS, 'r')
plt.title('Monthly Interaction Effect Out-of-Sample')

plt.subplot(132)
#Allocation effect
allocation_OS = B_S_OS - B_OS
plt.plot(allocation_OS, 'b')
plt.title('Monthly Allocation Effect Out-of-Sample')

plt.subplot(133)
#Selection effect
selection_OS = R_S_OS - B_OS
plt.plot(selection_OS, 'g')
plt.title('Monthly Selection Effect Out-of-Sample')
plt.savefig('Plot/attribution_OS.png')
plt.close()

###Collect all annualized Performance Attribution###
performance_attribution = pd.DataFrame({'In-Sample': [np.mean(R_IS, 0)*12, np.mean(B_IS, 0)*12, np.mean(R_S_IS, 0)*12, np.mean(B_S_IS, 0)*12, np.mean(interaction_IS, 0)*12, np.mean(allocation_IS, 0)*12, np.mean(selection_IS, 0)*12],
                                        'Out-of-Sample': [np.mean(R_OS, 0)*12, np.mean(B_OS, 0)*12, np.mean(R_S_OS, 0)*12, np.mean(B_S_OS, 0)*12, np.mean(interaction_OS, 0)*12, np.mean(allocation_OS, 0)*12, np.mean(selection_OS, 0)*12]},
                                       index=['R', 'B', 'R_S', 'B_S', 'Interaction', 'Allocation Effect', 'Selection Effect'])
performance_attribution.to_latex('Output/performance_attribution.tex')


n = int(input())
ans = 0
li = []
for i in range(n):
    a,b = map(int,input().split())
    li.append((a,b))
li = li[::-1]
for (a,b) in li:
    ans += (b-((a+ans)%b))%b
print(ans)


from gi.repository import Gtk, Gdk
Gdk.threads_init()
from gi.repository import GLib
from threading import Thread, RLock, Condition
from .utils import debug
from ._threadutils import Require
import math
import pickle

VERTICAL = Gtk.Orientation.VERTICAL
HORIZONTAL = Gtk.Orientation.HORIZONTAL

class VectorSpin(Gtk.Box):
    def __init__(self, var, valRange, normalize=False, orientation=VERTICAL):
        super().__init__(orientation=orientation)
        self.var = var
        self.normalize = normalize
        for i in range(min(3, len(var))):
            spin = Gtk.SpinButton(
                adjustment=Gtk.Adjustment(var[i], -valRange, valRange, 0.05),
                numeric=True, digits=3)
            self.pack_start(spin, True, True, 0)
            spin.connect('value-changed', self.update_var, i)

    def update_var(self, spin, i):
        with ControlPanel.lock:
            self.var[i] = spin.get_value()
            if self.normalize:
                self.var /= self.var.dot(self.var) ** .5

class GridControl(Gtk.Frame):
    def __init__(self, label):
        super().__init__(label=label, border_width=2)
        self.grid = Gtk.Grid(column_spacing=5, border_width=5)
        self.add(self.grid)

class BoxControl(Gtk.Frame):
    def __init__(self, label, orientation=VERTICAL):
        super().__init__(label=label, border_width=2)
        self.box = Gtk.Box(orientation=orientation)
        self.add(self.box)

class PropsControl(GridControl):
    def __init__(self, label, props):
        super().__init__(label)
        self.widgets = {}
        for i, prop in enumerate(props):
            name, cls, attrs = prop
            label = Gtk.Label(name, halign=Gtk.Align.END)
            self.grid.attach(label, 0, i, 1, 1)
            edit = cls(**attrs)
            self.grid.attach(edit, 1, i, 1, 1)
            self.widgets[name] = edit

class LightControl(PropsControl):
    def __init__(self, light):
        self.light = light
        props = [
            ('enable', Gtk.Switch, dict(active=light.enabled)),
            ('power', Gtk.SpinButton, dict(
                adjustment=Gtk.Adjustment(light.power, 0, light.MAX_POWER, 0.2),
                numeric=True, digits=1)),
            ('pos', VectorSpin,
                dict(var=light.pos, valRange=light.MAX_RANGE, normalize=False)),
        ]
        super().__init__('', props)
        widgets = self.widgets
        widgets['enable'].connect('notify::active', self.update_enable)
        widgets['power'].connect('value-changed', self.update_power)

    def update_enable(self, button, *args):
        with ControlPanel.lock:
            self.light.enabled = button.get_active()

    def update_power(self, spin):
        with ControlPanel.lock:
            self.light.power = float(spin.get_value())

class ColorAdjust(Gtk.ColorButton):
    def __init__(self, target, prop):
        super().__init__(color=Gdk.Color.from_floats(*getattr(target, prop)))

        def update_color(colorButton):
            setattr(target, prop, colorButton.get_color().to_floats())

        self.connect('color-set', update_color)

class MaterialControl(PropsControl):
    def __init__(self, material):
        self.material = material
        props = [
            ('name', Gtk.Entry, dict(text=material.name, editable=False)),
            ('ambient', ColorAdjust, dict(target=material, prop='Ka')),
            ('specular', ColorAdjust, dict(target=material, prop='Ks')),
            ('shininess', Gtk.SpinButton, dict(
                adjustment=Gtk.Adjustment(
                    material.shininess, 0, material.MAX_SHININESS, 1),
                numeric=True, digits=0)),
        ]
        if material.diffuseType == material.DIFFUSE_COLOR:
            props.append(
                ('diffuse', ColorAdjust, dict(target=material, prop='diffuse')))
        super().__init__('', props)
        widgets = self.widgets
        widgets['shininess'].connect('value-changed', self.update_shininess)

    def update_shininess(self, spin):
        self.material.shininess = float(spin.get_value())

class EdgesControl(GridControl):
    def __init__(self, viewer):
        super().__init__('Edges Control')
        self.viewer = viewer

        self.grid.attach(Gtk.Label('Enable'), 0, 0, 1, 1)
        enable = Gtk.Switch(active=viewer.renderer.toonRenderEnable)
        enable.connect('notify::active', self.update_enable)
        insertButton = Gtk.Button('Insert')
        removeButton = Gtk.Button('Remove')
        for i, widget in enumerate((enable, insertButton, removeButton)):
            self.grid.attach(widget, 1 + i, 0, 1, 1)

        self.edgeList = Gtk.ListBox(selection_mode=Gtk.SelectionMode.SINGLE)
        edges = viewer.renderer.toonRenderEdges
        viewer.renderer.toonRenderEdges = []
        for e in edges:
            self.insert_edge(e)
        self.grid.attach(self.edgeList, 0, 1, 4, 1)

        insertButton.connect('clicked', lambda _: self.insert_edge(None))
        removeButton.connect('clicked', lambda _: self.remove_edge())

        save = self.save = Gtk.TextView(editable=False, wrap_mode=Gtk.WrapMode.WORD)
        save.get_buffer().set_text(str(list(sorted(edges))))
        self.grid.attach(save, 0, 2, 4, 2)

        self.update_enable(enable)

    def get_selected_row(self):
        return self.edgeList.get_selected_row()

    def get_selected_index(self):
        row = self.get_selected_row()
        return row.get_index() if row else len(self.edgeList)

    def insert_edge(self, value):
        edges = self.viewer.renderer.toonRenderEdges
        index = self.get_selected_index()
        if value is None:
            try:
                value = edges[index]
            except IndexError:
                value = 0.
        row = Gtk.Scale(adjustment=Gtk.Adjustment(value, 0, 1, 0), digits=5,
            orientation=HORIZONTAL)
        row.connect('format-value', lambda s, value: self.update_value(index, value))
        self.edgeList.insert(row, index)
        self.edgeList.show_all()
        edges.insert(index, value)
        edges.sort()

    def update_enable(self, switch, *args):
        enabled = switch.get_active()
        self.viewer.renderer.toonRenderEnable = enabled
        for widget in self.grid.get_children():
            if widget is switch:
                continue
            widget.set_sensitive(enabled)

    def update_value(self, index, value):
        edges = self.viewer.renderer.toonRenderEdges = [r.get_children()[0].get_value()
            for r in self.edgeList.get_children()]
        self.save.get_buffer().set_text(str(list(sorted(edges))))

    def remove_edge(self):
        row = self.get_selected_row()
        self.edgeList.remove(row)
        self.viewer.renderer.toonRenderEdges.pop(row.get_index())

class JointControl(BoxControl):
    def __init__(self, viewer, joints):
        super().__init__('Joints Control')
        self.viewer = viewer
        self.joints = joints
        for joint in sorted(joints, key=lambda joint: joint.name):
            scale = Gtk.Scale(adjustment=Gtk.Adjustment(0, -180, 180, 0),
                digits=0, orientation=HORIZONTAL)
            scale.connect('format-value', self.update_angle, joint)
            row = Gtk.Box(orientation=HORIZONTAL)
            row.pack_start(Gtk.Label(joint.name), False, False, 2)
            row.pack_end(scale, True, True, 0)
            self.box.pack_start(row, False, False, 0)

    def update_angle(self, scale, value, joint):
        with ControlPanel.lock:
            self.viewer.selectedJoint = joint
            # debug('seleted', joint)
            joint.angle = scale.get_value() / 180 * math.pi

class State:
    PATH = '/tmp/raygllib_state'
    vars = {}

    @staticmethod
    def save():
        pickle.dump(State.vars, open(State.PATH, 'wb'), -1)

    @staticmethod
    def load():
        State.vars = pickle.load(open(State.PATH, 'rb'))

class CameraControl(BoxControl):
    def __init__(self, viewer):
        super().__init__('Camera Control', HORIZONTAL)
        saveButton = Gtk.Button('Save')
        loadButton = Gtk.Button('Load')
        self.viewer = viewer
        self.box.pack_start(saveButton, False, False, 0)
        self.box.pack_start(loadButton, False, False, 0)

        saveButton.connect('clicked', self.save)
        loadButton.connect('clicked', self.load)

    def load(self, *args):
        State.load()
        cam = self.viewer.camera
        with ControlPanel.lock:
            cam.up[:] = State.vars['up']
            cam.pos[:] = State.vars['pos']
            cam.center[:] = State.vars['center']
            cam._scale = State.vars['scale']
            cam._gen_view_mat()
            cam._gen_proj_mat()

    def save(self, *args):
        cam = self.viewer.camera
        with ControlPanel.lock:
            State.vars['up'] = cam.up
            State.vars['pos'] = cam.pos
            State.vars['center'] = cam.center
            State.vars['scale'] = cam._scale
        State.save()

class SilhouetteControl(GridControl):
    def __init__(self, viewer):
        super().__init__('Silhoette Control')
        self.viewer = viewer
        self.grid.attach(Gtk.Label('Enable'), 0, 0, 1, 1)
        enable = Gtk.Switch(active=viewer.silhouetteEnable)
        enable.connect('notify::active', self.update_enable)
        self.grid.attach(enable, 1, 0, 1, 1)

        edgeWidth = Gtk.Scale(
            adjustment=Gtk.Adjustment(viewer.silhouetteWidth, 0.001, 0.1),
            digits=3, orientation=HORIZONTAL)
        edgeWidth.connect('format-value', self.update_edge_width)
        self.grid.attach(edgeWidth, 0, 1, 4, 1)

    def update_enable(self, switch, *args):
        enabled = switch.get_active()
        with ControlPanel.lock:
            self.viewer.silhouetteEnable = enabled

    def update_edge_width(self, widget, value):
        with ControlPanel.lock:
            self.viewer.silhouetteWidth = widget.get_value()

class FileLoader(BoxControl):
    RECENT_LIMIT = 5
    MAX_HEIGHT = 5

    def __init__(self, viewer):
        super().__init__('File')
        self.viewer = viewer
        # Add recent chooser
        recentFilter = Gtk.RecentFilter()
        recentFilter.add_pattern('*.dae')
        self.recentList = recentList = Gtk.RecentChooserWidget(
            select_multiple=False,
            limit=self.RECENT_LIMIT,
            sort_type=Gtk.RecentSortType.MRU,
            filter=recentFilter,
        )
        # height = min(self.MAX_HEIGHT, self.RECENT_LIMIT)
        self.box.pack_start(recentList, True, True, 2)

        # Put file chooser and load button in the same row
        hbox = Gtk.Box()
        self.box.pack_end(hbox, False, False, 2)
        # Add file chooser button
        self.fileChooser = Gtk.FileChooserButton(
            'Select model...', action=Gtk.FileChooserAction.OPEN)
        hbox.pack_start(self.fileChooser, True, True, 0)
        # Add load button
        loadButton = Gtk.Button('Load')
        hbox.pack_end(loadButton, False, False, 0)
        loadButton.connect('clicked', self.load)

    def load(self, *args):
        filename = self.fileChooser.get_filename()
        if not filename:
            item = self.recentList.get_current_item()
            if item:
                filename = item.get_uri_display()
        if filename:
            self.viewer.require.load_scene(filename)

def glib_idle_add(func):
    def new_func():
        func()
    GLib.idle_add(new_func)

class ControlPanel(Thread):
    lock = RLock()

    def __init__(self):
        Thread.__init__(self)
        self.daemon = True
        self.require = Require(self)

    def add_light(self, light):
        self._add_control('lights', LightControl, light)

    def add_material(self, material):
        self._add_control('materials', MaterialControl, material)

    def add_misc(self, viewer):
        self._add_control('misc', CameraControl, viewer)
        self._add_control('misc', FileLoader, viewer)
        self._add_control('misc', SilhouetteControl, viewer)
        self._add_control('misc', EdgesControl, viewer)

    def add_joints(self, viewer, joints):
        self._add_control('joints', JointControl, viewer, joints)

    def _add_control(self, columnName, controlClass, *args):
        @glib_idle_add
        def add_control():
            control = controlClass(*args)
            control.show_all()
            column = self.columns[columnName]
            column.pack_start(control, True, True, 2)

    def run(self):
        Gdk.threads_init()
        self.window = window = Gtk.Window(title='Control Panel')
        window.connect("delete-event", Gtk.main_quit)
        self.box = Gtk.Box(orientation=HORIZONTAL)
        self.columns = {
            'misc': Gtk.Box(orientation=VERTICAL),
            'joints': Gtk.Box(orientation=VERTICAL),
            'lights': Gtk.Box(orientation=VERTICAL),
            'materials': Gtk.Box(orientation=VERTICAL),
        }
        for name in ('misc', 'joints', 'lights', 'materials'):
            frame = Gtk.Frame(label=name.capitalize())
            frame.add(self.columns[name])
            column = Gtk.ScrolledWindow()
            column.add(frame)
            self.box.pack_start(column, True, True, 5)
        window.add(self.box)
        window.show_all()
        Gtk.main()
        Gdk.threads_quit()

    clearCondition = Condition()

    def clear(self):
        self._cleared = False

        @glib_idle_add
        def clear():
            self.clearCondition.acquire()
            for columnName in ('lights', 'materials', 'joints'):
                for child in self.columns[columnName].get_children():
                    child.destroy()
            self.clearCondition.notify_all()
            self.clearCondition.release()
            self._cleared = True


# 建立/断开连接代码
import persional
import socks
from telethon import TelegramClient, sync

def Connect():
    # 机器人id
    api_id = persional.getId()
    # 机器人Hash
    api_hash = persional.getHash()
    # 需要代理用这个
    proxy = (socks.SOCKS5, '127.0.0.1', 1080)
    # proxy = (socks.PROXY_TYPE_HTTP, '127.0.0.1', 1080)
    print("connecting...")
    # 建立连接
    client = TelegramClient('my_session', api_id=api_id, api_hash=api_hash, proxy=proxy).start()
    print("connected!")
    print(client)
    # 函数返回此时的链接
    return client

def disConnect(client):
    # 断开已经建立的连接
    client.disconnect()
    print("disConnect,Done")

if __name__ == '__main__':
    Connect()
    # print("connected!")
    disConnect(Connect())
    # print("disconnected!")


from scipy.sparse import csr_matrix
import numpy as np
import pagerank

def initp():
    #make list to build sparse transition matrix
    txt_ptspr = open('data/doc_topics.txt', 'r')
    row_ptspr = []
    col_ptspr = []
    data_ptspr = []
    for line in txt_ptspr:
        temp = line.split(' ')
        row_ptspr.append(int(temp[1]) - 1)
        col_ptspr.append(int(temp[0]) - 1)
        data_ptspr.append(1)
    dim_row = max(row_ptspr) + 1
    dim_col = max(col_ptspr) + 1
    p = csr_matrix((data_ptspr, (row_ptspr, col_ptspr)), shape=(dim_row, dim_col), dtype=np.float)
    return p

def qtspr_off():
    alpha=0.8
    beta=0.1
    gamma=0.1
    p=initp()
    M_,M_empty = pagerank.transition_matrix()# get matrix from pagerank
    M=M_.transpose()
    #M_empty=M_empty_.transpose()
    [row, col] = M_.shape
    topic_num = p.shape[0]
    off_vec = []
    for i in range(topic_num):
        p_ = np.transpose(p[i].toarray())
        p_t=(p_/p_.sum(axis=0)).squeeze()
        #p_t=p_.squeeze()      when not dividing
        p_0 = np.divide(np.ones(row), row)
        type_max = 1 << np.finfo(np.float64).nmant
        r_t = np.random.randint(0, type_max, size=row) / np.float64(type_max)
        n=1
        while n < 1000:
            r_new=(alpha*M*r_t+beta*p_t+gamma*p_0)+alpha*(M_empty*r_t)[0]
            r_t=r_new
            if pow(r_t-r_new,2).sum(axis=0)**0.5 <pow(10,-309):
                break
            n+=1
        off_vec.append(r_t)
    return off_vec

def qtspr_on():
    # calculate online ptspr
    import time
    ptspr_on_start = time.time()
    p=initp()
    row = p.shape[0]
    col = p.shape[1]
    topic = open("data/query-topic-distro.txt", 'r')
    QTSPR = []
    off_vec=qtspr_off()
    for line in topic:
        data = line.split(' ')
        online_q = np.empty((row, col))
        for i in range(2, len(data)):
            online_q[i-2] = off_vec[i-2] * float(data[i].split(':')[1])
        QTSPR.append(online_q.sum(axis=0))
    print("{} secs for qts_pagerank_online".format(round(time.time() - ptspr_on_start,4)),"/ round(time,4)")
    return QTSPR

def filewrite():
    QTSPR=qtspr_on()
    f=open('QTSPR-U2Q1.txt', 'w')
    n=0
    for i in QTSPR:
        n+=1
        f.write(str(n)+" "+str(i)+'\n')

if __name__ == "__main__":
    print("Calculating QTSPR.....")
    f=filewrite()
    print("Done!\n\n")


import logging
import os
from sysinternalsupdater.utils import download_file

class Tool():
    def __init__(self, name, last_update_date, url):
        self.name = name
        self.url = url
        self.last_update_date = last_update_date

    def __str__(self):
        return "%s updated on %s" % (self.name, self.last_update_date.strftime("%d/%m/%Y at %H:%M"))

    def is_newer(self, tool):
        return self.last_update_date > tool.last_update_date

def is_tool_updated(tool, have_tools):

    try:
        current_tool = have_tools[tool.name]
        return tool.is_newer(current_tool)
    except KeyError:
        # The tool doesn't already exist.
        pass

    return True

def get_updated_tools(tools, have_tools):

    updated_tools = {}

    for tool_name, tool in tools.items():

        is_updated = is_tool_updated(tool, have_tools)

        if is_updated:
            updated_tools[tool_name] = tool

    return updated_tools

def update_tools_record(tools_record, updated_tools):

    # Update the tools record.
    for tool_name, tool in updated_tools.items():
        tools_record[tool_name] = tool

def update_tools(tools_to_update, destination):

    updated_tools = {}

    for tool_name,tool in tools_to_update.items():
        success = download_tool(tool, destination)

        if success is True:
            updated_tools[tool_name] = tool
        else:
            logging.info("{0} could not be updated.".format(tool_name))

    return updated_tools

def download_tool(tool, destination):

    success = False
    try:
        destination = os.path.join(destination, tool.name)
        success = download_file(tool.url, destination)

    except Exception as e:
        logging.warning(str(e) + " " + tool.url)

    return success


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution:
    """
        dp in tree form.
        https://leetcode.com/problems/house-robber-iii/discuss/79330/step-by-step-tackling-of-the-problem
    """

    def rob1(self, root: TreeNode) -> int:
        dic = {}

        def helper(root):
            if not root:
                return 0
            if root in dic:
                return dic[root]

            val = 0
            if root.left:
                val += helper(root.left.left) + helper(root.left.right)
            if root.right:
                val += helper(root.right.left) + helper(root.right.right)

            val = max(val + root.val, helper(root.left) + helper(root.right))
            dic[root] = val
            return val

        return helper(root)

    def rob2(self, root: TreeNode) -> int:
        """
            Reduce space complexity.
        """
        def helper(root):
            if not root:
                return [0, 0]

            left = helper(root.left)
            right = helper(root.right)
            res = [0] * 2

            # next level of root
            res[0] = max(left) + max(right)
            # root + next of next level of root
            res[1] = root.val + left[0] + right[0]

            return res

        res = helper(root)
        return max(res)


#!usr/bin/env python
# -*- coding:utf-8 _*-
"""
@author:cugxy
@file: jsonify_util
@time: 2018/10/30
"""

from flask import jsonify, current_app

def return_msg(status, msg, data=None, log=None):
    """
    统一服务返回
    :param status:  状态 ERROR_CODE
    :param msg:     返回消息
    :param data:    服务成功时，返回数据
    :param log:     服务异常时，打印日志
    :return: dict
    """
    result = {
        "status": status,
        "msg": msg,
    }
    if data is not None:
        result["data"] = data
    if log is not None:
        current_app.logger.error(log)
    return jsonify(result)


# file will contain classes and other functionality
# in order to check and keep track of physical locations
# so that engine plugins can use them to initialize
# their repositories
import os
from uniback.dictionary.uniback_constants import PhysicalLocation as PL
from uniback.models.general import PhysicalLocation, PhysicalLocationType
from uniback.tools.local_session import LocalSession

def init_physical_location(name, address, type, concurrent_jobs=1):
    # the location where the ub folder will go
    ub_directory = os.path.join(address, PL.UB_FOLDER_NAME)
    # we check if a folder already exists for ub usage
    if os.path.exists(ub_directory):
        if not os.path.isdir(ub_directory):
            raise Exception(
                f"A file named {PL.UB_FOLDER_NAME} "
                "exists, but it's not a folder.")
    # create a new folder in case one does not already exist
    else:
        try:
            os.mkdir(ub_directory)
        except Exception:
            raise Exception("Failed to initialize UB directory")

    physical_location = PhysicalLocation(
        name=name,
        type=type,
        address=ub_directory,
        concurrent_jobs=concurrent_jobs)
    with LocalSession() as session:
        session.add(physical_location)
        session.commit()

def delete_location(id):
    with LocalSession() as session:
        session.query(PhysicalLocation).filter_by(id=id).delete()
        session.commit()

def get_location_info(id):
    with LocalSession() as session:
        location = session.query(PhysicalLocation).filter_by(id=id).first()
        if location:
            info_dict = dict(
                id=location.id,
                name=location.name,
                address=location.address,
                type=location.type,
                concurrent_jobs=location.concurrent_jobs
            )
        else:
            info_dict = dict(
                id=-1,
                name="UNDEFINED",
                address="UNDEFINED",
                type="UNDEFINED",
                concurrent_jobs=0)
        return info_dict

def set_location_info(id, info):
    with LocalSession() as session:
        location = session.query(PhysicalLocation).filter_by(id=id).first()
        if location:
            location.name = info.get('name')
            location.address = info.get('address')
            location.type = info.get('type')
            location.concurrent_jobs = info.get('concurrent_jobs')
            session.commit()

def add_location(info):
    with LocalSession() as session:
        location = (
            PhysicalLocation(
                name=info.get('name'),
                address=info.get('address'),
                type=info.get('type'),
                concurrent_jobs=info.get('concurrent_jobs')
            )
        )
        session.add(location)
        session.commit()

def get_physical_locations(get_status=False):
    with LocalSession() as session:
        physical_locations = session.query(PhysicalLocation)
        return_list = []
        for location in physical_locations:
            return_list.append(
                dict(
                    id=location.id,
                    name=location.name,
                    address=location.address,
                    type=location.type,
                    concurent_jobs=location.concurrent_jobs,
                    status=get_location_status(location.id) if get_status else ''
                )
            )
        return return_list

def get_location_status(id):
    with LocalSession() as session:
        physical_location = session.query(PhysicalLocation).filter_by(id=id).first()
        if physical_location:
            # physical_location_type = session.query(PhysicalLocationType).filter_by(id=physical_location.type).first()
            address = physical_location.address
            type = physical_location.physical_location_type.name
        else:
            address = None
            type = None

    if type == 'localhost':
        return 'Online' if os.path.exists(address) and os.path.isdir(address) else 'Offline'
    else:
        return 'Unknown Type'

def get_location_types():
    return_list = []
    with LocalSession() as session:
        types = session.query(PhysicalLocationType).order_by(PhysicalLocationType.name.desc())
        for type in types:
            return_list.append(dict(
                id=type.id,
                name=type.name,
                subtype=type.subtype,
                description=type.description
            ))
    return return_list

def get_location_type(id):
    return_dict = None
    with LocalSession() as session:
        type = session.query(PhysicalLocationType).filter_by(id=id).first()
        if type:
            return_dict = dict(
                id=type.id,
                name=type.name,
                subtype=type.subtype,
                description=type.description
            )
    return return_dict

def add_location_type(info):
    with LocalSession() as session:
        type = (
            PhysicalLocationType(
                name=info['name'],
                subtype=info['subtype'],
                description=info.get('description')
            )
        )
        session.add(type)
        session.commit()

def set_location_type(id, info):
    with LocalSession() as session:
        type = session.query(PhysicalLocationType).filter_by(id=id).first()
        if type:
            type.name = info.get('name')
            type.type = info.get('subtype')
            type.description = info.get('description')
            session.commit()

def delete_location_type(id):
    with LocalSession() as session:
        session.query(PhysicalLocationType).filter_by(id=id).delete()
        session.commit()


import serial   # 아두이노 통신 라이브러리
from time import *
from multiprocessing import Queue, Process
from import_detect import *
import Adafruit_DHT # 온습도 라이브러리
from socket import *
from xml.etree.ElementTree import Element, SubElement, ElementTree, dump    # XML 모듈
from xml.etree.ElementTree import Element, SubElement, ElementTree, dump
import xml.etree.ElementTree as ET

BAUDRATE = 9600
time_flag = 0
last_time = 0

sensor = Adafruit_DHT.DHT11
pin = '4'

clientSock = socket(AF_INET, SOCK_STREAM)
#clientSock.connect(('220.69.249.226', 4000))

# XML 형식의 온습도 문자열
def getTempHumid(temp, humid):
    xmlStr ='''
        <SECS2_XML_MESSAGE>
            <HEAD>
                <SystemByte>00001</SystemByte>
                <CMD>3</CMD>
                <Stream>2</Stream>
                <Function>42</Function>
            </HEAD>
            <BODY>
                <HCACK>0</HCACK>
                <TEMP>'''  + temp  + '''</TEMP>
                <HUMID>''' + humid + '''</HUMID>
            </BODY>
        </SECS2_XML_MESSAGE>
        '''
    return xmlStr

# 온습도 받아오는 함수
def humanity_temp():

    # 온습도 라이브러리
    humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)

    # 온습도 값이 존재할 때 -> 온습도값 리턴
    if humidity is not None and temperature is not None:
        return int(temperature), int(humidity)

    # 온습도 값이 존재하지 않을 때
    else:
        #print('Failed to get reading. Try again!')
        temperature = 0
        humidity = 0
        return temperature, humidity

# 시리얼 객체 생성(ser) - 아두이노 통신
def serial_open():
    ser = serial.Serial('/dev/ttyAMA0', baudrate = BAUDRATE)
    return ser

# 아두이노에게 데이터 전송
def command_arduino(ser, i):
    command = ["go\n", "stop\n", "rgrab\n", "fgrab\n"]

    command[i] = command[i].encode('utf-8')
    ser.write(command[i])

# 아두이노로 부터 데이터 받는 함수
def receive_arduino(ser, q):
    if ser.readable():
        data = ser.readline()

        # decode() : 바이트로 들어온 데이터 해결
        data = str(data[:-1].decode())
        q.put(data)

# 온습도 출력(10초기준)
def get_H_T():
    global time_flag
    global last_time

    if time_flag == 0:
        last_time = time()
        time_flag = 1

    if time() - last_time >= 10:

        # 온습도 받아오는 함수
        temp, humid = humanity_temp()
        time_flag = 0

        # 서버에 전송할 XML 형식의 온습도 문자열
        SendTempHumid = getTempHumid(temp, humid)
        clientSock.send(SendTempHumid.encode('utf-8'))  # 서버로 전송
        print('Temp={0}*C  Humidity={1}%'.format(temp, humid))

# 이미지 처리(카메라)
def image_process(cap, ser, q, state_flag, state_list):
    goods_x, signal, barcode = cam(cap)

    # 정상, 비정상 저장할 변수, 정상 - 1, 비정상 - 2
    global Color

    # 파랑색 제품(정상품) - 1
    if signal == 'P' and (goods_x >= 60 and goods_x <= 272):
        Color = 1
        if state_flag == "SEND_STOP":
            command_arduino(ser, 1)
            state_flag = "SEND_GRAB"
            print("P, SEND_STOP")

        if state_flag == "SEND_GRAB"  and len(barcode) > 5:
            sleep(0.01)
            command_arduino(ser, 2)
            state_flag = "WAIT"
            print("P, SEND_GRAB")

    # 빨강색 제품(불량품) - 2
    elif signal == 'F' and (goods_x >= 60 and goods_x <= 272):
        Color = 2
        if state_flag == "SEND_STOP":
            command_arduino(ser, 1)
            state_flag = "SEND_GRAB"

        if state_flag == "SEND_GRAB"  and len(barcode) > 5:
            sleep(0.01)
            command_arduino(ser, 3)
            state_flag = "WAIT"

    else:
        if q.empty() == False and signal == 'N':
            rx_data = q.get()
            print(rx_data)
            if rx_data == "complete":
                command_arduino(ser, 0)
                state_flag = "SEND_STOP"
    return state_flag

def main_process(ser, q):

    state_list = ["WAIT", "SEND_STOP", "SEND_GRAB" ]

    cap = open_cam()
    command_arduino(ser, 0)
    state_flag = state_list[1]
    q.put("start")
    sleep(1)

    while True:
        state_flag = image_process(cap, ser, q, state_flag, state_list)
        if cv.waitKey(1) & 0xFF == 27:
            break

    cap.release()
    cv.destroyAllWindows()

def serve_process(ser, q):
    while True:
        receive_arduino(ser, q)

def temp_huminity_process(q):
    factory_state = ""
    temp_state = 1

    while True:
        if q.empty() == False and temp_state == 1:
            factory_state = q.get()
            temp_state = 2

        if factory_state == "start" and temp_state == 2:
            get_H_T()
try:
    if __name__ == "__main__":
        print("start \n")
        q = Queue()
        ser = serial_open()
        p1 = Process(target = main_process, args = (ser,q))
        p2 = Process(target = serve_process, args = (ser,q))
        p3 = Process(target = temp_huminity_process, args = (q, ))
        p1.start()
        p2.start()
        p3.start()
        print("ggggg")

except KeyboardInterrupt:
    print("exit() \n")
    p1.join()
    p2.join()
    p3.join()


# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import os
import tempfile
import shutil

from nipype.testing import (assert_equal, assert_true,
                            skipif)
import nipype.interfaces.fsl.model as fsl
from nipype.interfaces.fsl import Info
from nipype.interfaces.fsl import no_fsl

tmp_infile = None
tmp_dir = None
cwd = None

@skipif(no_fsl)
def setup_infile():
    global tmp_infile, tmp_dir, cwd
    cwd = os.getcwd()
    ext = Info.output_type_to_ext(Info.output_type())
    tmp_dir = tempfile.mkdtemp()
    tmp_infile = os.path.join(tmp_dir, 'foo' + ext)
    file(tmp_infile, 'w')
    os.chdir(tmp_dir)
    return tmp_infile, tmp_dir

def teardown_infile(tmp_dir):
    os.chdir(cwd)
    shutil.rmtree(tmp_dir)

@skipif(no_fsl)
def test_MultipleRegressDesign():
    _, tp_dir = setup_infile()
    foo = fsl.MultipleRegressDesign()
    foo.inputs.regressors = dict(reg1=[1,1,1],reg2=[0.2,0.4,0.5],reg3=[1,-1,2])
    con1 = ['con1','T',['reg1','reg2'],[0.5,0.5]]
    con2 = ['con2','T',['reg3'],[1]]
    foo.inputs.contrasts = [con1,con2,['con3','F',[con1,con2]]]
    res = foo.run()
    yield assert_equal, res.outputs.design_mat, os.path.join(os.getcwd(),'design.mat')
    yield assert_equal, res.outputs.design_con, os.path.join(os.getcwd(),'design.con')
    yield assert_equal, res.outputs.design_fts, os.path.join(os.getcwd(),'design.fts')
    yield assert_equal, res.outputs.design_grp, os.path.join(os.getcwd(),'design.grp')


import h5py

def save_hdf5(filename, datasets, attrs=None, compression="gzip"):

    with h5py.File(filename, "w") as f:
        if attrs is not None:
            for attr_name, attr_data in attrs.items():
                f.attrs[attr_name] = attr_data

        for array_name, array_data in datasets.items():
            f.create_dataset(array_name, data=array_data, compression=compression)

def load_hdf5(filename):
    with h5py.File(filename, 'r') as f:
        attrs = dict(f.attrs)
        array_dict = {key:f[key][()] for key in f.keys()}

    if len(attrs) == 0:
        return array_dict
    else:
        return array_dict, attrs