import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from collections import OrderedDict
import os
import sys
sys.path.append('/home/mldp/ML_with_bigdata')
import data_utility as du
import CNN_RNN.utility

root_dir = '/home/mldp/ML_with_bigdata'
logger = CNN_RNN.utility.setlog('analysize')

def analysis_energy_effi():
	'''
	not in use anymore
	'''
	def get_energyEFFI_pd(file_path):
		data_array = du.load_array(file_path)
		energy_array = data_array[:, -149:, 2]  # (144, 1486)
		cell_num_array = data_array[:, 0, 0]

		energy_array = np.mean(energy_array, axis=1)  # 144
		data_array = np.stack((cell_num_array, energy_array), axis=-1)
		df = pd.DataFrame({
			'EE': data_array[:, 1],
			'cell_num': [int(cell_num) for cell_num in data_array[:, 0]]})
		df = df.set_index('cell_num')
		# print(df.describe())

		return df

	def plot_KDE(data_frame, title):
		# data_frame = data_frame[['ARIMA(STL)', 'LM(STL)', 'RNN(MTL)', '3D CNN(MTL)', 'CNN-RNN(*)', 'CNN-RNN(STL)', 'CNN-RNN(MTL)']]
		ax = data_frame.plot(kind='kde', title=title + ' KDE plot')
		# ax.set_xlabel('Energy efficiency')
		# ax.set_xlim(0, 0.09)
		ax.legend(loc='upper left', fontsize='large')

	def grouping_by_macro_load(EFFI_df):
		# load_groups_list = get_groups_load_list()
		# print(EFFI_df.describe())
		without_loading_data_path = os.path.join(root_dir, 'offloading/result/without_offloading_without_RL', 'all_cell_result_array_0731.npy')
		cell_result = du.load_array(without_loading_data_path)
		# cell_result = cell_result[:, :]
		macro_load_array = cell_result[:, -149:, 4]  # (144, 149)
		macro_load_array_mean = np.mean(macro_load_array, axis=1)
		macro_load_array = cell_result[:, :, 4]  # (144, 1486)
		macro_cell_num = cell_result[:, 0, 0]
		macro_load_pd = pd.DataFrame({
			'cell_num': [int(cell_num) for cell_num in macro_cell_num],
			'macro_load': macro_load_array_mean})
		macro_load_pd = macro_load_pd.set_index('cell_num')
		# print(macro_load_pd)

		EFFI_df = pd.concat((EFFI_df, macro_load_pd), axis=1)
		bins = (0, 0.3, 0.7, 1, 2, 10)
		cats_macro_load_pd = pd.cut(EFFI_df['macro_load'], bins)
		EFFI_df_group_by_macro_load = EFFI_df.groupby(cats_macro_load_pd)
		key_list = list(EFFI_df_group_by_macro_load.groups.keys())
		key_list = sorted(key_list, key=lambda x: x)
		# print(EFFI_df_group_by_macro_load.get_group(key_list[0]))
		# print(key_list)
		logger.info('\n{}'.format(EFFI_df_group_by_macro_load.count()))
		logger.info('\n{}'.format(EFFI_df_group_by_macro_load.mean()))
		# for group in EFFI_df_group_by_macro_load:
		# 	print(group[1].head(3))
		return key_list, EFFI_df_group_by_macro_load

	prediction_offloading_RL_path = os.path.join(root_dir, 'offloading/result/with_preidction_with_RL', 'all_cell_result_array_0731.npy')
	without_offloading_RL_path = os.path.join(root_dir, 'offloading/result/without_offloading_without_RL', 'all_cell_result_array_0731.npy')
	offloading_without_RL_path = os.path.join(root_dir, 'offloading/result/offloading_without_RL', 'all_cell_result_array_0731.npy')
	God_prediction_offload_RL_path = os.path.join(root_dir, 'offloading/result/RL_with_god_prediction', 'all_cell_result_array.npy')
	offloading_RL_without_prediction_path = os.path.join(root_dir, 'offloading/result/RL_without_prediction', 'all_cell_result_array_0726.npy')
	_10mins_offloading_RL_without_prediction_path = os.path.join(root_dir, 'offloading/result/RL_10mins_without_prediction', 'all_cell_result_array_0731.npy')
	offloading_q_table_path = os.path.join(root_dir, 'offloading/result/with_Q_table_without_prediction', 'all_cell_result_array_0927.npy')

	prediction_offloading_RL = get_energyEFFI_pd(prediction_offloading_RL_path)
	without_offloading_RL = get_energyEFFI_pd(without_offloading_RL_path)
	offloading_without_RL = get_energyEFFI_pd(offloading_without_RL_path)

	God_prediction_offload_RL = get_energyEFFI_pd(God_prediction_offload_RL_path)
	offloading_RL_without_prediction = get_energyEFFI_pd(offloading_RL_without_prediction_path)
	_10mins_offloading_RL_without_prediction = get_energyEFFI_pd(_10mins_offloading_RL_without_prediction_path)
	offloading_with_Q_table = get_energyEFFI_pd(offloading_q_table_path)

	EFFI_df = pd.concat((
		prediction_offloading_RL, without_offloading_RL, offloading_without_RL, _10mins_offloading_RL_without_prediction, offloading_with_Q_table),
		axis=1)
	EFFI_df.columns = ['Pre_off_RL', 'without_off_RL', 'off_without_RL', '10_mins_without_pre', 'off_Qtable']
	# df = EFFI_df[EFFI_df < 0].dropna()
	logger.info('\n {}'.format(EFFI_df.describe()))
	# EFFI_df.plot(kind='hist', subplots=True)
	# plot_KDE(EFFI_df, 'Energy efficiency')
	# plt.show()
	grouping_by_macro_load(EFFI_df[['without_off_RL', 'off_without_RL', 'Pre_off_RL', '10_mins_without_pre', 'off_Qtable']])

def get_dataframe(file_path):
	data_array = du.load_array(file_path)
	data_array = data_array[:, -149:]

	cell_num_array = data_array[:, 0, 0]
	reward_array = np.mean(data_array[:, :, 1], axis=1)
	energy_array = np.mean(data_array[:, :, 2], axis=1)
	traffic_digested_array = np.sum(data_array[:, :, 3], axis=1)
	macro_load_array = np.mean(data_array[:, :, 4], axis=1)
	small_load_array = data_array[:, :, 5]
	small_load_array[small_load_array == 0] = np.nan
	small_load_array = np.nanmean(small_load_array, axis=1)
	power_consumption_array = np.sum(data_array[:, :, 6], axis=1)
	action_array = np.mean(data_array[:, :, 7], axis=1)
	traffic_demand_array = np.sum(data_array[:, :, 8], axis=1)  # sum of data within 149
	df = pd.DataFrame({
		'cell_num': [int(cell_num) for cell_num in cell_num_array[:]],
		'reward': reward_array,
		'energy efficiency': energy_array,
		'traffic digested': traffic_digested_array,
		'macro load': macro_load_array,
		'small load': small_load_array,
		'power consumption': power_consumption_array,
		'small cell number': action_array,
		'traffic demand': traffic_demand_array})

	df = df.set_index('cell_num')
	return df

def grouping_by_macro_load(df):
	without_loading_data_path = os.path.join(root_dir, 'offloading/result/without_offloading_without_RL', 'all_cell_result_array_0731.npy')
	cell_result = du.load_array(without_loading_data_path)
	macro_load_array = cell_result[:, -149:, 4]  # (144, 1486)  # according 149
	macro_load_array_mean = np.mean(macro_load_array, axis=1)
	macro_load_array = cell_result[:, :, 4]  # (144, 1486)
	macro_cell_num = cell_result[:, 0, 0]
	macro_load_pd = pd.DataFrame({
		'cell_num': [int(cell_num) for cell_num in macro_cell_num],
		'base_macro_load': macro_load_array_mean})
	macro_load_pd = macro_load_pd.set_index('cell_num')

	df = pd.concat((df, macro_load_pd), axis=1)
	bins = (0, 0.3, 0.7, 1, 2, 10)
	# bins = (0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 2, 3, 10)
	cats_macro_load_pd = pd.cut(df['base_macro_load'], bins)
	df_group = df.groupby(cats_macro_load_pd)
	logger.debug('macro load group count:{}'.format(df_group.count()))
	key_list = list(df_group.groups.keys())
	key_list = sorted(key_list, key=lambda x: x)

	return key_list, df_group

def get_groups_load_dict():
	'''
	group each cells load into different level
	'''
	without_loading_data_path = os.path.join(root_dir, 'offloading/result/without_offloading_without_RL', 'all_cell_result_array_0731.npy')
	cell_result = du.load_array(without_loading_data_path)
	logger.debug('cell result shape:{}'.format(cell_result.shape))  # (144, 1486, 8) 144: cell_num, 1486: time sequence, 8:cell_num, reward, energy, traffic_demand, macro load, small load, power consumption
	macro_load_array = cell_result[:, -149:, 4]  # (144, 149)
	macro_load_array_mean = np.mean(macro_load_array, axis=1)

	macro_cell_num = cell_result[:, 0, 0]

	macro_load_pd = pd.DataFrame({
		'cell_num': macro_cell_num,
		'macro_load': macro_load_array_mean})
	# print(macro_load_pd)
	bins = (0, 0.3, 0.7, 1, 2, 10)
	cats_macro_load_pd = pd.cut(macro_load_pd['macro_load'], bins)
	# print(cats_macro_load_pd)
	macro_load_pd_group = macro_load_pd.groupby(cats_macro_load_pd)
	load_groups_dict = OrderedDict()
	for index, group in enumerate(macro_load_pd_group):
		# print(group[1]['cell_num'])
		group_list = [cell_num for cell_num in group[1]['cell_num']]
		# print(group_list)
		load_groups_dict[group[0]] = group_list
		# load_group_dict[group[0]] = group[1]['cell_num']
	# print(macro_load_pd_group.count())
	# for key, v in load_group_dict.items():
	return load_groups_dict

def get_column_method(method_dict, column_name):
	new_dict = OrderedDict()
	for key, method in method_dict.items():
		df = method[column_name]
		df.name = key
		new_dict[key] = df

	return new_dict

def get_column_data_frame(method_dict, column_name):
	method_dict = get_column_method(method_dict, column_name)
	df = pd.DataFrame()
	for key, method in method_dict.items():
		df = pd.concat((df, method), axis=1)
	return df

def analysis_low_load(method_dict):
	def get_low_load_group(method_dict):
		for key, method in method_dict.items():
			key_list, df_group = grouping_by_macro_load(method)
			logger.debug('low load, key:{}'.format(key_list[0]))
			method_dict[key] = df_group.get_group(key_list[0])
		return method_dict

	def energy_effi(energy_effi_dict):
		energy_effi_df = pd.DataFrame()
		for key, method in energy_effi_dict.items():
			# print(key)
			# print(method.describe())
			# print(method)
			energy_effi_df = pd.concat((energy_effi_df, method), axis=1)

		energy_effi_mean_df = energy_effi_df.mean()
		ax = energy_effi_mean_df.plot(kind='bar', alpha=0.7, rot=45, width=0.2, color=['red', 'green', 'blue', 'cyan', 'magenta'])

	method_dict = get_low_load_group(method_dict)
	energy_effi_dict = get_column_method(method_dict, 'energy efficiency')

	energy_effi(energy_effi_dict)

def analysis_origin_macro_load_group(method_dict):
	traffic_demand_df = get_column_data_frame(method_dict, 'traffic demand')['Without offloading']
	origin_macro_load_df = get_column_data_frame(method_dict, 'macro load')['Without offloading']

	key_list, traffic_demand_df_group = grouping_by_macro_load(traffic_demand_df)
	key_list, origin_macro_load_group = grouping_by_macro_load(origin_macro_load_df)
	mean_traffic_demand = traffic_demand_df_group.mean().drop('base_macro_load', 1) / 1000  # GB
	mean_origin_macro_load = origin_macro_load_group.mean().drop('base_macro_load', 1)
	logger.info('\n{}'.format(mean_traffic_demand))
	logger.info('\n{}'.format(mean_origin_macro_load))

def plot_energy_efficiency(method_dict):
	energy_effi_df = get_column_data_frame(method_dict, 'energy efficiency')
	logger.info('\n{}'.format(energy_effi_df.describe()))
	# print(energy_effi_df.head(5))
	key_list, energy_effi_df_group = grouping_by_macro_load(energy_effi_df)
	mean_energy_effi = energy_effi_df_group.mean()
	mean_energy_effi = mean_energy_effi.drop('base_macro_load', 1)
	# # print(mean_energy_effi.columns)
	mean_energy_effi.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	mean_energy_effi.index.name = 'Macro load rate'
	ax = mean_energy_effi.plot(kind='bar', alpha=0.7, rot=0, width=0.8)
	plt.legend(loc='upper left', fontsize='large')
	plt.title('Energy efficiency comparison')
	plt.xlabel('Macro load rate', size=15)
	plt.ylabel('Energy efficiency (Mb/Joule)', size=15)

def plot_power_consumption(method_dict):
	power_consumption_df = get_column_data_frame(method_dict, 'power consumption')
	logger.info('\n{}'.format(power_consumption_df.mean() / 1000000))  # M joule
	key_list, power_df_group = grouping_by_macro_load(power_consumption_df)
	mean_power = power_df_group.mean() / 1000000
	mean_power = mean_power.drop('base_macro_load', 1)
	mean_power.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	mean_power.index.name = 'Macro load rate'
	mean_power.plot(kind='bar', alpha=0.7, rot=0, width=0.8, figsize=(8, 6))
	plt.legend(loc='upper left', fontsize='large')
	plt.title('Energy consumption comparison')
	plt.xlabel('Macro load rate', size=15)
	plt.ylabel('Average energy consumption (MJoule)', size=15)  # per cell

def plot_energy_saving(method_dict):
	fig = plt.figure()
	ax_1 = fig.add_subplot(1, 1, 1)
	# ax_2 = fig.add_subplot(2, 1, 2)
	power_consumption_df = get_column_data_frame(method_dict, 'power consumption')
	key_list, power_df_group = grouping_by_macro_load(power_consumption_df)
	mean_power = power_df_group.mean() / 1000000  # M joule
	mean_power = mean_power.drop('base_macro_load', 1)
	mean_power.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	mean_power.index.name = 'Macro load rate'
	power_saving_rate = mean_power.apply(lambda x: (x['Without offloading'] - x[['Offloading without prefiction', 'Offloading with prediction']]) * 100 / x['Without offloading'], axis=1)
	# power_saving_rate = sum_power.apply(lambda x: (x['Offloading without prefiction'] - x['Offloading with prediction']) * 100 / x['Offloading without prefiction'], axis=1)
	# print(power_saving_rate)
	# power_save_rate_perdiction = (sum_power.loc[:, 'Without offloading'] - sum_power.loc[:, 'Offloading without prefiction']) / sum_power.loc[:, 'Without offloading']
	# power_save_rate_wihtou_perdiction = (sum_power.loc[:, 'Without offloading'] - sum_power.loc[:, 'Offloading without prefiction']) / sum_power.loc[:, 'Without offloading']
	power_saving_rate.plot(ax=ax_1, kind='bar', alpha=0.7, rot=0, width=0.8, color=['C2', 'C3'])
	ax_1.legend(loc='upper left', fontsize='large')
	ax_1.set_title('Power saving rate comparison')
	ax_1.set_xlabel('Macro load rate', size=15)
	ax_1.set_ylabel('Power saving rate (%)', size=15)

	# power_saving = sum_power.apply(lambda x: x['Without offloading'] - x[['Offloading without prefiction', 'Offloading with prediction']], axis=1)
	# print(power_saving)
	# power_saving.plot(ax=ax_2, kind='bar', alpha=0.7, rot=0)
	# ax_2.legend(loc='upper left', fontsize='large')
	# ax_2.set_title('Power saving comparison')
	# ax_2.set_xlabel('Macro load rate', size=15)
	# ax_2.set_ylabel('Power saving (MW)', size=15)

def plot_cell_load(method_dict):
	fig = plt.figure('cell loading rate', figsize=(10, 6))
	ax_1 = fig.add_subplot(3, 1, 1)
	ax_2 = fig.add_subplot(3, 1, 2)
	ax_3 = fig.add_subplot(3, 1, 3)

	macro_load_df = get_column_data_frame(method_dict, 'macro load')
	key_list, macro_load_df_group = grouping_by_macro_load(macro_load_df)
	macro_load = macro_load_df_group.mean()
	macro_load = macro_load.drop('base_macro_load', 1)
	macro_load.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	macro_load.index.name = 'Macro load rate'
	macro_load.plot(ax=ax_1, kind='bar', alpha=0.7, rot=0, width=0.8)
	ax_1.legend(loc='upper left', fontsize='x-large')
	ax_1.set_title('Cell loading comparison', size=18)
	ax_1.set_xlabel('', size=0)
	ax_1.set_ylim(0., 3)

	ax_1.set_ylabel('Macro cell load rate \n after offloading', size=14)

	small_load_df = get_column_data_frame(method_dict, 'small load')
	key_list, small_load_df_group = grouping_by_macro_load(small_load_df)
	small_load = small_load_df_group.mean()
	small_load = small_load.drop('base_macro_load', 1)
	small_load.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']

	small_load.index.name = 'Macro loading rate'
	# small_load = small_load[['Offloading without prefiction', 'Offloading with prediction']]
	small_load.plot(ax=ax_2, kind='bar', alpha=0.7, rot=0, width=0.8)
	ax_2.legend(loc='upper left', fontsize='large')
	ax_2.set_ylim(0., 1)
	# ax_2.set_title('Small cell load comparison')
	# ax_2.set_xlabel('Macro cell load rate', size=12)
	ax_2.set_xlabel('')
	ax_2.set_ylabel('Small cell loading rate', size=14)

	small_cell_num_df = get_column_data_frame(method_dict, 'small cell number')
	key_list, small_num_df_group = grouping_by_macro_load(small_cell_num_df)
	small_num = small_num_df_group.mean()
	small_num = small_num.drop('base_macro_load', 1)
	small_num.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']

	small_num.index.name = 'Macro load rate'
	# small_load = small_load[['Offloading without prefiction', 'Offloading with prediction']]
	small_num.plot(ax=ax_3, kind='bar', alpha=0.7, rot=0, width=0.8)
	ax_3.legend(loc='upper left', fontsize='large')
	# ax_2.set_title('Small cell load comparison')
	ax_3.set_xlabel('Macro cell loading rate', size=16)
	ax_3.set_ylabel('Small cell average number', size=14)

	ax_2.legend_.remove()
	ax_3.legend_.remove()

	for p in ax_1.patches:
		string = '{:.2f}'.format(float(p.get_height()))
		ax_1.annotate(string, xy=(p.get_x(), p.get_height()))
	for p in ax_2.patches:
		string = '{:.2f}'.format(float(p.get_height()))
		ax_2.annotate(string, xy=(p.get_x(), p.get_height()))
	for p in ax_3.patches:
		string = '{:.2f}'.format(float(p.get_height()))
		ax_3.annotate(string, xy=(p.get_x(), p.get_height()))

def plot_traffic_digested(method_dict):
	traffic_digested_df = get_column_data_frame(method_dict, 'traffic digested')
	logger.info('\n{}'.format(traffic_digested_df.mean() / 1000))  # Gb
	# print(energy_effi_df.head(5))
	key_list, traffic_df_group = grouping_by_macro_load(traffic_digested_df)
	mean_traffic = traffic_df_group.mean() / 1000  # Gb
	mean_traffic = mean_traffic.drop('base_macro_load', 1)
	# # print(mean_energy_effi.columns)
	mean_traffic.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	mean_traffic.index.name = 'Macro load rate'
	ax = mean_traffic.plot(kind='bar', alpha=0.7, rot=0, width=0.8)
	plt.legend(loc='upper left', fontsize='large')
	plt.title('Internet traffic digestion comparison')
	plt.xlabel('Macro loading rate', size=15)
	plt.ylabel('Internet traffic (Gb)', size=15)

def plot_energy_efficiency_energy_consump_traffic_throughput(method_dict):
	traffic_digested_df = get_column_data_frame(method_dict, 'traffic digested')
	energy_consumption_df = get_column_data_frame(method_dict, 'power consumption')
	energy_effi_df = get_column_data_frame(method_dict, 'energy efficiency')

	key_list, traffic_throughput_df_group = grouping_by_macro_load(traffic_digested_df)
	key_list, energy_consumption_df_group = grouping_by_macro_load(energy_consumption_df)
	key_list, energy_effi_df_group = grouping_by_macro_load(energy_effi_df)

	mean_traffic_throughput = traffic_throughput_df_group.mean().drop('base_macro_load', 1) / 1000  # Gb
	mean_energy_consumption = energy_consumption_df_group.mean().drop('base_macro_load', 1) / 1000000  # M joule
	mean_energy_effi = energy_effi_df_group.mean().drop('base_macro_load', 1)

	mean_traffic_throughput.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	mean_traffic_throughput.index.name = 'Macro loading rate'
	mean_energy_consumption.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	mean_energy_consumption.index.name = 'Macro loading rate'
	mean_energy_effi.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
	mean_energy_effi.index.name = 'Macro loading rate'

	fig = plt.figure('Energy power EE', figsize=(10, 6))
	ax_1 = fig.add_subplot(3, 1, 1)
	ax_2 = fig.add_subplot(3, 1, 2)
	ax_3 = fig.add_subplot(3, 1, 3)
	mean_energy_effi.plot(ax=ax_1, kind='bar', alpha=0.7, rot=0, width=0.8)
	mean_energy_consumption.plot(ax=ax_2, kind='bar', alpha=0.7, rot=0, width=0.8)
	mean_traffic_throughput.plot(ax=ax_3, kind='bar', alpha=0.7, rot=0, width=0.8)

	ax_1.legend(loc='upper left', fontsize='x-large')
	ax_1.set_title('Energy efficiency comparison', size=18)
	ax_1.set_xlabel('', size=0)
	ax_1.set_ylabel('Energy efficiency\n(Mb/Joule)', size=14)

	ax_2.legend(loc='upper left', fontsize='large')
	ax_2.set_xlabel('')
	ax_2.set_ylabel('Energy comsumption\n(M-Joule)', size=14)

	ax_3.legend(loc='upper left', fontsize='large')
	ax_3.set_xlabel('Macro cell loading rate group', size=17)
	ax_3.set_ylabel('Mobile traffic throughput\n(Gb)', size=14)

	ax_2.legend_.remove()
	ax_3.legend_.remove()

	for p in ax_1.patches:
		string = '{:.3f}'.format(float(p.get_height()))
		ax_1.annotate(string, xy=(p.get_x(), p.get_height()))
	for p in ax_2.patches:
		string = '{:.0f}'.format(float(p.get_height()))
		ax_2.annotate(string, xy=(p.get_x(), p.get_height()))
	for p in ax_3.patches:
		string = '{:.0f}'.format(float(p.get_height()))
		ax_3.annotate(string, xy=(p.get_x(), p.get_height()))

def ananlyzie_each_method():
	prediction_offloading_RL_path = os.path.join(root_dir, 'offloading/result/with_preidction_with_RL', 'all_cell_result_array_0731.npy')
	without_offloading_RL_path = os.path.join(root_dir, 'offloading/result/without_offloading_without_RL', 'all_cell_result_array_0731.npy')
	offloading_without_RL_path = os.path.join(root_dir, 'offloading/result/offloading_without_RL', 'all_cell_result_array_0731.npy')
	# God_prediction_offload_RL_path = os.path.join(root_dir, 'offloading/result/RL_with_god_prediction', 'all_cell_result_array.npy')
	# offloading_RL_without_prediction_path = os.path.join(root_dir, 'offloading/result/RL_without_prediction', 'all_cell_result_array_0726.npy')
	_10mins_offloading_RL_without_prediction_path = os.path.join(root_dir, 'offloading/result/RL_10mins_without_prediction', 'all_cell_result_array_0731.npy')
	offloading_q_table_path = os.path.join(root_dir, 'offloading/result/with_Q_table_without_prediction', 'all_cell_result_array_0928.npy')

	prediction_offloading_RL = get_dataframe(prediction_offloading_RL_path)
	without_offloading_RL = get_dataframe(without_offloading_RL_path)
	offloading_without_RL = get_dataframe(offloading_without_RL_path)
	_10mins_offloading_RL_without_prediction = get_dataframe(_10mins_offloading_RL_without_prediction_path)
	offloading_with_Q_table = get_dataframe(offloading_q_table_path)

	method_dict = OrderedDict()
	method_dict['Without offloading'] = without_offloading_RL
	method_dict['Offloading without RL'] = offloading_without_RL
	method_dict['Offloading with Q table'] = offloading_with_Q_table
	method_dict['Offloading without prediction (DQN)'] = _10mins_offloading_RL_without_prediction
	method_dict['Offloading with prediction (DQN)'] = prediction_offloading_RL

	# analysis_origin_macro_load_group(method_dict)
	plot_energy_efficiency(method_dict)
	# plot_power_consumption(method_dict)
	# plot_traffic_digested(method_dict)
	# plot_energy_saving(method_dict)

	''' the two main plot function'''
	# plot_cell_load(method_dict)
	# plot_energy_efficiency_energy_consump_traffic_throughput(method_dict)
	# analysis_low_load(method_dict)
	plt.show()

def comparison_predicion_and_without_prediction():
	def get_array(file_path):
		data_array = du.load_array(file_path)
		data_array = data_array[:, -149:, (0, 4)]
		return data_array
	prediction_offloading_RL_path = os.path.join(root_dir, 'offloading/result/with_preidction_with_RL', 'all_cell_result_array_0731.npy')
	_10mins_offloading_RL_without_prediction_path = os.path.join(root_dir, 'offloading/result/RL_10mins_without_prediction', 'all_cell_result_array_0731.npy')
	without_offloading_RL_path = os.path.join(root_dir, 'offloading/result/without_offloading_without_RL', 'all_cell_result_array_0731.npy')
	ARIMA_prediction_offloading_RL_path = os.path.join(root_dir, 'offloading/result/RL_real_ARIMA_prediction', 'all_cell_result_array_0805.npy')
	offloading_with_Q_table_path = os.path.join(root_dir, 'offloading/result/with_Q_table_without_prediction', 'all_cell_result_array_0927.npy')

	prediction_offloading_RL = get_array(prediction_offloading_RL_path)
	_10mins_offloading_RL_without_prediction = get_array(_10mins_offloading_RL_without_prediction_path)
	without_offloading_RL = get_array(without_offloading_RL_path)
	ARIMA_prediction_offloading_RL = get_array(ARIMA_prediction_offloading_RL_path)
	offloading_with_Q_table = get_array(offloading_with_Q_table_path)

	def overload_comparison():
		def count_overload(cell_list, load_array):
			overload_count = 0
			for cell in cell_list:
				for cell_index in range(load_array.shape[0]):
					if cell == load_array[cell_index, 0, 0]:
						cell_load_array = load_array[cell_index, :, 1]
						count = (cell_load_array > 1).sum()
						# print(cell, count)
						# count = (cell_load_array > 200).sum()
						overload_count += count

			return overload_count
		load_groups_dict = get_groups_load_dict()
		for key, group_list in load_groups_dict.items():
			without_prediction_overload_count = count_overload(group_list, _10mins_offloading_RL_without_prediction)
			prediction_overload_count = count_overload(group_list, prediction_offloading_RL)
			print('{} overload times: without prediction:{} with prediction:{}'.format(key, without_prediction_overload_count, prediction_overload_count))
		# without_offloading_RL_count = count_overload(group_list, without_offloading_RL)
		# print(without_offloading_RL_count)

	def macro_load_distribution():
		def group_array(load_groups_dict, load_array):
			'''
			find cell number from load array in with load group by load_groups_dict
			'''
			for key, load_cell_list in load_groups_dict.items():  # loop load_groups_dict
				cell_list = []
				for load_cell_index in range(load_array.shape[0]):  # loop load array cell
					cell_num = load_array[load_cell_index, 0, 0]
					if cell_num in load_cell_list:
						cell_list.append(load_array[load_cell_index, :, 1])

				group_cell_load_array = np.stack(cell_list)
				yield group_cell_load_array

		def plot_count_bar(data_frame):
			'''plot each group's new loading rate after offloading'''

			# bins = np.arange(0, 1.1, 0.1, dtype=np.float)
			# print(data_frame)
			bins = (0, 0.3, 0.7, 1, 2, 10)
			cats_Prediction_pd = pd.cut(data_frame['Prediction'], bins)
			cats_non_Prediction_pd = pd.cut(data_frame['Non prediction'], bins)
			cats_ARIMA_prediction_pd = pd.cut(data_frame['ARIMA prediction'], bins)
			cats_Q_table_pd = pd.cut(data_frame['Q table'], bins)

			Prediction_group = data_frame['Prediction'].groupby(cats_Prediction_pd)
			non_prediction_group = data_frame['Non prediction'].groupby(cats_non_Prediction_pd)
			ARIMA_prediction_group = data_frame['ARIMA prediction'].groupby(cats_ARIMA_prediction_pd)
			Q_table_group = data_frame['Q table'].groupby(cats_Q_table_pd)

			# print(non_prediction_group.mean())
			Prediction_group_counts = Prediction_group.count()
			non_prediction_counts = non_prediction_group.count()
			ARIMA_prediction_counts = ARIMA_prediction_group.count()
			Q_table_counts = Q_table_group.count()
			# print(Prediction_group_counts)
			count_df = pd.concat([Q_table_counts, non_prediction_counts, Prediction_group_counts, ARIMA_prediction_counts], axis=1)
			count_df.columns = ['Offloading with Q table', 'Offloading without prediction (DQN)', 'Offloading with CNN-RNN prediction (DQN)', 'Offloading with ARIMA prediction (DQN)']
			count_df.index = ['0~0.3', '0.3~0.7', '0.7~1', '1~2', '2~10']
			fig = plt.figure(figsize=(15, 6))
			ax = fig.add_subplot(1, 1, 1)
			ax = count_df.plot(ax=ax, kind='bar', color=['C2', 'C3', 'C4', 'C5'], alpha=0.7, rot=0, width=0.8)
			ax.set_title('Macro cell loading rate distributinon after offloading', size=18)
			ax.set_xlabel('Macro cell loading rate group', size=14)
			ax.set_ylabel('Times', size=14)
			ax.legend(loc='upper right', fontsize='large')
			for p in ax.patches:
				string = '{}'.format(int(p.get_height()))
				ax.annotate(string, xy=(p.get_x(), p.get_height()))

		load_groups_dict = get_groups_load_dict()

		""" non prediction case """
		offloading_with_Q_table_generator = group_array(load_groups_dict, offloading_with_Q_table)
		non_prediction_group_load_array_generator = group_array(load_groups_dict, _10mins_offloading_RL_without_prediction)

		"""prediction case"""
		ARIMA_prediction_group_load_array_generator = group_array(load_groups_dict, ARIMA_prediction_offloading_RL)
		prediction_group_load_array_generator = group_array(load_groups_dict, prediction_offloading_RL)

		'''from low loading to higher loading'''
		for group_prediction_load_array in prediction_group_load_array_generator:
			non_prediction_group_load_array = next(non_prediction_group_load_array_generator)
			ARIMA_prediction_group_load_array = next(ARIMA_prediction_group_load_array_generator)
			Q_table_group_load_array = next(offloading_with_Q_table_generator)

			group_prediction_load_array = group_prediction_load_array.reshape((-1, ))
			non_prediction_group_load_array = non_prediction_group_load_array.reshape((-1, ))
			ARIMA_prediction_group_load_array = ARIMA_prediction_group_load_array.reshape((-1, ))
			Q_table_group_load_array = Q_table_group_load_array.reshape((-1, ))

			macro_load_df = pd.DataFrame({
				'Prediction': group_prediction_load_array,
				'Non prediction': non_prediction_group_load_array,
				'ARIMA prediction': ARIMA_prediction_group_load_array,
				'Q table': Q_table_group_load_array})
			# macro_load_df.plot(kind='line')
			# print(macro_load_df.describe())
			'''plot each group's new loading rate after offloading'''
			plot_count_bar(macro_load_df)

		plt.show()

	overload_comparison()
	macro_load_distribution()

if __name__ == "__main__":
	analysis_energy_effi()
	# ananlyzie_each_method()
	# comparison_predicion_and_without_prediction()


import datetime
import requests

common = "https://lonqie.github.io/SchaleDB/data/common.json"
localization = "https://lonqie.github.io/SchaleDB/data/cn/localization.json"
raids = "https://lonqie.github.io/SchaleDB/data/cn/raids.min.json"
student_cn = "https://lonqie.github.io/SchaleDB/data/cn/students.min.json"
student_jp = "https://lonqie.github.io/SchaleDB/data/jp/students.min.json"

def get_item(dict,key,value):
    for item in dict:
        if item[key] == value:
            return item
    return None

def get_json_data(url):
    for i in range(3):
        try:
            res = requests.get(url,timeout=15)
            if res.status_code == 200:
                return res.json()
        except:
            continue
    return None

def extract_calendar_data(server):
    event_list = []

    common_data = get_json_data(common)
    student_data = get_json_data(student_cn)
    localization_data = get_json_data(localization)
    raid_data = get_json_data(raids)
    if common_data == None or student_data == None or localization_data == None or raid_data == None:
        return None

    if server == "jp":
        data = common_data["regions"][0]
    else:
        data = common_data["regions"][1]

    #gacha
    for gacha in data["current_gacha"]:
        characters = gacha["characters"]
        for character in characters:
            stu_info = get_item(student_data,"Id",character)
            title = "本期卡池: " + stu_info["Name"]
            start_time = datetime.datetime.fromtimestamp(gacha["start"]).strftime("%Y/%m/%d %H:%M")
            end_time = datetime.datetime.fromtimestamp(gacha["end"]).strftime("%Y/%m/%d %H:%M")
            event_list.append({'title': title, 'start': start_time, 'end': end_time})

    #event
    for event in data["current_events"]:
        event_rerun = ""
        event_id = event["event"]
        # 复刻活动似乎是在原id前面加上10
        if event_id > 1000:
            event_id = str(event_id)[2:]
            event_rerun = "(复刻)"
        event_name = localization_data["EventName"][str(event_id)] + event_rerun
        start_time = datetime.datetime.fromtimestamp(event["start"]).strftime("%Y/%m/%d %H:%M")
        end_time = datetime.datetime.fromtimestamp(event["end"]).strftime("%Y/%m/%d %H:%M")
        event_list.append({'title': event_name, 'start': start_time, 'end': end_time})

    #raid
    for raid in data["current_raid"]:
        raid_id = raid["raid"]
        title = ""
        #总力
        if raid_id < 999:
            raid_info = get_item(raid_data["Raid"], "Id", raid_id)
            title = "总力战: " + raid_info["Name"]
            if "terrain" in raid:
                title = title + f'({raid["terrain"]})'
        #演习
        if raid_id > 999 and raid_id < 99999:
            dungeon_types = {"Shooting":"射击","Defense":"防御","Destruction":"突破"}
            raid_info = get_item(raid_data["TimeAttack"], "Id", raid_id)
            title = raid_info["DungeonType"]
            if raid_info["DungeonType"] in dungeon_types:
                title = dungeon_types[raid_info["DungeonType"]]
            title += "演习"
            if "Terrain" in raid_info:
                title = title + f'({raid_info["Terrain"]})'
        #世界boss
        if raid_id > 800000 and raid_id < 900000:
            raid_info = get_item(raid_data["WorldRaid"], "Id", raid_id)
            title = raid_info["Name"]

        if title != "":
            start_time = datetime.datetime.fromtimestamp(raid["start"]).strftime("%Y/%m/%d %H:%M")
            end_time = datetime.datetime.fromtimestamp(raid["end"]).strftime("%Y/%m/%d %H:%M")
            event_list.append({'title': title, 'start': start_time, 'end': end_time})
    return event_list

def transform_schaledb_calendar(server):
    data = extract_calendar_data(server)
    return data


"""
#############################################
#
# planning.py
#
# object for planning call
#
#############################################
"""
import os
from datetime import date, time, datetime, timedelta
from calendar_view.calendar import Calendar
from calendar_view.core.event import Event, EventStyles, EventStyle
from calendar_view.core import data
import pandas as pd
import streamlit as st
from system import Call
from helpers import Configuration, Endpoint

class Planning:
    """Planning class.
    Everything about planning object.
    """

    def __init__(self):
        """Init Planning object."""
        self.endpoint = Endpoint.PLAS
        self.json_pd = None
        self.label = "planning"
        self.req_code = 0
        self.get_data()

        # Put/Post planning
        self.id = 0
        self.name_pla = ""
        self.type_pla = ""
        self.location_pla = "Mars"
        self.date_begins_pla = date(1970, 1, 1)
        self.date_end_pla = date(1970, 1, 1)
        self.hour_begins_pla = time(0, 0, 0)
        self.hour_end_pla = time(23, 59, 59)

        # Calendar view
        current_dir = os.path.dirname(os.path.abspath(__file__))
        self.cal_pla = "{}/{}/".format(current_dir, "../../resources") + "cal_pla.png"

    def get_data(self) -> True:
        """Get planning data."""
        get_req = Call()
        to_return = get_req.get_data(self)
        self.req_code = get_req.status_code
        self.json_pd = get_req.response
        self.json_pd["date_begins"] = pd.to_datetime(self.json_pd["date_begins"])
        self.json_pd["date_begins"] = self.json_pd["date_begins"].dt.strftime(
            "%Y-%m-%d"
        )
        self.json_pd["date_end"] = pd.to_datetime(self.json_pd["date_end"])
        self.json_pd["date_end"] = self.json_pd["date_end"].dt.strftime("%Y-%m-%d")
        self.json_pd["hour_begins"] = pd.to_datetime(self.json_pd["hour_begins"])
        self.json_pd["hour_begins"] = self.json_pd["hour_begins"].dt.strftime("%H:%M")
        self.json_pd["hour_end"] = pd.to_datetime(self.json_pd["hour_end"])
        self.json_pd["hour_end"] = self.json_pd["hour_end"].dt.strftime("%H:%M")
        return to_return

    def post_put_data(self, protocol: str):
        """Post or put planning data.

        Args:
            protocol: protocol to use, can be `post` or `put`
        """
        post_put_req = Call()
        payload = {
            "name": f"{self.name_pla}",
            "type": f"{self.type_pla}",
            "location": f"{self.location_pla}",
            "date_begins": f"{self.date_begins_pla}",
            "date_end": f"{self.date_end_pla}",
            "hour_begins": f"{self.hour_begins_pla}",
            "hour_end": f"{self.hour_end_pla}",
        }
        post_put_req.post_put_data(obj=self, payload=payload, protocol=protocol)
        self.req_code = post_put_req.status_code

    @staticmethod
    def color_pla(type_pla: str) -> EventStyle:
        """Add color to planning.

        Args:
            type_pla: type of the planning
        Return:
            color_pla_conf: color style for the planning of type EventStyle
        """
        color_pla_conf = EventStyles.GRAY
        for key_name, value_color in Configuration().planning_types.items():
            if type_pla == key_name:
                if value_color == "red":
                    color_pla_conf = EventStyles.RED
                elif value_color == "blue":
                    color_pla_conf = EventStyles.BLUE
                elif value_color == "green":
                    color_pla_conf = EventStyles.GREEN
        return color_pla_conf

    def view_planning(self):
        """View all Planning."""
        st.write("### View all plannings")

        if self.json_pd is None:
            st.warning("Data is empty !")
            return

        def gen_cal():
            """Function to gen the calendar."""
            date_selected = st.session_state["begin_cal"]

            if date_selected.weekday() != 0:
                # Increment date_selected's date with 1 week to get the previous Monday
                prev_monday = date_selected + timedelta(
                    days=-date_selected.weekday(), weeks=0
                )
            else:
                prev_monday = date_selected

            config = data.CalendarConfig(
                lang="en",
                title="Plannings",
                dates=f"{prev_monday} - {st.session_state['end_cal']}",
                show_year=True,
                mode=None,
                legend=False,
            )
            events = []
            for name, type_pla, date_begins, date_end, hour_begins, hour_end in zip(
                self.json_pd["name"],
                self.json_pd["type"],
                self.json_pd["date_begins"],
                self.json_pd["date_end"],
                self.json_pd["hour_begins"],
                self.json_pd["hour_end"],
            ):
                type_color = Planning.color_pla(type_pla)
                if date_begins != date_end:
                    events.append(
                        Event(
                            title=name,
                            day=date_begins,
                            start=hour_begins,
                            end=time(23, 59),
                            style=type_color,
                        )
                    )
                    events.append(
                        Event(
                            title=name,
                            day=date_end,
                            start=time(0, 00),
                            end=hour_end,
                            style=type_color,
                        )
                    )
                else:
                    events.append(
                        Event(
                            name,
                            day=date_begins,
                            start=hour_begins,
                            end=hour_end,
                            style=type_color,
                        )
                    )

            data.validate_config(config)
            data.validate_events(events, config)
            calendar = Calendar.build(config)
            calendar.add_events(events)
            calendar.save(self.cal_pla)

        st.date_input("Date begins", value=date.today(), key="begin_cal")
        st.date_input(
            "Date end", value=(date.today() + timedelta(days=6)), key="end_cal"
        )

        if (
            st.checkbox("View plannings")
            and st.session_state["begin_cal"] != st.session_state["end_cal"]
            and st.session_state["begin_cal"] < st.session_state["end_cal"]
        ):
            gen_cal()
            st.image(self.cal_pla)

    def list_plannings(self):
        """List plannings."""
        st.write("## List of Plannings !")

        if self.json_pd is None:
            st.warning("Data is empty !")
            return

        s_filter = st.checkbox("Search filters", False)
        if s_filter:
            f_col, _, _ = st.columns([1, 1, 5])
            fname_filter = f_col.checkbox("Name", True)
            fold_filter = f_col.checkbox("Past", False)

            selected_name = st.selectbox(
                "Select name :",
                self.json_pd["name"],
                disabled=not fname_filter,
            )

            s_fname = selected_name if fname_filter else ""

            if fname_filter and fold_filter:
                selected_rows = self.json_pd.loc[
                    (self.json_pd["name"] == s_fname)
                    & (
                        pd.to_datetime(self.json_pd["date_end"])
                        <= pd.to_datetime("today")
                    )
                ]
            elif fname_filter or fold_filter:
                selected_rows = self.json_pd.loc[
                    (self.json_pd["name"] == s_fname)
                    | (
                        pd.to_datetime(self.json_pd["date_end"])
                        <= pd.to_datetime("today")
                    )
                ]
            else:
                selected_rows = self.json_pd.loc[
                    (
                        pd.to_datetime(self.json_pd["date_end"])
                        >= (pd.to_datetime("today") - pd.Timedelta("1 days"))
                    )
                ]

            data_pla = selected_rows
        else:
            data_pla = self.json_pd.loc[
                (
                    pd.to_datetime(self.json_pd["date_end"])
                    >= (pd.to_datetime("today") - pd.Timedelta("1 days"))
                )
            ]

        st.write(data_pla)

    def update_planning(self):
        """Update a planning to the api."""
        st.write("## Update a Planning")
        st.markdown(
            "If you want to auto complete most of the item, selected the `id` of the planning."
        )

        if self.json_pd is None:
            st.warning("Data is empty !")
            return

        up_pla = st.checkbox("Update a Planning ?", False)

        if up_pla:
            selected_indices = st.selectbox("Select rows:", self.json_pd.index)

            with st.form("Update", clear_on_submit=False):
                self.id = selected_indices
                self.name_pla = st.text_input(
                    "Name", self.json_pd.loc[selected_indices, "name"]
                )
                self.type_pla = st.text_input(
                    "Type", self.json_pd.loc[selected_indices, "type"]
                )
                self.location_pla = st.text_input(
                    "Location", self.json_pd.loc[selected_indices, "location"]
                )
                date_begins_format = datetime.strptime(
                    self.json_pd.loc[selected_indices, "date_begins"], "%Y-%m-%d"
                )
                self.date_begins_pla = st.date_input("Date begins", date_begins_format)
                date_end_format = datetime.strptime(
                    self.json_pd.loc[selected_indices, "date_end"], "%Y-%m-%d"
                )
                self.date_end_pla = st.date_input("Date end", date_end_format)
                hour_begins_format = datetime.strptime(
                    self.json_pd.loc[selected_indices, "hour_begins"], "%H:%M"
                )
                self.hour_begins_pla = st.time_input("Hour begins", hour_begins_format)
                hour_end_format = datetime.strptime(
                    self.json_pd.loc[selected_indices, "hour_end"], "%H:%M"
                )
                self.hour_end_pla = st.time_input("Hour end", hour_end_format)

                submitted = st.form_submit_button("Submit")
                if submitted:
                    if self.id != 0 and len(self.name_pla) > 0:
                        self.post_put_data(protocol="put")
                        if self.req_code == 200:
                            st.success("Planning updated ✌")
                        else:
                            error_up_eve = (
                                "Update planning : " + str(self.req_code)
                                if self.req_code != 200
                                else "Update planning : OK"
                            )
                            st.error(f"{error_up_eve}")
                    else:
                        st.warning(
                            """
                            You forget some info...

                            All fields are **MANDATORY**.
                            """
                        )

    def new_planning(self):
        """Create a new planning."""
        st.write("## Create a new Planning")

        with st.form("New planning", clear_on_submit=False):
            self.name_pla = st.text_input("Name")
            self.type_pla = st.selectbox(
                "Type", ("permanence", "event", "meeting", "other")
            )
            self.location_pla = st.text_input("Location")
            self.date_begins_pla = st.date_input("Date begins", value=date.today())
            self.date_end_pla = st.date_input("Date end", value=date.today())
            self.hour_begins_pla = st.time_input("Hour begins")
            self.hour_end_pla = st.time_input("Hour end")

            print("End : ", self.hour_end_pla)

            submitted = st.form_submit_button("Submit")
            if submitted:
                if len(self.name_pla) > 0 and len(self.location_pla) > 0:
                    # Post planning
                    self.post_put_data(protocol="post")

                    if self.req_code == 200:
                        st.success("Planning added ✌")
                    else:
                        error_add_eve = (
                            "Add planning : " + str(self.req_code)
                            if self.req_code != 200
                            else "Add planning : OK"
                        )
                        st.error(f"{error_add_eve}")
                else:
                    st.warning(
                        """
                        You forget some info...

                        The `name`, `location` are **MANDATORY**.
                        """
                    )


from django.urls import path
from .controller import *

app_name = 'vaksin'

urlpatterns = [
    path('', list_vaksin, name='list_vaksin'),
    path('create', create_vaksin, name='create_vaksin'),
    path('reservasi/list', list_reservasi_vaksin, name='list_reservasi_vaksin'),
    path('reservasi/create/<int:jadwal_id>', create_reservasi_vaksin, name='create_reservasi_vaksin'),
]


import json
from back.playlist import Playlist, make_list_of_all, create_node_sequence
from back.composition import get_comp

class Relator:

    def __init__(self) -> None:
        """
        Конструктор класса, открывает и считывает json файл
        """
        self.json_file = r"C:\Education\algo2\tracks\main_playlist.json"
        with open(self.json_file, 'r', encoding="utf-8") as file:
            tmp = json.load(file)
            tmp[0] = make_list_of_all().get_dict()
        with open(self.json_file, 'w', encoding="utf-8") as file:
            json.dump(tmp, file, indent=4, ensure_ascii=False)

    def save(self, list_of_playlists: list[Playlist]) -> None:
        """
        сохранение
        :param list_of_playlists: список плейлистов
        """
        with open(self.json_file, 'w', encoding='utf-8') as file:
            res = []
            for pllst in list_of_playlists:
                res.append(pllst.get_dict())
            json.dump(res, file, indent=4, ensure_ascii=False)

    def load(self) -> list:
        """
        загрузка файла
        """
        with open(self.json_file, 'r', encoding='utf-8') as file:
            return json.load(file)

    def load_playlists(self) -> list:
        """
        загрузка плейлистов
        :return: список плейлистов
        """
        list_of_playlists = self.load()
        res = []
        for playlist_data in list_of_playlists:
            res.append(Playlist(name=playlist_data['name'],
                                head=create_node_sequence(
                                    get_comp(playlist_data['tracks']))))
        return res

if __name__ == '__main__':
    rel = Relator()
    for playlist in rel.load_playlists():
        print(playlist)


import nltk
nltk.download('gutenberg')
nltk.download('genesis')
nltk.download('inaugural')
nltk.download('nps_chat')
nltk.download('webtext')
nltk.download('treebank')
nltk.download('stopwords')

from nltk.book import *
from nltk.corpus import stopwords

##########
print("------ENCONTRAR PALABRA EXACTA-----")
text1.concordance("monstrous")

##########

print("------ENCONTRAR PALABRA SIMILAR-------")
text1.similar("monstrous")

##########
text7.dispersion_plot(["citizens", "democracy", "freedom", "duties", "America"])

##########
print("--------CANTIDAD DE PALABRAS----")
print(len(text3))

##########
print("-------------------------")
stop_words = stopwords.words('english')
filtered_sentence = [w.upper() for w in text3 if not w in stop_words and w.isalnum() and len(w)>1]
fdist1 = FreqDist(filtered_sentence)
print(fdist1)
fdist1.plot(50, cumulative=False)

##########
print("-------------------------")
V = set(text1)
long_words = [w for w in V if len(w) > 15]
print(sorted(long_words))


import logging
import json
import networkx as nx
# import igraph as ig
import time
import sys
sys.path.insert(0, '/home/mf3jh/workspace/cp3_s2_data_preprocessing')
import data_preprocessing_utils

g_path_prefix = '/home/mf3jh/workspace/data/white_helmet/White_Helmet/Twitter/'
g_time_series_data_path_prefix = g_path_prefix + 'time_series_data/'
g_time_series_data_format = g_time_series_data_path_prefix + '{0}/{1}_nec_data'
g_time_format = '%Y%m%d'
g_fsgraph_output_gml_format = g_time_series_data_path_prefix + '{0}/{1}_fsgraph.gml'

# edge: (src, trg, {r: int, t: int, q:int, a:int, weight:real(normalized)})
# 'weight' will not be handled in this function.
def gen_one_fsgraph(time_int_str):
    fsgraph = nx.DiGraph()
    with open(g_time_series_data_format.format(time_int_str, time_int_str), 'r') as in_fd:
        t_nec_str = in_fd.readline()
        while t_nec_str:
            t_nec_json = json.loads(t_nec_str)
            src = t_nec_json['src']
            trg = t_nec_json['uid']
            t_type = t_nec_json['type']
            if t_type != 'n':
                if trg is None or trg == '' or src is None or src == '':
                    logging.error('The following tweet necessary data has trouble with uid or src: %s' % t_nec_str)
                else:
                    if not fsgraph.has_edge(src, trg):
                        if t_type == 'r':
                            fsgraph.add_edge(src, trg, r=1, t=0, q=0, a=1, weight=0.0)
                        elif t_type == 't':
                            fsgraph.add_edge(src, trg, r=0, t=1, q=0, a=1, weight=0.0)
                        elif t_type == 'q':
                            fsgraph.add_edge(src, trg, r=0, t=0, q=1, a=1, weight=0.0)
                        else:
                            logging.error('Undefined tweet type occurs: %s.' % t_nec_str)
                    else:
                        if t_type == 'r':
                            fsgraph[src][trg]['r'] += 1
                            fsgraph[src][trg]['a'] += 1
                        elif t_type == 't':
                            fsgraph[src][trg]['t'] += 1
                            fsgraph[src][trg]['a'] += 1
                        elif t_type == 'q':
                            fsgraph[src][trg]['q'] += 1
                            fsgraph[src][trg]['a'] += 1
                        else:
                            logging.error('Undefined tweet type occurs: %s.' % t_nec_str)
            t_nec_str = in_fd.readline()
        in_fd.close()
    return fsgraph

def compute_weights(fsgraph, time_int_str):
    l_counts = [edge[2] for edge in fsgraph.edges(data='a')]
    max_count = max(l_counts)
    min_count = min(l_counts)
    if max_count - min_count == 0:
        logging.debug('fsgraph for %s has all equal counts, which may be incorrect.' % time_int_str)
        return fsgraph
    for src, trg in fsgraph.edges:
        fsgraph[src][trg]['weight'] = float(fsgraph[src][trg]['a'] - min_count) / (max_count - min_count)
    return fsgraph

def dump_fsgraphs(d_fsgraphs):
    for time_int_str in d_fsgraphs:
        nx.write_gml(d_fsgraphs[time_int_str], g_fsgraph_output_gml_format.format(time_int_str, time_int_str))
    logging.debug('fsgraphs dump is done.')

def gen_fsgraphs(l_time_ints):
    d_fsgraphs = dict()
    timer_start = time.time()
    for time_int in l_time_ints:
        time_int_str = time_int[0].strftime(g_time_format) + '_' + time_int[1].strftime(g_time_format)
        fsgraph = gen_one_fsgraph(time_int_str)
        d_fsgraphs[time_int_str] = compute_weights(fsgraph, time_int_str)
        logging.debug('The fsgraph for %s is done in %s seconds.' % (time_int_str, str(time.time()-timer_start)))
    return d_fsgraphs

def extract_influential_users_from_one_fsgraph(time_int_str, top_num):
    fsgraph = nx.read_gml(g_fsgraph_output_gml_format.format(time_int_str, time_int_str))
    l_degrees = list(fsgraph.out_degree(fsgraph.nodes, 'a'))
    l_degrees = sorted(l_degrees, key=lambda d: d[1], reverse=True)
    l_top_users = [d[0] for d in l_degrees][:top_num]
    top_sub_fsgraph = nx.subgraph(fsgraph, l_top_users)
    return l_top_users, top_sub_fsgraph
    # print(l_top_users)

def main():
    l_time_ints = data_preprocessing_utils.read_time_ints()
    d_fsgraphs = gen_fsgraphs(l_time_ints)
    dump_fsgraphs(d_fsgraphs)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    main()
    # extract_influential_users_from_one_fsgraph('20180718_20180724', 50)


from django.shortcuts import redirect,reverse

# Create your views here.
from django.http import HttpResponse

def say_hello(request):
    if request.method == "GET":
        name = request.GET.get("name","")
        return HttpResponse("hello,%s!"%(name))

def index(request):
    username = request.GET.get("username","")
    if username:
        return HttpResponse("已登录")
    else:
        # 步骤4、这里用到了url反转
        login_url = reverse("myblog_app:login")
        print("="*30)
        print(login_url)
        print("="*30)
        redirect(login_url)

def login(request):
    return HttpResponse("您所在的页面是登录注册页！！")


"""create tb_users2

Revision ID: 1313e4778551
Revises: 63e71ac9579a
Create Date: 2020-12-15 15:46:05.247405

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1313e4778551'
down_revision = '63e71ac9579a'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('tb_users',
    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
    sa.Column('create_time', sa.DateTime(), nullable=True),
    sa.Column('update_time', sa.DateTime(), nullable=True),
    sa.Column('is_deleted', sa.Boolean(), nullable=True),
    sa.Column('username', sa.String(length=20), nullable=False),
    sa.Column('password', sa.String(length=20), nullable=False),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('username')
    )
    op.create_index(op.f('ix_tb_users_create_time'), 'tb_users', ['create_time'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_tb_users_create_time'), table_name='tb_users')
    op.drop_table('tb_users')
    # ### end Alembic commands ###


import glob
from Client import Client
from Transaction import Transaction
from ConsoleCodes import ConsoleCodes
from TransactionType import TransactionType
from tabulate import tabulate
import matplotlib.pyplot as plt
import numpy as np
import datetime

class Accounting:

    def __init__(self):
        self.clients = []

    # Load all clients from text files located in the folder clients
    def init(self):

        # Read all text files located in the clients folder
        clientFiles = glob.glob('./clients/*.txt')

        # Iterate over all text file paths
        for clientFile in clientFiles:
            with open(clientFile) as f:

                # Open text file for reading it
                f = open(clientFile, "r")

                # Get the content of the text file and split the string by semicolon
                data = f.read().split(';')

                # Make a new object of type client and initialize it with id, firstname, lastname and balance we just read from the text file
                if (len(data) >= 4):
                    client = Client(data[0], data[1], data[2], data[3])

                    #  Get transactions if there are ones
                    for i in range(4, len(data)):
                        transactionArr = data[i].split('/')
                        newTransaction = Transaction(
                            # Type, otherClientId, amount, newBalance, day, month, year
                            transactionArr[0], transactionArr[1], transactionArr[2], transactionArr[3], transactionArr[4], transactionArr[5], transactionArr[6])
                        client.addTransaction(newTransaction)

                    self.clients.append(client)

    # This method will save the data of the clients objects into there proper text file
    def updateClient(self, client):
        with open("./clients/" + str(client.id) + ".txt", 'wt') as f:
            clientString = self.getClientString(client)
            transactionString = self.getTransactionsString(client)

            if (transactionString != None):
                clientString += transactionString

            f.write(clientString)
            f.close()

    # This method creates the correct string for saving client data into the text file
    def getClientString(self, client):
        return str(client.id) + ";" + client.firstname + ";" + client.lastname + ";" + str(client.balance)

    def getTransactionsString(self, client):
        transactionStr = ""
        for transaction in client.transactions:
            transactionStr += ";" + str(transaction.type) + "/" + \
                str(transaction.otherClientId) + "/" + str(transaction.amount) + "/" + str(transaction.newBalance) + "/" + str(transaction.day) + "/" + str(transaction.month) + "/" + str(transaction.year)

        if len(client.transactions) > 0:
            return transactionStr
        else:
            return None

    # Make a transaction from one client to another client
    def makeTransaction(self, fromClientId, toClientId, amount):

        if (fromClientId == toClientId):
            print("Client cannot pay himself")
            return

        fromClient = self.getClientById(fromClientId)

        toClient = self.getClientById(toClientId)

        if (fromClient == None):
            print("Client who shall pay does not exist")
            return

        if (toClient == None):
            print("Client who shall receive does not exist")
            return

        if (float(fromClient.balance) - float(amount) < 0):
            print("Client who shall pay has not enough money")
            return

        # Current date time
        now = datetime.datetime.now()

        # New Balance for fromClient
        fromClient.balance = int(fromClient.balance) - int(amount)

        # Create new transaction for fromClient
        fromClientTransaction = Transaction(
            TransactionType.PAID.value, toClientId, amount, fromClient.balance, now.day, now.month, now.year)

        # Add the transaction to the fromClient object
        fromClient.addTransaction(fromClientTransaction)

        # New Balance for toClient
        toClient.balance = int(toClient.balance) + int(amount)

        # Create new transaction for toClient
        toClientTransaction = Transaction(
            TransactionType.RECEIVED.value, fromClientId, amount, toClient.balance, now.day, now.month, now.year)

        # Add the transaction to the toClient object
        toClient.addTransaction(toClientTransaction)

        # Update fromClient file
        self.updateClient(fromClient)

        # Update toClient file
        self.updateClient(toClient)

    def displayClients(self):
      headers = ["ID", "Firstname", "Lastname", "Balance"]
      data = []
      for client in self.clients:
        data.append([client.id, client.firstname, client.lastname, client.balance]);

      print(tabulate(data, headers, tablefmt="grid"))

    def displayTransactions(self, clientId):

      client = self.getClientById(clientId)

      if (client == None):
            print("Client not found")
            return

      headers = ["Date", "Type", "Amount", "from/to", "Other Client", "New Balance"]

      data = []

      print("Client: ", client.firstname, client.lastname)

      for transaction in client.transactions:

        transactionType, relation = ("Paid", "to") if int(transaction.type) == int(TransactionType.PAID.value) else ("Received", "from");

        otherClient = self.getClientById(transaction.otherClientId);

        otherClientName = otherClient.firstname + " " + otherClient.lastname if otherClient != None else "Unknown"

        transactionAmountStr = "{:.2f}".format(float(transaction.amount)) + '€'

        newBalanceStr = "{:.2f}".format(float(transaction.newBalance)) + '€'

        data.append([transaction.date(), transactionType, transactionAmountStr, relation, otherClientName, newBalanceStr]);

      print(tabulate(data, headers, tablefmt="grid"))

    def getClientById(self, clientId):
        for client in self.clients:
            if int(client.id) == int(clientId):
                return client

        return None

    def newClient(self, firstname, lastname, balance):
        clientId = self.getNewClientId()
        client = Client(clientId, firstname, lastname, balance)
        self.clients.append(client)
        self.updateClient(client)

    def getNewClientId(self):
        clientId = 0
        for client in self.clients:
            if client.id >= clientId:
                clientId = client.id + 1
        return clientId

    def displayCashflowGraph(self, clientId):
      client = self.getClientById(clientId)

      if (client == None):
            print("Client not found")
            return

      dates = list(map(lambda transaction : transaction.date(), client.transactions))

      balances = list(map(lambda transaction : float(transaction.newBalance), client.transactions))

      fig, ax = plt.subplots()
      ax.plot(dates, balances)

      ax.set(xlabel='Date', ylabel='Balance (€)',
             title="Client: " + client.firstname + " " + client.lastname)
      ax.grid()

      plt.show()


import boto3
import logging
import botocore.exceptions

def lambda_handler(event, context):

    eks_versions = ['1.25', '1.24', '1.23', '1.22', '1.21', '1.20', '1.19']
    eks_versions_len = len(eks_versions)

    logger = logging.getLogger()
    logger.setLevel(logging.INFO)

    master_session = boto3.session.Session()
    sts = master_session.client('sts')

    current_account = sts.get_caller_identity()['Account']

    id = event["account"]
    regions = boto3.session.Session().get_available_regions('eks')

    resource_output = dict()

    for region in regions:
        if id != current_account:
            role = "arn:aws:iam::{}:role/aws-ver-dash-lambda-eks"
            role_arn = role.format(id)
            try:
                assume_role_response = sts.assume_role(
                    RoleArn=role_arn, RoleSessionName="LambdaExecution")

                sts_connection = boto3.client('sts')
                acct_b = sts_connection.assume_role(
                    RoleArn=role_arn,
                    RoleSessionName="cross_acct_lambda"
                )

                ACCESS_KEY = acct_b['Credentials']['AccessKeyId']
                SECRET_KEY = acct_b['Credentials']['SecretAccessKey']
                SESSION_TOKEN = acct_b['Credentials']['SessionToken']

                # create service client using the assumed role credentials, e.g. S3
                eks = boto3.client(
                    'eks',
                    aws_access_key_id=ACCESS_KEY,
                    aws_secret_access_key=SECRET_KEY,
                    aws_session_token=SESSION_TOKEN,
                    region_name=region,
                )
            except Exception as e:
                logger.error("Assume role error %s", e)

        else:
            eks = boto3.client('eks', region_name=region)

        try:
            clusterslist = eks.list_clusters()
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == 'AuthFailure':
                print('Region {region} seems to be disabled for this account, skipping')
                continue
        except:
            raise e
        else:
            clusterslist = clusterslist['clusters']

            for clusterName in clusterslist:
                instance = eks.describe_cluster(name=clusterName)
                for i in range(eks_versions_len):
                    if instance['cluster']['version'] == eks_versions[i]:
                        versions_back = i

                resource_output[instance['cluster']['name']] = {'Check': 'eks',
                                                                'Account': id,
                                                                'Region': region,
                                                                'Resource ID': instance['cluster']['name'],
                                                                'Sub Type': '',
                                                                'Deployed Version': instance['cluster']['version'],
                                                                'Latest Version': eks_versions[0],
                                                                'Versions Back': versions_back
                                                                }

    return resource_output


import csv

from bs4 import BeautifulSoup
import requests

base_url = "http://everydayword.ru%s"
url = "http://everydayword.ru/glossary?page=%d"
url_list = []
for page_number in range(0, 55):
    url_list.append(url % page_number)

word_list = []
href_list = []

for url in url_list:
    #print(url)
    r = requests.get(url)
    data = r.text
    soup = BeautifulSoup(data, 'html.parser')
    data = soup.findAll('h2')
    for row in data:
        link = row.find('a')
        if link is not None:
            href = link['href']
            #print(href)
            #print(link.string)
            word_list.append(link.string)
            href_list.append(base_url % href)

word_dict = dict(zip(word_list, href_list))
print(word_dict)
w = csv.writer(open("output.csv", "w"))
for key, val in word_dict.items():
    #print(key,val)
    w.writerow([key, val])


import os
import hydra
import pickle
from abc import ABC
import torch.nn as nn
from glob import glob
from pathlib import Path
import torch.multiprocessing
import pytorch_lightning as pl
from trainer import create_trainer
from util.stylegan_utils import *
from model.mappers.mlp import Mapper
from criteria.clip_loss import ClipLoss
from util.misc import transform_points
from dataset.face_mesh_video import FaceMesh
from torch.utils.data import DataLoader
from torchvision.utils import save_image
from model.flame.flame_model import FLAME
from model.stylegan_ada.generator import Generator
from pytorch_lightning.utilities import rank_zero_only
from model.renderer.differentiable_renderer import DifferentiableRenderer, transform_pos_mvp

torch.multiprocessing.set_sharing_strategy('file_system')  # a fix for the "OSError: too many files" exception
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.allow_tf32 = True
torch.backends.cudnn.deterministic = True

os.environ['CUDA_LAUNCH_BLOCKING'] = '1'

class StyleGANOptimizer(pl.LightningModule, ABC):

    def __init__(self, config):
        super().__init__()
        self.save_hyperparameters(config)
        self.config = config
        self.R = None
        self.flame = FLAME(config)
        self.clip_loss = ClipLoss(config=config)
        self.l2_loss = nn.MSELoss(reduce=False, reduction='none')
        self.G = Generator(z_dim=config.latent_dim, w_dim=config.latent_dim,
                           w_num_layers=config.num_mapping_layers, img_resolution=config.image_size,
                           img_channels=3, synthesis_layer=config.generator)
        self.train_set = FaceMesh(w_codes_pth=config.w_train_pth, mesh_path="data/flame/head_template.obj",
                                  flame_params="data/clip/flame_params.pkl", verts_pth='data/clip/verts.pkl',
                                  deca_warped_path='data/clip/sample_deca.pkl')
        self.val_set = FaceMesh(w_codes_pth=config.w_val_pth, mesh_path="data/flame/head_template.obj",
                                flame_params="data/clip/flame_params.pkl", verts_pth='data/clip/verts.pkl',
                                deca_warped_path='data/clip/sample_deca.pkl', mode='val')
        self.automatic_optimization = False
        if config.batch_gpu is None:
            config.batch_gpu = config.batch_size
        print(f"batch_size = {config.batch_size} / {config.batch_gpu}")
        assert config.batch_size >= config.batch_gpu and config.batch_size % config.batch_gpu == 0

        # The list of expression codes
        exp_codes_pth = sorted(glob(config.exp_codes_pth + '*.pkl'))
        exp_codes = []
        pose_codes = []
        for exp_code_pth in exp_codes_pth:
            with open(exp_code_pth, 'rb') as f:
                flame_params_frames = pickle.load(f)
                exp_codes.append(torch.from_numpy(flame_params_frames['exp']).float())
                pose_codes.append(torch.from_numpy(flame_params_frames['pose']).float())
        self.exp_codes = torch.stack(exp_codes)
        self.pose_codes = torch.stack(pose_codes)
        self.num_frames = len(exp_codes)

        # The softmax expression deltas
        exp_neutral = torch.zeros(1, config.expression_params).float()
        softmax_scores_unnormalized = []
        for exp_code, pose_code in zip(exp_codes, pose_codes):
            exp_pose_neutral = torch.cat((exp_neutral, pose_code[:, :3], torch.zeros(1, 3)), dim=1)
            exp_pose_modified = torch.cat((exp_code, pose_code), dim=1)
            delta = self.l2_loss(exp_pose_modified, exp_pose_neutral).sum()
            softmax_scores_unnormalized.append(delta)
        softmax_scores_unnormalized = torch.as_tensor(softmax_scores_unnormalized)
        normalized = (softmax_scores_unnormalized - min(softmax_scores_unnormalized)) / (max(softmax_scores_unnormalized) - min(softmax_scores_unnormalized))
        self.normalized_scores = normalized.unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + 1e-6  # For numerical stability

        # Generator
        self.G.load_state_dict(torch.load(config.pretrained_stylegan_pth, map_location=torch.device("cpu")))
        self.G.eval()

        # Mapper for expression conditioned textures
        self.texture_mapper_list = []
        state_dict = torch.load(config.pretrain_video_mapper_pth, map_location=torch.device("cpu"))
        for i in range(18):
            mapper = Mapper(z_dim=config.latent_dim + config.expression_params + 6, w_dim=config.latent_dim, num_layers=1)
            mapper.load_state_dict(state_dict["texture_mapper"])
            self.texture_mapper_list.append(mapper)

    def configure_optimizers(self):
        trainable_texure_params = []
        for mapper in self.texture_mapper_list:
            trainable_texure_params += list(mapper.parameters())
        code_opt = torch.optim.Adam([{'params': trainable_texure_params, 'lr': self.config.lr_tex}], lr=self.config.lr_tex)
        return code_opt

    def gather_temporal_frames(self, w_code, batch):
        tex_frames, img_frames, vtx_frames = [], [], []
        predicted_tex_frames = self.G.synthesis(w_code.squeeze(), noise_mode='const')
        for idx in range(self.num_frames):
            expression_params = self.exp_codes[idx].repeat(self.config.batch_size, 1)
            pose_params = self.pose_codes[idx].repeat(self.config.batch_size, 1)
            vertices = self.flame(shape_params=batch['flame_shape'], expression_params=expression_params, pose_params=pose_params)
            img = self.render(predicted_tex_frames[idx].repeat(self.config.batch_size, 1, 1, 1), vertices, batch)
            tex_frames.append(predicted_tex_frames[idx])
            img_frames.append(img)
            vtx_frames.append(vertices)
        return {'tex_frames': tex_frames, 'img_frames': img_frames, 'vtx_frames': vtx_frames}

    def forward(self, batch):

        with torch.no_grad():
            w_init = batch['w_code'].unsqueeze(0).repeat(1, self.num_frames, 1, 1)
            results_init = self.gather_temporal_frames(w_init, batch)

        # Get the new texture
        w_offsets = None
        for idx, mapper in enumerate(self.texture_mapper_list):
            mapper_input = torch.cat((batch['w_code'][:, idx, :].unsqueeze(0).repeat(self.num_frames, 1, 1), self.exp_codes, self.pose_codes), dim=-1).squeeze()
            w_offset_layer = mapper(mapper_input).unsqueeze(1)
            if w_offsets is None:
                w_offsets = w_offset_layer
            else:
                w_offsets = torch.cat((w_offsets, w_offset_layer), dim=1)
        w_offsets = w_offsets.unsqueeze(0)

        w_vid = w_init + self.normalized_scores * w_offsets
        results_final = self.gather_temporal_frames(w_vid, batch)
        return results_init, results_final

    def training_step(self, batch, batch_idx):
        code_opt = self.optimizers()
        results_init, results_final = self.forward(batch)
        clip_loss = 0
        for idx in range(self.num_frames):
            clip_loss += self.clip_loss(results_final['img_frames'][idx], results_init['img_frames'][idx])
        self.log(f"train/clip_loss", clip_loss.item(), on_step=True, on_epoch=False, prog_bar=True, logger=True)
        self.manual_backward(clip_loss)
        step(code_opt, self.texture_mapper_list)
        torch.cuda.empty_cache()

    def render(self, texture_image, flame_verts, batch):
        vertices, uvs = [], []
        faces, uv_indices = [], []
        num_vertex, num_uvs = 0, 0
        start_index = 0
        ranges = []
        for i in range(self.config.batch_size):
            # Translate and scale according to Flame camera parameters
            custom_scale_factor = 2.0
            custom_scale_factor_image = 1024 / self.config.image_size
            camera = batch['flame_cam'][i]
            trans_verts = flame_verts[i][:, :2] + camera[1:]
            trans_verts = torch.cat([trans_verts, flame_verts[i][:, 2:]], dim=1)
            scaled_verts = custom_scale_factor * trans_verts * camera[0]

            # Apply model-view and projection transform.
            projection_matrix, view_matrix = batch['projection_matrix'][i], batch['view_matrix'][i]
            vertices_mvp = transform_pos_mvp(scaled_verts, torch.matmul(projection_matrix, view_matrix).to(self.device).unsqueeze(0))

            # Apply scaling to fit-in the image size
            points_scale = [self.config.deca_size, self.config.deca_size]  # Increases scale and shifts right + bottom for large values
            h, w = [custom_scale_factor_image * self.config.image_size, custom_scale_factor_image * self.config.image_size]  # Increases scale and shifts right + bottom for smaller values
            tform = torch.inverse(self.train_set.tform[None, ...]).transpose(1, 2).to(self.device)
            vertices_mvp = transform_points(vertices_mvp.unsqueeze(0), tform, points_scale, [h, w])[0]

            vertices.append(vertices_mvp)
            uvs.append(self.train_set.uvs)
            faces.append(self.train_set.faces + num_vertex)
            uv_indices.append(self.train_set.uv_indices + num_uvs)
            num_vertex += self.train_set.vertices.shape[0]
            num_uvs += self.train_set.uvs.shape[0]
            ranges.append(torch.tensor([start_index, self.train_set.faces.shape[0]]).int())
            start_index += self.train_set.faces.shape[0]
        vertices, uvs, faces, uv_indices, ranges = torch.cat(vertices, 0), torch.cat(uvs, 0), torch.cat(faces, 0), torch.cat(uv_indices, 0), torch.stack(ranges, dim=0)
        return self.R.render_with_texture_map(vertices, faces, uvs, uv_indices, texture_image, ranges=ranges).permute((0, 3, 1, 2)).contiguous()

    def validation_step(self, batch, batch_idx):
        self.exp_codes = self.exp_codes.to(self.device)
        self.pose_codes = self.pose_codes.to(self.device)
        self.normalized_scores = self.normalized_scores.to(self.device)
        results_init, results_final = self.forward(batch)
        clip_loss = 0
        for idx in range(self.num_frames):
            clip_loss += self.clip_loss(results_final['img_frames'][idx], results_init['img_frames'][idx])
        self.log(f"val/clip_loss", clip_loss.item(), on_step=True, on_epoch=False, prog_bar=True, logger=True)
        torch.cuda.empty_cache()

    @rank_zero_only
    def validation_epoch_end(self, _val_step_outputs):
        odir_texture, odir_samples, odir_ckpts = self.create_directories()
        self.export_images(odir_texture, odir_samples, odir_ckpts)

    def create_directories(self):
        output_dir_textures = Path(f'{self.config.base_dir}/runs/{self.config.experiment}/texture/')
        output_dir_images = Path(f'{self.config.base_dir}/runs/{self.config.experiment}/images/')
        output_dir_ckpts = Path(f'{self.config.base_dir}/runs/{self.config.experiment}/ckpts/{self.global_step:06d}')
        for odir in [output_dir_textures, output_dir_images, output_dir_ckpts]:
            odir.mkdir(exist_ok=True, parents=True)
        return output_dir_textures, output_dir_images, output_dir_ckpts

    def export_images(self, odir_texture, odir_samples, odir_ckpts):
        with torch.no_grad():
            for iter_idx, batch in enumerate(self.val_dataloader()):
                if iter_idx < self.config.num_vis_images:
                    batch['flame_shape'] = batch['flame_shape'].to(self.device)
                    batch['flame_exp'] = batch['flame_exp'].to(self.device)
                    batch['flame_pose'] = batch['flame_pose'].to(self.device)
                    batch['flame_cam'] = batch['flame_cam'].to(self.device)
                    self.exp_codes = self.exp_codes.to(self.device)
                    self.pose_codes = self.pose_codes.to(self.device)
                    self.normalized_scores = self.normalized_scores.to(self.device)

                    _, results_final = self.forward(batch)

                    odir_frame_images_pth = Path(f"{self.config.base_dir}/runs/{self.config.experiment}/frame_images/{self.global_step:06d}/{batch['name'][0]}")
                    odir_frame_texture_pth = Path(f"{self.config.base_dir}/runs/{self.config.experiment}/frame_texture/{self.global_step:06d}/{batch['name'][0]}")
                    odir_frame_images_pth.mkdir(exist_ok=True, parents=True)
                    odir_frame_texture_pth.mkdir(exist_ok=True, parents=True)
                    for frame_idx in range(self.num_frames):
                        save_image(results_final['img_frames'][frame_idx], odir_frame_images_pth / f"{frame_idx:02d}.png", value_range=(-1, 1), normalize=True)
                        save_image(results_final['tex_frames'][frame_idx], odir_frame_texture_pth / f"{frame_idx:02d}.png", value_range=(-1, 1), normalize=True)

                    vis_generated_images = torch.stack(results_final['img_frames'])[:, 0, :, :, :]
                    vis_generated_textures = torch.stack(results_final['tex_frames'])
                    torch.cuda.empty_cache()
                    save_image(vis_generated_images, odir_samples / f"{self.global_step:06d}_{iter_idx:02d}.png", nrow=int(math.sqrt(vis_generated_images.shape[0])), value_range=(-1, 1), normalize=True)
                    save_image(vis_generated_textures, odir_texture / f"{self.global_step:06d}_{iter_idx:02d}.png", nrow=int(math.sqrt(vis_generated_images.shape[0])), value_range=(-1, 1), normalize=True)

            mapper_state_dict = {}
            for level, mapper in enumerate(self.texture_mapper_list):
                mapper_state_dict[f"level_{level}"] = mapper.state_dict()
            torch.save(mapper_state_dict, odir_ckpts / f"{self.global_step:06d}.pt")

    def train_dataloader(self):
        return DataLoader(self.train_set, self.config.batch_size, shuffle=False, pin_memory=False, drop_last=True, num_workers=self.config.num_workers)

    def val_dataloader(self):
        return DataLoader(self.val_set, self.config.batch_size, shuffle=False, drop_last=True, num_workers=self.config.num_workers)

    def on_train_start(self):
        self.run_post_device_setup()

    def on_validation_start(self):
        self.run_post_device_setup()

    def run_post_device_setup(self):
        if self.R is None:
            self.R = DifferentiableRenderer(self.config.image_size, "standard")
        self.R.to(self.device)
        self.G.to(self.device)
        self.flame.to(self.device)
        self.exp_codes.to(self.device)
        self.pose_codes.to(self.device)
        self.clip_loss.set_device(self.device)
        self.normalized_scores.to(self.device)
        for mapper in self.texture_mapper_list:
            mapper.to(self.device)
        self.train_set.set_device(self.device)
        self.val_set.set_device(self.device)

def step(opt, module_list):
    for module in module_list:
        for param in module.parameters():
            if param.grad is not None:
                torch.nan_to_num(param.grad, nan=0, posinf=1e5, neginf=-1e5, out=param.grad)
        opt.step()

@hydra.main(config_path='../../config', config_name='clipface')
def main(config):
    trainer = create_trainer("StyleGANClipMLPVideo", config)
    model = StyleGANOptimizer(config)
    trainer.fit(model)

if __name__ == '__main__':
    main()


"""Show example usage of the API
    - Load a csv file
    - Detect spike
    - plot raw + threshold
    - plot scatter of spike freq (Hz) and half width (ms)
"""
import numpy as np
import scipy.stats

import seaborn as sns
import matplotlib.pyplot as plt  # just to show plots during script

import sanpy

def run():

    # load raw data
    path = 'model-manuscript/stoch-hh-gk-20230312-152309-36-30.csv'
    ba = sanpy.bAnalysis(path)

    # get detection presets and choose 'Fast Neuron'
    _detectionParams = sanpy.bDetection()
    _dDict = _detectionParams.getDetectionDict('Fast Neuron')

    # spike detect
    ba.spikeDetect(_dDict)

    # set spike conditions
    ba.setSpikeStat_time(0, 0.99, 'condition', 'control')
    ba.setSpikeStat_time(1.01, 2, 'condition', 'drug')

    sns.set_context("talk")
    fig, axs = plt.subplots(3,1, sharex=True, figsize=(10, 8))

    bap = sanpy.bAnalysisPlot(ba)

    # plot raw data with threshold
    bap.plotSpikes(plotThreshold=True, ax=axs[0])

    bap.plotStat('thresholdSec', 'spikeFreq_hz', hue='condition', ax=axs[1])

    #bap.plotStat('thresholdSec', 'widths_50', hue='condition')

    df = ba.spikeDict.asDataFrame()  # regenerates, can be expensive
    sns.scatterplot(x='thresholdSec', y='widths_50',
                    hue='condition', data=df,
                    legend=False, ax=axs[2])

    sns.despine(fig)

    #
    # run stats
    statList = [
         'spikeFreq_hz',
         'widths_50',
         'thresholdVal',  # NOT SIGNIFICANT !!!
    ]

    dfPlot = df

    for statName in statList:
        x1 = dfPlot[dfPlot['condition']=='control'][statName]
        x1 = x1[~np.isnan(x1)]  # first spike has nan freq

        x2 = dfPlot[dfPlot['condition']=='drug'][statName]

        _stat = scipy.stats.mannwhitneyu(x1, x2)
        print(f'{statName}: Control vs Iso {_stat}')

        _mean = np.mean(x1)
        _std = np.std(x1)
        _n = len(x1)
        print(f'  Control mean:{_mean}  std:{_std} n:{_n}')

        _mean = np.mean(x2)
        _std = np.std(x2)
        _n = len(x2)
        print(f'   Drug    mean:{_mean}  std:{_std} n:{_n}')

        # print('x1 nan', np.count_nonzero(np.isnan(x1)))
        # print('x2 nan', np.count_nonzero(np.isnan(x2)))

    plt.show()

if __name__ == '__main__':
    run()


import logging
import os
import cv2
import imageio
import numpy as np
import torch
from skimage.metrics import structural_similarity
from tqdm import tqdm

def compute_psnr_from_mse(mse):
    return -10.0 * torch.log(mse) / np.log(10.0)

def compute_psnr(pred, target, mask=None):
    """Compute psnr value (we assume the maximum pixel value is 1)."""
    if mask is not None:
        pred, target = pred[mask], target[mask]
    mse = ((pred - target) ** 2).mean()
    # import pdb; pdb.set_trace()
    return compute_psnr_from_mse(mse).cpu().numpy()

def compute_ssim(pred, target, mask=None):
    """Computes Masked SSIM following the neuralbody paper."""
    assert pred.shape == target.shape and pred.shape[-1] == 3
    if mask is not None:
        x, y, w, h = cv2.boundingRect(mask.cpu().numpy().astype(np.uint8))
        pred = pred[y : y + h, x : x + w]
        target = target[y : y + h, x : x + w]
    try:
        ssim = structural_similarity(
            pred.cpu().numpy(), target.cpu().numpy(), channel_axis=-1
        )
    except ValueError:
        ssim = structural_similarity(
            pred.cpu().numpy(), target.cpu().numpy(), multichannel=True
        )
    return ssim

def save_rendered_img(img_meta, rendered_results):
    filename = img_meta[0]["filename"]
    scenes = filename.split('/')[-2]
    metrics = dict()

    for ret in rendered_results:
        depth = ret['outputs_coarse']['depth']
        rgb = ret['outputs_coarse']['rgb']
        gt  = ret['gt_rgb']
        gt_depth = ret['gt_depth']

    # psnr = compute_psnr(rgb, gt, mask=None)
    # # save images
    psnr_total = 0
    ssim_total = 0
    rsme = 0
    for v in range(gt.shape[0]):
        rsme += ((depth[v] - gt_depth[v])**2).cpu().numpy()
        depth_ = ((depth[v]-depth[v].min()) / (depth[v].max() - depth[v].min()+1e-8)).repeat(1, 1, 3)
        img_to_save = torch.cat([rgb[v], gt[v], depth_], dim=1)
        image_path = os.path.join('nerf_vs_rebuttal', scenes)
        if not os.path.exists(image_path):
            os.makedirs(image_path)
        save_dir = os.path.join(image_path, 'view_'+str(v)+'.png')

        font = cv2.FONT_HERSHEY_SIMPLEX
        org = (50, 50)
        fontScale = 1
        color = (255, 0, 0)
        thickness = 2
        image = np.uint8(img_to_save.cpu().numpy() * 255.0)
        psnr = compute_psnr(rgb[v], gt[v], mask=None)
        psnr_total += psnr
        ssim = compute_ssim(rgb[v], gt[v], mask=None)
        ssim_total += ssim
        image = cv2.putText(image, 'PSNR: ' + "%.2f" % compute_psnr(rgb[v], gt[v], mask=None),
            org, font, fontScale, color, thickness, cv2.LINE_AA)

        cv2.imwrite(save_dir, image)

    return psnr_total/gt.shape[0], ssim_total/gt.shape[0], rsme/gt.shape[0]


# Write program to demonstrate the following aspects of signal processing on suitable data
# 1.Upsampling and downsampling on Image/speech signal
# 2.Fast Fourier Transform to compute DFT

import cv2
from matplotlib import pyplot as plt
import numpy as np
from sympy import fft

# downsampling
def Downsampling(img1,m,n,f):
    img2 = np.zeros((m//f ,n//f),dtype = np.int32)

    for i in range(0,m,f):
        for j in range(0,n,f):
            try:
                img2[i//f][j//f] = img1[i][j]
            except IndexError:
                pass

    plt.imshow(img2,cmap="gray")
    plt.show()

    Upsampling(img1,img2,m,n,f)

# Upsampling
def Upsampling(img1,img2,m,n,f):
    img3 = np.zeros((m, n), dtype = np.int32)

    for i in range(0,m-1,f):
        for j in range(0,n-1,f):
            img3[i][j] = img2[i//f][j//f]

    for i in range(1,m-(f-1),f):
        for j in range(0,n-(f-1)):
            img3[i:i+(f-1), j] = img3[i-1][j]

    for i in range(0,m-1):
        for j in range(1,n-1,f):
            img3[i, j:j+(f-1)] = img3[i][j-1]

    plt.imshow(img3,cmap="gray")
    plt.show()

# 2.Fast Fourier Transform to compute DFT
def FFT():
    seq = [15,21,13,44]

    tranform = fft(seq)
    print(tranform)

def main():
    img1 = cv2.imread('lenna.jpg',0)

    m,n = img1.shape
    print("Dimensions ",m,n)

    plt.imshow(img1,cmap="gray")
    plt.show()

    Downsampling(img1,m,n,4)
    FFT()

if __name__ == "__main__":
    main()