"""Plot value function results."""
import argparse
from protos import results_pb2
import numpy as np
import matplotlib as mlp
mlp.use('Agg')
from matplotlib import pyplot as plt

from matplotlib import rcParams
import os

def bool_argument(parser, name, default=False, msg=''):
    dest = name.replace('-', '_')
    parser.add_argument('--%s' % name, dest=dest, type=bool, default=default, help=msg)
    parser.add_argument('--no-%s' % name, dest=dest, type=bool, default=default, help=msg)

parser = argparse.ArgumentParser()
parser.add_argument('result_directory', help=help)
FLAGS = parser.parse_args()

rcParams['text.latex.preamble'] = [r'\usepackage{sfmath} \boldmath']

PSEC_LRS = set()

def read_proto(filename):
    results = results_pb2.MethodResult()
    with open(filename, 'rb') as f:
        results.ParseFromString(f.read())
    return results

def plot_data(data, file_name, plot_params):
    fig, ax = plt.subplots()
    fig.set_size_inches(13.5, 12.0, forward=True)

    for method in sorted(data):
        label = method

        print ('method ' + method)

        sorted_trajs = sorted(data[method])
        x = []
        y = []
        ylower = []
        yupper = []
        for num_traj in sorted_trajs:
            sorted_lrs = sorted(data[method][num_traj])
            if 'PSEC' in method:
                for lr in sorted_lrs:
                    #if lr == 0.05:
                    #    continue
                    errors = np.array(data[method][num_traj][lr])

                    n = len(errors) # number of trials
                    mean = np.mean(errors)
                    std = np.std(errors)
                    #if 'Lin' in method:
                    print ('number of errors for {} trajs: {}, mean MSE {} std {} '.format(lr, len(data[method][num_traj][lr]), mean, 1.96 * std / np.sqrt(float(n))))
                    #if mean < 450 :
                    #    print (', '.join(map(str, errors)))

                    #if mean >= 400:
                    #    continue
                    yerr = 1.96 * std / np.sqrt(float(n))
                    y.append(mean)
                    ylower.append(mean - yerr)
                    yupper.append(mean + yerr)
                    x.append(lr)
            else:
                x = list(PSEC_LRS)
                x = sorted(x)
                errors = np.array(data[method][num_traj][0])

                n = len(errors) # number of trials

                mean = np.mean(errors)
                std = np.std(errors)
                print ('number of errors for {} trajs: {}, MSE mean {}, std {}'.format(0, len(data[method][num_traj][0]), mean, 1.96 * std / np.sqrt(float(n))))
                #print (', '.join(map(str, errors)))
                yerr = 1.96 * std / np.sqrt(float(n))
                y = [mean for _ in range(len(x))]
                ylower = [mean - yerr for _ in range(len(x))]
                yupper = [mean + yerr for _ in range(len(x))]

            linestyle = '-'
            #linestyle = '-.'
            #linestyle = ':'
            if method == 'TD(0)':
                color = 'red'
                line, = plt.plot(x, y, label=label, linestyle = linestyle, linewidth = 5, color = color)
            else:
                line, = plt.plot(x, y, label=label, linestyle = linestyle, linewidth = 5)
            color = line.get_color()

            alpha = 0.5
            plt.fill_between(x, ylower, yupper, facecolor=color, alpha=alpha)

    if plot_params['log_scale']:
        ax.set_xscale('log')
        #ax.set_yscale('log')
    if plot_params['x_range'] is not None:
        plt.xlim(plot_params['x_range'])
    if plot_params['y_range'] is not None:
        plt.ylim(plot_params['y_range'])

    ax.xaxis.set_tick_params(labelsize=plot_params['tfont'])
    ax.yaxis.set_tick_params(labelsize=plot_params['tfont'])

    ax.set_xlabel(plot_params['x_label'], fontsize=plot_params['bfont'])
    ax.set_ylabel(plot_params['y_label'], fontsize=plot_params['bfont'])

    if plot_params['legend']:
        plt.legend(fontsize=plot_params['lfont'], loc=plot_params['legend_loc'],
                   ncol=plot_params['legend_cols'])
    fig.tight_layout()
    plt.savefig('{}.pdf'.format(file_name))
    plt.close()
    #plt.show()

def collect_data():

    data = {}
    td_method = 'td-0-vs-mdp'
    ristd_method = 'ris-' + td_method

    for basename in os.listdir(FLAGS.result_directory):
        filename = os.path.join(FLAGS.result_directory, basename)
        if '.err' in filename:
            continue
        try:
            results = read_proto(filename)
        except Exception as e:
            if 'Tag' not in str(e):
                raise e

        #method_TD(0)_trial_847277_trajs_9_alpha_0.010000_tiling_20_t1_15_t2_9_psec-act_0_psec-nh_0_psec-ne_0_psec-lr_0.000000
        #method_TD(0)_trial_847277_trajs_9_alpha_0.010000_tiling_20_t1_15_t2_9_psec-act_0_psec-nh_0_psec-ne_0_psec-lr_0.000000
        #print (basename)
        specs = basename.split('_')

        # thing that is fixed is the seed and traj size, finding best performing based on these two fixed points
        # if td looking at lr, tc, etc
        # if psec, looking at above AND the extra PSEC stuff

        method = specs[specs.index('method') + 1]

        num_trajs = int(results.num_trajs) #int(results.num_steps_observed[0])

        if method not in data:
            data[method] = {}
        if num_trajs not in data[method]:
            data[method][num_trajs] = {}

        psec_lr = float(specs[specs.index('psec-lr') + 1])

        if psec_lr not in data[method][num_trajs]:
            data[method][num_trajs][psec_lr] = []

        errs = results.value_error
        data[method][num_trajs][psec_lr].append(errs)

        if 'PSEC' in method:
            PSEC_LRS.add(psec_lr)

    return data

def main():

    if not FLAGS.result_directory:
        print ('No result directory given. Exiting.')
        return

    data = collect_data()

    plot_params = {'bfont': 40,
               'lfont': 35,
               'tfont': 40,
               'legend': True,
               'legend_loc': 0,
               'legend_cols': 2,
               'y_range': None,
               'x_range': None,
               'log_scale': True,
               'x_label': 'PSEC Learning Rate',
               'y_label': 'MSVE',
               'shade_error': True,
               'x_mult': 1}
    plot_data(data, 'temp1', plot_params)

if __name__ == '__main__':
    main()


from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import *

class About(QWidget):
    def __init__(self, parent):
        QWidget.__init__(self)
        self.parent = parent
        self.initUI()

    def initUI(self):
        font = QtGui.QFont()
        font.setFamily("微软雅黑 Light")
        font.setPointSize(12)

        # self.create_about_box()

        mainLayout = QVBoxLayout()
        VboxLayout = QVBoxLayout()
        text_edit = QTextBrowser()
        text = '''
作者：dandh811
软件版本：v0.1.2
邮箱: dandh811@163.com
网址：www.dongjianjun.com

这是一款安全测试辅助软件，可以记录各位洞主的挖洞生涯。
由于使用pyinstaller打包，运行EXE文件时360等杀毒软件会报毒，目前还没找到解决办法，暂时需要添加信任该软件。
请放心使用，代码完全开源。
水平有限，欢迎提建议。
        '''
        text_edit.setText(text)
        VboxLayout.addWidget(text_edit)
        VboxLayout.addStretch()

        mainLayout.addLayout(VboxLayout)
        self.setLayout(mainLayout)


from . import estimate

import cv2
from time import time

def capture():

	# open camera
    cap = cv2.VideoCapture(701)

    # set resolution
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)

    while cap.isOpened():

        # read image
        ret, img = cap.read()
        if ret != True:
            print('read image failed')
            continue

		# pose estimate
        estimate.pose(img)

		# show image
        cv2.imshow('camera', img)
        key = cv2.waitKey(delay = 2)

		# save image
        if key == ord('t'):
            name = str(round(time() * 1000)) + '.jpg'
            cv2.imwrite('./assets/img/' + name, img)

		# quit
        if key == ord('q') or key == 27:
            break

    cap.release()
    cv2.destroyAllWindows()

capture()


"""
Dispatch script for imputation/time-gridding
"""

import subprocess
import argparse
import pickle
import os
import sys

import functions.util_filesystem as mlhc_fs

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

pe = os.path.exists
pj = os.path.join
HOME = os.path.expanduser("~")

def timegrid_all_patients(configs):
    job_index=0
    subprocess.call(["source activate default_py36"],shell=True)
    mem_in_mbytes=configs["mem_in_mbytes"]
    n_cpu_cores=1
    n_compute_hours=configs["nhours"]
    compute_script_path=configs["compute_script_path"]

    with open(configs["patient_batch_path"],'rb') as fp:
        obj=pickle.load(fp)
        batch_to_lst=obj["batch_to_lst"]
        batches=list(sorted(batch_to_lst.keys()))

    create_pars = []
    for p in ["create_static", "create_dynamic", "create_async"]:
        if configs[p]:
            create_pars.append("--" + p + " True")
    if configs["save_pts_separately"]:
        create_pars.append("--save_pts_separately")

    for batch_idx in batches:
        print("Dispatching imputation for batch {}".format(batch_idx))
        job_name = "impute_batch_{}".format(batch_idx)
        log_result_file = pj( configs["log_base_dir"],
                "impute_batch_{}_RESULT.txt".format(batch_idx) )
        mlhc_fs.delete_if_exist(log_result_file)

        cmd_line=" ".join(["python3", configs["compute_script_path"],
            "--run_mode INTERACTIVE", "--batch_id {}".format(batch_idx)] \
                    + create_pars)

        assert(" rm " not in cmd_line)
        job_index+=1

        if not configs["dry_run"]:
            subprocess.call([cmd_line], shell=True)
        else:
            print("Generated cmd line: [{}]".format(cmd_line))
        if configs["just_one_batch"]:
            break

if __name__=="__main__":

    parser=argparse.ArgumentParser()

    # Input paths
    parser.add_argument("--patient_batch_path",
            default=pj(HOME, "Datasets/eicu-2.0/patient_batches.pickle"),
            help="The path of the PID-Batch map")
    parser.add_argument("--compute_script_path",
            default=pj(HOME, "Repos/mattphillipsphd/dpsom/eicu_preproc" \
                    "/timegrid_one_batch.py"),
            help="Script to dispatch")

    # Output paths
    parser.add_argument("--log_base_dir",
            default=pj(HOME, "Datasets/eicu-2.0/logs"),
            help="Log base directory")

    # Parameters
    parser.add_argument("--dry_run", action="store_true",
            default=False,
            help="Dry run, do not generate any jobs")
    parser.add_argument("--j1", "--just-one-batch", dest="just_one_batch",
            action="store_true")
    parser.add_argument("--mem_in_mbytes", type=int,
            default=5000,
            help="Number of MB to request per script")
    parser.add_argument("--nhours", type=int,
            default=4,
            help="Number of hours to request")
    parser.add_argument("--create_static", action="store_true")
    parser.add_argument("--create_dynamic", action="store_true")
    parser.add_argument("--create_async", action="store_true")
    parser.add_argument("--save_pts_separately", action="store_true",
            help="If selected, each patient's data will be saved to a " \
                    "separate file")

    args=parser.parse_args()
    configs=vars(args)

    timegrid_all_patients(configs)


import pandas as pd

from pandas import DataFrame

from matplotlib import pyplot as plt

import seaborn as sns

from fpdf import FPDF

from algorithm import knn,naive,SVM,nn

import time

def details(data,df,newcolumn,image):

    pdf = FPDF(orientation='P', unit='mm',format='A4')

    pdf.add_page()

    pdf.set_font("Arial", size=12)

    epw = pdf.w - 2 * pdf.l_margin

    col_width = epw / 4

    pdf.set_font('Times', 'B', 35)

    pdf.cell(epw, 0.0, 'Drioditor', align='C')

    pdf.set_font('Times', '', 10.0)

    pdf.cell(200, 10, txt="\n", ln=1)

    pdf.cell(200, 10, txt="\n", ln=1)

    pdf.set_font('Times', 'B', 20)

    pdf.cell(epw, 0.0, 'Recommendation Report', align='C')

    pdf.set_font('Times', '', 10.0)

    pdf.ln(0.5)

    column = len(data)

    row = len(data[0])

    pdf.cell(200, 10, txt="\n", ln=1)

    pdf.cell(200, 10, txt="\n", ln=1)

    pdf.cell(200, 10, txt="Rows : {}".format(row), ln=1)

    pdf.cell(200, 10, txt="Columns : {}".format(column), ln=1)

    pdf.cell(200, 10, txt="Missing Values : {}".format(0), ln=1)

    avg=df.mean()

    median=df.median()

    mode=df.mode()[:1]

    pdf.set_font('Times', 'B', 14.0)

    pdf.set_font('Times', '', 10.0)

    pdf.ln(0.5)

    temp=[]

    temp.append("Feature \t" + "Average\t" +"Median\t" + "Mode\n" )

    for i in range(len(newcolumn)):

        temp.append(str(newcolumn[i])+"\t"+str(round(avg[i],5)) + "\t" + str(median[i])+"\t"+str(mode[i][0])+"\n")

    #print(temp)

    th = pdf.font_size

    """

    for row in temp:

        for datum in row.split(','):

            pdf.cell(col_width, th, str(datum), border=1)

        pdf.ln(th)

    pdf.ln(4 * th)

    pdf.set_font('Times', 'B', 14.0)

    pdf.cell(epw, 0.0, 'With more padding', align='C')

    pdf.set_font('Times', '', 10.0)

    """

    for row in temp:

        for datum in row.split(','):

            pdf.cell(col_width, 2 * th, str(datum), border=1)

        pdf.ln(2 * th)

    pdf.add_page()

    pdf.set_font('Times', 'B', 14)

    pdf.cell(epw, 0.0, 'Correlation Heatmap', align='C')

    pdf.set_font('Times', '', 10.0)

    pdf.ln(0.5)

    pdf.image('{}'.format(image),x=10,y=50)

    pdf.add_page()

    pdf.set_font('Times', 'B', 14)

    pdf.cell(epw, 0.0, 'Algorithm ', align='C')

    pdf.set_font('Times', '', 10.0)

    pdf.ln(0.5)

    max_accuracy,accuracy_name=0,""

    min_time, time_name=1000,""

    t=time.time()

    accuracy,tnr,tpr = knn('{}'.format(filename))

    t=time.time() -t

    if max_accuracy<accuracy :

        max_accuracy=accuracy

        accuracy_name='Knn'

    if min_time>t:

        min_time=t

        time_name='Knn'

    pdf.set_font('Times', 'B', 12)

    pdf.cell(200, 10, txt="Knn", ln=1)

    pdf.set_font('Times', '', 10.0)

    pdf.cell(210, 10, txt="Accuracy : {}".format(accuracy), ln=1)

    pdf.cell(210, 10, txt="TNR : {}".format(tnr), ln=1)

    pdf.cell(210, 10, txt="TPR : {}".format(tpr), ln=1)

    pdf.cell(210, 10, txt="Time Taken : {}".format(t), ln=1)

    #print(accuracy,tnr,tpr,t)

    t = time.time()

    accuracy, tnr, tpr = naive('{}'.format(filename))

    t = time.time() - t

    if max_accuracy<accuracy :

        max_accuracy=accuracy

        accuracy_name='Naive Bayes'

    if min_time>t:

        min_time=t

        time_name='Naive Bayes'

    pdf.set_font('Times', 'B', 12)

    pdf.cell(200, 10, txt="Naive Bayes", ln=1)

    pdf.set_font('Times', '', 10.0)

    pdf.cell(210, 10, txt="Accuracy : {}".format(accuracy), ln=1)

    pdf.cell(210, 10, txt="TNR : {}".format(tnr), ln=1)

    pdf.cell(210, 10, txt="TPR : {}".format(tpr), ln=1)

    pdf.cell(210, 10, txt="Time Taken : {}".format(t), ln=1)

    #print(accuracy, tnr, tpr, t)

    t = time.time()

    accuracy, tnr, tpr = SVM('{}'.format(filename))

    t = time.time() - t

    if max_accuracy<accuracy :

        max_accuracy=accuracy

        accuracy_name='SVM'

    if min_time>t:

        min_time=t

        time_name='SVM'

    pdf.set_font('Times', 'B', 12)

    pdf.cell(200, 10, txt="SVM", ln=1)

    pdf.set_font('Times', '', 10.0)

    pdf.cell(210, 10, txt="Accuracy : {}".format(accuracy), ln=1)

    pdf.cell(210, 10, txt="TNR : {}".format(tnr), ln=1)

    pdf.cell(210, 10, txt="TPR : {}".format(tpr), ln=1)

    pdf.cell(210, 10, txt="Time Taken : {}".format(t), ln=1)

    #print(accuracy, tnr, tpr, t)

    t = time.time()

    accuracy, tnr, tpr = nn('{}'.format(filename))

    t = time.time() - t

    if max_accuracy<accuracy :

        max_accuracy=accuracy

        accuracy_name='Neural Network'

    if min_time>t:

        min_time=t

        time_name='Neural Network'

    pdf.set_font('Times', 'B', 12)

    pdf.cell(200, 10, txt="Neural Network", ln=1)

    pdf.set_font('Times', '', 10.0)

    pdf.cell(210, 10, txt="Accuracy : {}".format(accuracy), ln=1)

    pdf.cell(210, 10, txt="TNR : {}".format(tnr), ln=1)

    pdf.cell(210, 10, txt="TPR : {}".format(tpr), ln=1)

    pdf.cell(210, 10, txt="Time Taken : {}".format(t), ln=1)

    #print(accuracy, tnr, tpr, t)

    pdf.add_page()

    pdf.set_font('Times', 'B', 14)

    pdf.cell(epw, 0.0, 'Recommendation', align='C')

    pdf.set_font('Times', '', 10.0)

    pdf.ln(0.5)

    pdf.cell(200, 10, txt="Based on Accuracy    : {} with accuracy {}%".format(accuracy_name,max_accuracy*100), ln=1)

    pdf.cell(200, 10, txt="Based on Time        : {} with time taken {}s".format(time_name,round(min_time,6)), ln=1)

    #print(max_accuracy,accuracy_name)

    #print(min_time,time_name)

    pdf.output('data/output.pdf', 'F')

def updateDataset(check, data):
    temp=[]
    for i in range(len(check)):
        if check[i] == 1:
            temp.append(data[i])
    return temp

def updateColumn(check, column):
    temp=[]
    for i in range(len(check)):
        if check[i] == 1:
            temp.append(column[i])
    return temp

def checkData(data,length):
    temp=[]
    for i in range(length):
        try:
            float(data[i][0])
            temp.append(1)
        except :
            temp.append(0)
    return temp

def convertData(data):
    temp=[]
    length = len(data[0].split(","))
    for i in range(length):
        test=[]
        for j in range(len(data)):
            test.append(data[j].split(',')[i])
        temp.append(test)
    return temp

if __name__ == '__main__':

    filename='data/iris.csv'
    f=open('{}'.format(filename) ,'r')
    data= f.readlines()
    f.close()

    column=data[0].split(",")
    column[-1] =column[-1].replace('\n','')
    data=data[1:]
    data=convertData(data)
    check = checkData(data,len(column))
    newColumn = updateColumn(check,column)
    newData = updateDataset(check,data)
    df = DataFrame(newData)
    df= df.transpose()
    df= df.apply(pd.to_numeric)
    correlation = df.corr()
    sns.heatmap(correlation,xticklabels=newColumn,yticklabels=newColumn)
    plt.savefig('data/test.jpg')
    details(data,df,newColumn,'data/test.jpg')


#!/usr/bin/env python

"""
File: request_to_C42.py

A module for talking to the C42 API.

The GetRequest class can be used to send GET requests to the API.
Other classes can be added in the future for increased functionality (PUT, HEAD, etc).

The get_events_with_subscriptions method does exactly what we want:
send a couple of GET requests and filter and combine the results from them.

Every GET request to the API should be handled by a new instance of the GetRequest class,
which takes a EVENT_ID and a authentication TOKEN as arguments.
"""

import requests

__author__ = 'Pablo Barham'

class GetRequest:

    def __init__(self, token, event_id):
        self.TOKEN = token
        self.EVENT_ID = event_id

        authorization_string = 'Token {}'.format(self.TOKEN)
        self.headers = {
            'Accept': 'application/json',
            'Content-type': 'application/json',
            'Authorization': authorization_string
        }

    def get_events_with_subscriptions(self):
        # Send two GET requests to the C42 API, and filter and combine them
        print("Calling the Calendar42 API...")
        title, status_code = self.get_title()
        first_names = self.get_first_names()

        if status_code == 200:
            json_to_return = {
                'id': self.EVENT_ID,
                'title': title,
                'first_names': first_names
            }

        elif status_code == 401:
            json_to_return = {
                'error': {
                    'status_code': 401,
                    'message': 'A 401: Unauthorized error occurred when attempting to contact the C42 API. '
                               'Authentication credentials were not valid'
                }
            }
        elif status_code == 404:
            json_to_return = {
                'error': {
                    'status_code': 404,
                    'message': 'A 404: Not Found error occurred when attempting to contact the C42 API. '
                               'The EVENT_ID you requested does not exist'
                }
            }
        else:
            json_to_return = {
                'error': {
                    'status_code': status_code,
                    'message': 'Something strange occurred when attempting to contact the C42 API. '
                               'This was completely unexpected!'
                }
            }

        return json_to_return

    # Get event details (title)
    def get_title(self):
        url = 'https://demo.calendar42.com/api/v2/events/{}/'.format(self.EVENT_ID)

        try:
            r = requests.get(url, headers=self.headers)
            # Make HTTP status errors raise an exception
            r.raise_for_status()

            r_json = r.json()
            title = r_json['data'][0]['title']

            return title, r.status_code

        except requests.exceptions.RequestException as e:
            print('{} occurred while attempting to connect to {}'.format(e, url))
            return 'Could not be retrieved', r.status_code

    # Get event subscriptions (first names of participants)
    def get_first_names(self):
        url = 'https://demo.calendar42.com/api/v2/event-subscriptions/?event_ids=[{}]'.format(self.EVENT_ID)

        try:
            r = requests.get(url, headers=self.headers)
            # Make HTTP status errors raise an exception
            r.raise_for_status()

            r_json = r.json()
            number_of_subscribers = r_json['meta_data']['count']
            first_names = [r_json['data'][n]['subscriber']['first_name'] for n in range(number_of_subscribers)]

            return first_names

        except requests.exceptions.RequestException as e:
            print('{} occurred while attempting to connect to {}'.format(e, url))
            return 'Could not be retrieved'


from scipy import single,linalg
from matriz_laplaciana import matriz_laplaciana
from time import perf_counter
import matplotlib.pyplot as plt

#TAMAÑO MATRICES
Ns=[2,5,10,12,15,20,30,40,45,50,55,60,75,100,125,160,
    200,250,350,500,600,800,1000,2000,5000,10000]

#CREANDO ARCHIVO DE TEXTO
dts=[]
mem=[]
name=("timing_caso_2_single.txt")
fid=open(name,"w")

for N in Ns:

    print(f"N = {N}")

    A=matriz_laplaciana(N,single)

    t1=perf_counter()
    C=linalg.inv(A,overwrite_a=False)
    t2=perf_counter()

    dt=t2-t1

    size=2*(N**2)*8

    dts.append(dt)
    mem.append(size)

    fid.write(f"{N} {dt} {size}\n")

    print(f"Tiempo transcurrido = {dt} s")
    print(f"Memoria usada = {size} bytes")

    fid.flush()

fid.close()

#CURVA
plt.figure()

tamano=[]
tiempo=[]
memoria=[]

f = open(name, "r")

for i in f:
    lista_fila=i.split()
    tamano.append(int(lista_fila[0]))
    tiempo.append(float(lista_fila[1]))
    memoria.append(int(lista_fila[2]))
f.close()

plt.subplot(2,1,1)

plt.plot(tamano,tiempo,marker="o",color="green")
plt.loglog(tamano,tiempo,marker="o",color="green")

xTicks = [10,20,50,100,200,500,1000,2000,5000,10000,20000]
xTicks_Text = ["","","","","","","","","","",""]

yTicks = [10**-4,10**-3,10**-2,10**-1,1,10,60,600]
yTicks_Text = ["0.1 ms","1 ms","10 ms","0.1 s","1 s","10 s","1 min","10min"]

plt.yticks(yTicks, yTicks_Text)
plt.xticks(xTicks, xTicks_Text,rotation=45)

plt.title("Desempeño INV", fontsize=10)
plt.ylabel("Tiempo transcurrido")

plt.grid(True)

plt.subplot(2,1,2)

plt.plot(tamano, memoria,marker="o",color="green")
plt.loglog(tamano,memoria,marker="o",color="green")

xTicks = [10,20,50,100,200,500,1000,2000,5000,10000,20000]
xTicks_Text = ["10","20","50","100","200","500","1000",
                "2000","5000","10000","20000"]

yTicks = [10**3,10**4,10**5,10**6,10**7,10**8,10**9,10**10]
yTicks_Text = ["1 KB","10 KB","100 KB","1 MB","10 MB","100 MB","1 GB","10 GB"]

plt.yticks(yTicks, yTicks_Text)
plt.xticks(xTicks, xTicks_Text,rotation=45)

plt.xlabel("Tamaño matriz N")
plt.ylabel("Uso memoria")

plt.axhline(y=384000,color='k',ls="--")
plt.axhline(y=2*10**6,color='k',ls="--")
plt.axhline(y=4*10**6,color='k',ls="--")

plt.grid(True)

plt.tight_layout()

plt.show()


from django.conf import settings
from django.db import transaction

from box.shop.item.models import *
from box.shop.item.api.serializers import ItemSerializer

from io import StringIO
import pandas as pd
import csv

BASE_DIR = settings.BASE_DIR

count = 0

def get_family_tree(child):
  subcategories = child.subcategories.all()
  if not subcategories:
      return {
        "title": child.title,
        "slug":child.slug,
        "items_count":"",
        "subcategories": []
      }
  return {
      "title": child.title,
      "slug":child.slug,
      "subcategories": [get_family_tree(child) for child in subcategories],
  }

def get_subcategories_count(child):
  subcategories = child.subcategories.all()
  if not subcategories:
      return {
        "title": child.title,
        "slug":child.slug,
        "items_count":"",
        "subcategories": []
      }
  return {
      "title": child.title,
      "slug":child.slug,
      "subcategories": [get_family_tree(child) for child in subcategories],
  }

def read_items_from_xlsx(filename):
  items      = pd.read_excel(filename, sheet_name="Товары")
  items      = items.to_csv()
  categories = pd.read_excel(filename, sheet_name="Категории")
  categories = categories.to_csv()
  create_categories(
    [dct for dct in map(dict, csv.DictReader(StringIO(categories)))],
    list(csv.reader(StringIO(categories)))
  )
  create_items(
    [dct for dct in map(dict, csv.DictReader(StringIO(items)))],
    list(csv.reader(StringIO(items)))
  )
  return True

read_items_from_xlsx_admin = read_items_from_xlsx

def read_items_from_csv(filename):
  create_items(
    [dct for dct in map(dict, csv.DictReader(open(filename)))],
    list(csv.reader(open(filename))),
  )
  return True

def read_categories_from_csv(filename):
  create_categories(
    [dct for dct in map(dict, csv.DictReader(open(filename)))],
    list(csv.reader(open(filename))),
  )
  return True

def create_items(dict_file, list_file):
  items       = dict_file
  headers_row = list_file[0]
  items_rows  = list_file[1:]
  feature_name_indexes  = [i for i, o in enumerate(headers_row) if "Название_Характеристики" in o]
  feature_value_indexes = [i for i, o in enumerate(headers_row) if "Значение_Характеристики" in o]
  # feature_code_indexes  = [i for i, o in enumerate(headers_row) if "feature_code" in o]
  items_features = []
  item_features_names  = []
  item_features_values = []
  # item_features_codes  = []
  features = []
  for item_row in items_rows:
    for i in feature_name_indexes:
      item_features_names.append(item_row[i])
    for i in feature_value_indexes:
      item_features_values.append(item_row[i])
    # for i in feature_code_indexes:
    #   item_features_codes.append(item_row[i])
    # TODO: допиляти так, шоб можна було в ексель-файл записувати
    # крім назви характеристики і значення характеристики, ще й код
    # характеристики і батьківську категорію
    # for i in range(len(item_features_names)):
    #   features.append({
    #     "feature_name": ,
    #     "":,
    #     "":,
    #     "":,
    #   })
    features = dict(zip(item_features_names, item_features_values))
    items_features.append(features)

  for i in range(len(items_features)):
    items[i]["features"] = items_features[i]

  for item in items:
      # new_item.in_stock    = item["В_Наличии"]
      # new_item.is_new      = item["Новое"]
      # new_item.is_active   = item["Активен"]
      category_slug = item["Категория"].lower()
      item_slug     = item["Ссылка"]
      new_item, _          = Item.objects.get_or_create(
        slug = item_slug,
      )
      slugs = ItemCategory.objects.all().values_list('slug')
      if settings.MULTIPLE_CATEGORY:
        new_item.categories.add(ItemCategory.objects.get(slug__iexact = category_slug))
      else:
        new_item.category = ItemCategory.objects.get(slug__iexact=category_slug)

      new_item.meta_descr  = item["Мета_Описание"]
      new_item.meta_title  = item["Мета_Заголовок"]
      new_item.meta_key    = item["Мета_Ключевые_Слова"]
      new_item.title       = item["Заголовок"]
      new_item.description = item["Описание"]
      new_item.code        = item["Артикул"]
      new_item.old_price   = item["Старая_Цена"]
      new_item.new_price   = item["Новая_Цена"]
      currency             = item['Валюта']
      new_item.currency, _ = Currency.objects.get_or_create(name=currency)

      images = item["Изображения"]
      if images:
        images = images.split(",")
        for image in images:
          # print(image)
          image = ItemImage.objects.create(
            image = f"shop/items/{image.strip()}",
            item  =  new_item,
          )
        new_item.save()
        new_item.create_thumbnail_from_images()

      for k, v in item["features"].items():
        new_feature, _ = ItemFeature.objects.get_or_create(
          item = new_item,
          name =k,
          value=v,
        )
        # print(new_feature.id, new_feature.name)
      new_item.save()
      # print(new_item.id ,new_item.title)

def create_categories(dict_file, list_file):
  for item in dict_file:
    title       = item["title"]
    slug        = item["slug"]
    parent_slug = item["parent"]
    image       = item['image']
    image_name  = image.split('/')[-1]

    with transaction.atomic():
      new_category, _ = ItemCategory.objects.get_or_create(
        slug = slug,
      )
      new_category.title  = title
      parent      = ItemCategory.objects.filter(slug=parent_slug).first()
      new_category.parent = parent
      new_category.thumbnail = 'shop/categories/' + image
      new_category.save()
      print(new_category)
  return True

def import_items_from_xml_by_xmljson(filename):
  from xmljson import badgerfish as bf
  from xml.etree.ElementTree import fromstring
  import json
  items = '
Hellobold
'
  items = open(filename).read()
  items = bf.data(fromstring(items))
  items = json.dumps(items)
  items = json.loads(items)
  items = items['objects']['object']
  for item in items:
    fields = item['field']
    import pprint; pprint.pprint(fields)

class XMLItem(object):

  meta_title  = None
  meta_descr  = None
  meta_key    = None
  title       = None
  description = None
  code        = None
  slug        = None
  thumbnail   = None
  old_price   = None
  new_price   = None
  currency    = None
  category    = None
  in_stock    = None
  is_new      = None
  is_active   = None
  created     = None
  updated     = None
  order       = None
  fields = {}

  def __str__(self):
    return f'{self.fields}'

def import_items_from_xml_by_xml_etree(filename):
  import xml.etree.ElementTree as ET
  # *******
  # tree = ET.parse(filename)
  # xml_data = tree.getroot()
  # xmlstr = ET.tostring(xml_data, encoding='utf8', method='xml')
  # data_dict = dict(xmltodict.parse(xmlstr))
  # print(data_dict)
  # with open('new_data_2.json', 'w+') as json_file:
  #     json.dump(data_dict, json_file, indent=4, sort_keys=True)
  # *******
  # https://stackoverflow.com/questions/45144645/parsing-xml-file-in-django

  tree = ET.parse(filename)
  root = tree.getroot()
  for att in root:
    fields = att.findall('field')#.text
    item = Item()

    xml_item = XMLItem()
    for field in fields:
      name  = field.get('name')
      descr = field.get('descr')
      text  = field.text
      xml_item.fields.update({name:text})
      # setattr(xml_item, name, text)
    print(xml_item)
    # item.save()

def import_items_from_xml_by_xmltodict(filename):
  import xmltodict
  import json

  items = xmltodict.parse(open(filename).read())
  items = json.dumps(items)
  items = json.loads(items)
  items = items['objects']['object']
  for item in items:
    fields = item['field']
    for field in fields:
      item = Item()
      text = field.get('#text')
      name = field['@name']
      setattr(item, name, text)
      # item.save()
      print(item)
      import pprint; pprint.pprint(field)

def import_items_from_xml(filename):
  import_items_from_xml_by_xml_etree(filename)
  return True

def export_items_to_xml():
  from django.core import serializers
  # https://github.com/vinitkumar/json2xml
  # https://github.com/quandyfactory/dicttoxml/tree/master/dist
  # !!!!
  # https://github.com/quandyfactory/dicttoxml
  # dicttoxml + rest_framework.serializers.ModelSerializer
  # https://stackoverflow.com/questions/36021526/converting-an-array-dict-to-xml-in-python
  # !!!!
  items = Item.objects.all()
  result = "<objects>"
  for item in items:
    result += form_item(item)
  result += "</objects>"
  # result  = serializers.serialize('xml', items)
  with open('items.xml', 'w') as f:
    f.write(result)
  return True

def form_item(item):
  item = f"""
  <object>
    <field name="meta_title" descr="{item._meta.get_field('meta_title').verbose_name}">
      {item.meta_title}
    </field>
    <field name="meta_descr" descr="{item._meta.get_field('meta_descr').verbose_name}">
      {item.meta_descr}
    </field>
    <field name="meta_key" descr="{item._meta.get_field('meta_key').verbose_name}">
      {item.meta_key}
    </field>
    <field name="title" descr="{item._meta.get_field('title').verbose_name}">
      {item.title}
    </field>
    <field name="description" descr="{item._meta.get_field('description').verbose_name}">
      {item.description}
    </field>
    <field name="code" descr="{item._meta.get_field('code').verbose_name}">
      {item.code}
    </field>
    <field name="slug" descr="{item._meta.get_field('slug').verbose_name}">
      {item.slug}
    </field>
    <field name="thumbnail" descr="{item._meta.get_field('thumbnail').verbose_name}">
      {item.thumbnail}
    </field>
    <field name="old_price" descr="{item._meta.get_field('old_price').verbose_name}">
      {item.old_price}
    </field>
    <field name="new_price" descr="{item._meta.get_field('new_price').verbose_name}">
      {item.new_price}
    </field>
    <field name="currency" descr="{item._meta.get_field('currency').verbose_name}">
      {item.currency}
    </field>
    <field name="category" descr="{item._meta.get_field('category').verbose_name}">
      {item.category}
    </field>
    <field name="in_stock" descr="{item._meta.get_field('in_stock').verbose_name}">
      {item.in_stock}
    </field>
    <field name="is_new" descr="{item._meta.get_field('is_new').verbose_name}">
      {item.is_new}
    </field>
    <field name="is_active" descr="{item._meta.get_field('is_active').verbose_name}">
      {item.is_active}
    </field>
    <field name="created" descr="{item._meta.get_field('created').verbose_name}">
      {item.created}
    </field>
    <field name="updated" descr="{item._meta.get_field('updated').verbose_name}">
      {item.updated}
    </field>
    <field name="order" descr="{item._meta.get_field('order').verbose_name}">
      {item.order}
    </field>
  </object>
  """
  return item


# Main Code

import cv2

src = cv2.imread('./data/insect_damage.jpg', cv2.IMREAD_COLOR)

dst = src.copy()
roi = src[100:200, 200:400]
dst[0:100, 0:200] = roi

cv2.imshow('src', src)
cv2.imshow('dst', dst)
cv2.waitKey(0)
cv2.destroyAllWindows()


import sys
import torch
import numpy as np

from tools.ai import torch_utils
from tools.general import io_utils, time_utils

class Evaluator:
    def __init__(self, model, loader, class_names, task, amp):
        self.amp = amp
        self.task = task

        self.model = model
        self.loader = loader

        if self.task == 'multi-labels':
            self.evaluator = Evaluator_For_Multi_Label_Classification(class_names)
        else:
            self.evaluator = Evaluator_For_Mean_Accuracy(class_names)

        self.eval_timer = time_utils.Timer()
        self.num_iterations = len(self.loader)

    def step(self, detail=False):
        self.model.eval()
        self.eval_timer.tik()

        with torch.no_grad():
            ni_digits = io_utils.get_digits_in_number(self.num_iterations)

            for i, (images, labels) in enumerate(self.loader):
                i += 1
                progress_format = '\r# Evaluation [%0{}d/%0{}d] = %02.2f%%'.format(ni_digits, ni_digits)

                sys.stdout.write(progress_format%(i, self.num_iterations, i / self.num_iterations * 100))
                sys.stdout.flush()

                if torch.cuda.is_available():
                    images = images.cuda()
                    labels = labels.cuda()

                with torch.cuda.amp.autocast(enabled=self.amp):
                    logits = self.model(images)

                if self.task == 'multi-labels':
                    preds = torch.sigmoid(logits)
                else:
                    preds = torch.max(logits, 1)[1]

                preds = torch_utils.get_numpy_from_tensor(preds)
                labels = torch_utils.get_numpy_from_tensor(labels)

                for i in range(images.size()[0]):
                    self.evaluator.add(preds[i], labels[i])
        print('\r', end='')

        self.model.train()
        return self.evaluator.get(detail=detail), self.eval_timer.tok(clear=True)

class Evaluator_For_Mean_Accuracy:
    def __init__(self, class_names):
        self.class_names = class_names
        self.num_classes = len(self.class_names)

        self.clear()

    def add(self, pred, label):
        self.accuracy_list[label].append(pred == label)

    def get(self, detail=False, clear=True):
        accuracy_dict = {name:np.mean(self.accuracy_list[label]) * 100 for name, label in zip(self.class_names, range(self.num_classes))}
        mean_accuracy = np.mean(list(accuracy_dict.values()))

        if clear:
            self.clear()

        if detail:
            return mean_accuracy, accuracy_dict
        else:
            return mean_accuracy

    def clear(self):
        self.accuracy_list = [[] for _ in range(self.num_classes)]

class Evaluator_For_Multi_Label_Classification:
    def __init__(self, class_names, th_interval=0.1):
        self.thresholds = list(np.arange(0.01, 1.00, th_interval))

        self.class_names = class_names
        self.num_classes = len(self.class_names)

        self.clear()

    def add(self, pred, gt):
        for th in self.thresholds:
            binary_pred = (pred >= th).astype(np.float32)

            self.meter_dic[th]['P'] += binary_pred
            self.meter_dic[th]['T'] += gt
            self.meter_dic[th]['TP'] += (gt * (binary_pred == gt)).astype(np.float32)

    def get(self, detail=False, clear=True):
        op_list = []
        or_list = []
        o_f1_list = []

        cp_list = []
        cr_list = []
        c_f1_list = []

        for th in self.thresholds:
            data = self.meter_dic[th]

            P = data['P']
            T = data['T']
            TP = data['TP']

            overall_precision = np.sum(TP) / (np.sum(P) + 1e-5) * 100
            overall_recall = np.sum(TP) / (np.sum(T) + 1e-5) * 100
            overall_f1_score = 2 * ((overall_precision * overall_recall) / (overall_precision + overall_recall + 1e-5))

            op_list.append(overall_precision)
            or_list.append(overall_recall)
            o_f1_list.append(overall_f1_score)

            per_class_precision = np.mean(TP / (P + 1e-5)) * 100
            per_class_recall = np.mean(TP / (T + 1e-5)) * 100
            per_class_f1_score = 2 * ((per_class_precision * per_class_recall) / (per_class_precision + per_class_recall + 1e-5))

            cp_list.append(per_class_precision)
            cr_list.append(per_class_recall)
            c_f1_list.append(per_class_f1_score)

        best_index = np.argmax(o_f1_list)
        best_o_th = self.thresholds[best_index]

        best_op = op_list[best_index]
        best_or = or_list[best_index]
        best_of = o_f1_list[best_index]

        best_index = np.argmax(c_f1_list)
        best_c_th = self.thresholds[best_index]

        best_cp = cp_list[best_index]
        best_cr = cr_list[best_index]
        best_cf = c_f1_list[best_index]

        if clear:
            self.clear()

        if detail:
            return [best_c_th, best_cp, best_cr, best_cf], cp_list, cr_list, c_f1_list
        else:
            return best_cf

    def clear(self):
        self.meter_dic = {
            th : {
                'P':np.zeros(self.num_classes, dtype=np.float32),
                'T':np.zeros(self.num_classes, dtype=np.float32),
                'TP':np.zeros(self.num_classes, dtype=np.float32)
            } for th in self.thresholds}

class Evaluator_For_mIoU:
    def __init__(self, class_names, ignore_index=255):
        self.class_names = class_names
        self.num_classes = len(self.class_names)

        self.ignore_index = ignore_index

        self.clear()

    def clear(self):
        self.TP = []
        self.P = []
        self.T = []

        for _ in range(self.classes):
            self.TP.append(0)
            self.P.append(0)
            self.T.append(0)

    def get_data(self, pred_mask, gt_mask):
        obj_mask = gt_mask<self.ignore_index
        correct_mask = (pred_mask==gt_mask) * obj_mask

        P_list, T_list, TP_list = [], [], []
        for i in range(self.num_classes):
            P_list.append(np.sum((pred_mask==i)*obj_mask))
            T_list.append(np.sum((gt_mask==i)*obj_mask))
            TP_list.append(np.sum((gt_mask==i)*correct_mask))

        return (P_list, T_list, TP_list)

    def add_using_data(self, data):
        P_list, T_list, TP_list = data
        for i in range(self.num_classes):
            self.P[i] += P_list[i]
            self.T[i] += T_list[i]
            self.TP[i] += TP_list[i]

    def add(self, pred_mask, gt_mask):
        obj_mask = gt_mask<self.ignore_index
        correct_mask = (pred_mask==gt_mask) * obj_mask

        for i in range(self.num_classes):
            self.P[i] += np.sum((pred_mask==i)*obj_mask)
            self.T[i] += np.sum((gt_mask==i)*obj_mask)
            self.TP[i] += np.sum((gt_mask==i)*correct_mask)

    def get(self, detail=False, clear=True):
        IoU_dic = {}
        IoU_list = []

        FP_list = [] # over activation
        FN_list = [] # under activation

        for i in range(self.num_classes):
            IoU = self.TP[i]/(self.T[i]+self.P[i]-self.TP[i]+1e-10) * 100
            FP = (self.P[i]-self.TP[i])/(self.T[i] + self.P[i] - self.TP[i] + 1e-10)
            FN = (self.T[i]-self.TP[i])/(self.T[i] + self.P[i] - self.TP[i] + 1e-10)

            IoU_dic[self.class_names[i]] = IoU

            IoU_list.append(IoU)
            FP_list.append(FP)
            FN_list.append(FN)

        mIoU = np.mean(np.asarray(IoU_list))
        mIoU_foreground = np.mean(np.asarray(IoU_list)[1:])

        FP = np.mean(np.asarray(FP_list))
        FN = np.mean(np.asarray(FN_list))

        if clear:
            self.clear()

        if detail:
            return mIoU, mIoU_foreground, IoU_dic, FP, FN
        else:
            return mIoU, mIoU_foreground


from otree.api import (
	models,
	widgets,
	BaseConstants,
	BaseSubsession,
	BaseGroup,
	BasePlayer,
	Currency as c,
	currency_range,
)
import numpy as np
import pandas as pd
import json
import _static.design_evaluator.python as truss_model
import matplotlib.pyplot as plt
import PIL.Image as Image
import warnings
import os
from pygmo import hypervolume
from scipy.stats import rankdata

warnings.filterwarnings('ignore')

author = 'Ashish Chaudhari'

doc = """
Backend and frontend of a human-AI collaborative tool for metamaterial design
"""

class Constants(BaseConstants):
	name_in_url = 'outreach_activity'
	players_per_group = None
	num_rounds = 1
	num_features = 5
	num_ticks = 100

	#Mechanical metamaterial related variables
	E = 0.7e09; # Young's Modulus for polymeric material (example: 10000 Pa)
	sel = 0.01; # Unit square side length (NOT individual truss length) (example: 5 cm)
	r = 5e-4 #element radius in m
	edgelist = np.array([[1,2], [1,6], [1,5], [1,4], [1,8], [2,3], [2,6], [2,5], [2,4], [2,7],
					[2,9], [3,6], [3,5], [3,4], [3,8], [5,6], [6,7], [6,8], [6,9], [4,5],
					[5,7], [5,8], [5,9], [4,7], [4,8], [4,9], [7,8], [8,9]])-1;
	nucFac = 1;
	sidenum = (2*nucFac) + 1;
	pos = truss_model.generateNC(sel, sidenum);
	target_c_ratio = 1; # Ratio of C22/C11 to target
	ref_point = [0, 1] #Order of objectives [Vertical Stifness, Volume Fraction]
	score_magnifier = 1
	objectives = ['Vertical Stiffness', 'Volume Fraction']
	constraints = ['Feasibility', 'Material']
	materials = {#Young's Modulus
		'Polyethylene HDPE': 0.8e09,
		'PVC': 4e09,
		'Aluminium 6061-T6': 69e09,
		'Copper': 117e09,
		'Steel ASTM-A36': 200e09
	}
	radii = {
		'0.2 mm': 0.0002,
		'0.3 mm': 0.0003,
		'0.4 mm': 0.0004
	}

	data_keys = ['Main']

class Subsession(BaseSubsession):

	def get_goals(self):
		return ['Maximize', 'Minimize']

	def get_instruction_url(self):
		instruction_url= Constants.name_in_url + "/" + "instructions_automation.html"
		return instruction_url

	def nodes(self):
		pos = Constants.pos
		sel = Constants.sel
		nodes = list()
		for i, p in enumerate(pos):
			nodes.append({'id':int(i), 'x':float(p[0]/sel), 'y':float(p[1]/sel)})
		return nodes

	def edges(self):
		edgelist = Constants.edgelist
		nodes = self.nodes()
		edges = list()
		for i in range(len(edgelist)):
			e = edgelist[i]
			edges.append({'id': str(e[0])+'-'+str(e[1]), 'index':i, 'source':nodes[e[0]], 'target':nodes[e[1]]})
		return edges

	def is_pareto_efficient(self, costs):
		"""
		Find the pareto-efficient points
		:param costs: An (n_points, n_costs) array
		:return: A (n_points, ) boolean array, indicating whether each point is Pareto efficient
		"""
		is_efficient = np.ones(costs.shape[0], dtype = bool)
		for i, c in enumerate(costs):
			is_efficient[i] = np.all(np.any(costs[:i]>=c, axis=1)) and np.all(np.any(costs[i+1:]>=c, axis=1))
		return is_efficient

	def convert_to_img(self, x, r):
		pos = Constants.pos
		edgelist = Constants.edgelist
		max_r = max(Constants.radii.values())
		min_r = min(Constants.radii.values())
		lw = 20*np.array(r)/max_r

		plt.ioff()
		fig, ax = plt.subplots()
		truss_model.show_meshplot(pos, edgelist, x, lw, ifMatrix=False, ax=ax)
		fig.savefig('image.png', bbox_inches='tight')
		img = Image.open('image.png').convert('L').resize((50,50))
		x = np.array(img)<255

		return x.astype('int')

	def creating_session(self):
		if self.round_number == 1:
			cwd = os.getcwd()
			# Reading data #Columns = design stiffness	volume_fraction	feasibility	stability vertical_lines horizontal_lines	diagonals	triangles	three_stars	image
			data = pd.read_csv("./_static/metamaterial_designs_normalized_filtered1.csv")
			data = data.drop_duplicates(subset=['obj1', 'obj2'], ignore_index=True)
			data = data.sort_values(['obj1', 'obj2'], ascending=[True, True], ignore_index=True)
			n_data = data.shape[0]

			# Adding pareto front information
			costs = data[['obj1', 'obj2']].values*np.array([[-1, 1]]) #Maximize stiffness and minimize volume fraction
			# Check if feasibility constraint is satisfied
			is_constr1 = data['constr1'] == 1
			ret = self.is_pareto_efficient(costs[is_constr1])	# Pareto front of the historic data
			data['is_pareto'] = np.zeros((n_data,)).astype(bool)
			data['is_pareto'][is_constr1] = ret
			for p in self.get_players():
				player_id = p.id_in_subsession
				data['is_pareto_response'+str(player_id)] = data['is_pareto']	# Pareto front after considering new responses evaluated by the designer
			data['id'] = np.arange(data.shape[0])
			self.session.vars = data.to_dict('list')

			#Storing the reference hypervolume
			ref_hv = hypervolume(costs[is_constr1]).compute(Constants.ref_point)
			self.session.vars['reference_hv'] = ref_hv

			#Initialize initial ranks and scores
			for g in self.get_groups():
				g.update_scoreboard(initialize=True)

class Group(BaseGroup):

	def update_scoreboard(self, initialize=False):
		players = self.get_players()

		if not initialize:
			#Calculate scores
			#First approach
			scores = []
			for p in players:
				if 'hv' not in p.participant.vars.keys():
					scores.append(self.session.vars['reference_hv'])
				else:
					scores.append(p.participant.vars['hv'])
			#Second approach
			# scores = [sum(p.participant.vars['is_pareto']*p.participant.vars['is_unique']) for p in players]
			#third approach
			# is_pareto = p.participant.vars['is_pareto']
			# objs = np.array(p.participant.vars[['obj1', 'obj2']])[is_pareto]
			# utopia = 1 - np.array(ref_point)[None,:]
			# scores = np.mean(np.sqrt(np.sum((objs-utopia)**2, axis=1)), axis=1)

			names = []
			for p in players:
				if p.user_firstname:
					names.append(p.user_firstname + ' ' + p.user_lastname)
				else:
					names.append(str(p.id_in_subsession))

		else:
			#First approach
			ref_hv = self.session.vars['reference_hv']
			scores = [Constants.score_magnifier*ref_hv for p in players]
			names = [str(p.id_in_subsession) for p in players]
			#Third approach
			# scores = [0 for p in players]

		ranks = rankdata(-1*np.array(scores), method='min')

		#Store the current ranks and scores
		scoreboard = dict()
		for i in range(len(players)):
			scoreboard[players[i].id_in_subsession] = dict(
						rank=int(ranks[i]),
						score=scores[i],
						name=names[i])
		self.session.vars['scoreboard'] = scoreboard

class Player(BasePlayer):

	user_firstname = models.TextField()
	user_lastname = models.TextField()

	def update_pareto(self):
		# Historic dataset
		data = np.array([self.session.vars.get(key) for key in ['obj1', 'obj2']]).T #dictionary
		n_data = data.shape[0]

		# Gather old responses
		prev_response = np.array([self.participant.vars.get(key) for key in ['obj1', 'obj2']]).T  #dictionary
		n_response = prev_response.shape[0]

		# Gathering objectives data
		tem1 = data
		tem2 = prev_response
		costs = np.vstack([tem1, tem2])*np.array([[-1, 1]]) #Maximize stiffness and minimize volume fraction
		# Constraint Feasibility=1 should be satisfied
		if_constr1 = np.array([v==1 for v in self.session.vars['constr1']] + [v==1 for v in self.participant.vars['constr1']])
		# Find new pareto and update databse
		ret = self.subsession.is_pareto_efficient(costs[if_constr1])
		is_pareto = np.zeros((n_data+n_response,)).astype(bool)
		is_pareto[if_constr1] = ret

		# self.session.vars['is_pareto_new'] = ret[0:n_data].tolist()
		self.participant.vars['is_pareto'] =  is_pareto[n_data:].tolist()
		self.session.vars['is_pareto_response' + str(self.id_in_subsession)] =  is_pareto[0:n_data].tolist()

		#Calculate the hypervolume improvement
		hv = hypervolume(costs[if_constr1]).compute(Constants.ref_point)
		self.participant.vars['hv'] = np.around(hv,3)

		#Store boolean on whether designs are unique
		_, indices = np.unique(costs, axis=0, return_index=True)
		is_unique = np.zeros(n_data+n_response, dtype=bool)
		is_unique[indices] = True
		self.participant.vars['is_unique'] = is_unique[n_data:].tolist()
		self.session.vars['is_unique' + str(self.id_in_subsession)] =  is_unique[0:n_data].tolist()

		return is_pareto[0:n_data].tolist()

	def update_response_database(self, response):
		# Gather old responses
		prev_response = self.participant.vars
		for key in response.keys():
			if key not in prev_response:
				prev_response[key]=[]
			prev_response[key].append(response[key])

	def live_method(self, data):

		if data['message']=='clean image':
			#For cleaning data
			x = np.array(data['x'])
			r = data['r']#Must in meters
			x[x>0.25]=1
			x[x<0.25]=0
			x_img = self.subsession.convert_to_img(x, r)
			image = x_img.tolist()

			return {self.id_in_group: dict(message='clean image',image=image)}

		elif data['message']=='test design':
			#Load data even if it may be empty
			# x is a bitstring design vector
			# z is a feature vector
			x = np.array(data['x'])
			x[x>0.25]=1
			x[x<0.25]=0

			r = data['r']#Must in meters
			E = data['E']#Must be in Pa

			if np.any(np.isin(x, ['', None, 'nan'])):
				x = np.ones(Constants.edgelist.shape[0])

			# Running the design evaluator and creating a response
			x = truss_model.repeatable_des(x)
			r = truss_model.repeatable_des(r)
			obj1, obj2 = truss_model.multiobjectives(x, Constants.sidenum, Constants.sel, E, r, Constants.edgelist, 1)
			edges_des = Constants.edgelist[x.astype(bool)]
			constr1 = truss_model.feasibility(Constants.pos, edges_des, Constants.sidenum)
			constr2 = [key for key, value in Constants.materials.items() if value==E]
			x_img = self.subsession.convert_to_img(x, r)
			image = json.dumps(x_img.tolist())
			design = json.dumps(x.tolist())

			response = dict(design_bitstring=design, obj1=obj1, obj2=obj2, constr1=constr1, constr2=constr2, image=image,
			is_pareto=None)
			# response.update(data)

			# Check if any element is int32
			for k, v in response.items():
				if isinstance(v, np.int32) or isinstance(v, np.int64):
					response[k] = int(v)

			# Update databse and Check if pareto
			self.update_response_database(response)

			# Update pareto information; Returns pareto information for historic data
			is_pareto_response = self.update_pareto()

			# Retrive updated data rom database
			response_data = self.participant.vars

			#scoreboard
			self.group.update_scoreboard()
			scoreboard = self.session.vars['scoreboard']
			print(scoreboard)
			return {self.id_in_group: dict(message='test design',response_data=response_data, is_pareto_response=is_pareto_response, scoreboard=scoreboard)}

def custom_export(players):
	# header row
	ctr=0
	for p in players:
		data = p.participant.vars
		if data:
			if ctr==0:
				keys = list(data.keys())
				yield ['session', 'participant_code'] + keys
				ctr+=1
			n_data = len(data[keys[0]])

			for i in range(n_data):
				row = [p.session.code, p.participant.code]
				row = row + [data[k][i] for k in keys]
				yield row


from itertools import combinations

card_num, target_sum = map(int, input().split())
card_list = list(map(int, input().split()))
biggest_sum = 0

for cards in combinations(card_list, 3):
    temp_sum = sum(cards)
    if biggest_sum < temp_sum <= target_sum:
        biggest_sum = temp_sum

print(biggest_sum)

# mylist[2,3,4,5,6]
# newlist = [i for i in mylist if i%2==0]
# newlist 출력시 [2,4,6] 만 원소로 출력


'''

Hi man, thought I'd start with a joke to make your day...
but the only programming joke that I know is my code,
please enjoy the joke. :")

'''

import os
import glob
import json

print("\n---------------MERGE JSON FILES----------------\n")

def merge_json(path, ip_fname, op_fname, max_size):

	os.chdir(path) 					#navigate to the directory

	i = 1
	size = 0
	result = []
	result_dict = {}

	for json_file in glob.glob(ip_fname + '*.json'):

		json_opened = open(json_file, encoding='utf-8')
		logs = json.load(json_opened)

		json_opened.seek(0,2)
		fsize = json_opened.tell()		#obtain file size

		key = list(logs)				#obtain the root key of the json file

		if fsize <= max_size:			#check if the file is less than max_size
			if size + fsize <= max_size:
				result.extend(logs[key[0]])
			else:
				result_dict[key[0]] = result[:]
				result.extend(logs[key[0]])
				outfile = open(op_fname + str(i) +'.json', 'w')
				json.dump(result_dict, outfile, ensure_ascii=False)
				i += 1
				size = 0
				result_dict = {}
				result = []
				result.extend(logs[key[0]])

			size += fsize

		else:							#if file size is greater than max_size, file cannot be written and skipped automatically
			print("file "+ json_file + " is too large to write in a new file, try increasing the max size")

	if size <= max_size:
		result_dict[key[0]] = result[:]
		outfile = open(op_fname + str(i) +'.json', 'w')
		json.dump(result_dict, outfile, ensure_ascii=False)

#main
try:
	path = str(input("Enter path: "))
	input_base_name = str(input("Enter input json file name suffix: "))
	output_base_name = str(input("Enter the output json file name suffix: "))
	max_size = int(input("Enter the maximum file size(in bytes): "))
	merge_json(path, input_base_name, output_base_name, max_size)
	print("Successfully merged...")

except:
	print("Error occured, try entering valid parameters...")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
from pprint import pprint

from topic_extraction import score_keyphrases_by_textrank
import sys
from collections import defaultdict

def main(csv_file):
    # load CSV
    # create corpus of text for each rank
    # extract terms for each rank
    corpus = defaultdict(str)

    with open(csv_file, 'r') as csvfile:
        reader = csv.reader(csvfile)
        header = reader.__next__()
        for row in reader:
            row_data = dict(zip(header, row))
            corpus[row_data['rating']] += row_data['title'] + " " + row_data['body']

    analysis = {}
    for k, v in corpus.items():
        analysis[int(float(k))] = score_keyphrases_by_textrank(v)

    pprint(analysis)

if __name__=="__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("csv_file")
    args = parser.parse_args()
    main(args.csv_file)


#This incredible, rarted game was programmed by the one and only, Boyd Kirkman.
#Feel free to update/add to the game!

import sys
import random
import math
import pygame
import pygame.locals
from time import sleep

width = 1280
height = 720

pygame.init()

#~~~~~~~~~~~~~~~~~~~~~~~~~~Loading in Sounds/Music~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

global triple
global damnson
global gotcha
global boomheadshot
global screech
global sounds
global MKMLG
triple = pygame.mixer.Sound("triple.wav")
damnson = pygame.mixer.Sound("damnson.wav")
gotcha = pygame.mixer.Sound("gotcha.wav")
boomheadshot = pygame.mixer.Sound("boomheadshot.wav")
screech = pygame.mixer.Sound("screech.wav")
sounds = [triple, damnson, boomheadshot]
MKMLG = pygame.mixer.music.load("MKMLG.mp3")

#~~~~~~~~~~~~~~~~~~~~~~~~~~~NO LEAVING MECHANIC~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def No_Leaving():

    otherdoge = pygame.image.load('No.png')
    R = otherdoge.get_rect()
    X = width/2 - R.center[0] + 20
    if pygame.QUIT in [e.type for e in pygame.event.get()]:
            screen = pygame.display.set_mode((width,height))
            font = pygame.font.SysFont("freeserif", 72, bold = 1)
            textSurface = font.render("Don't Leave!", 1,
                                      pygame.Color(255,4,4))
            screen.blit(textSurface,(width/2-R.center[0], height/2-R.center[1]))
            while True:
                #Angry Doge:
                screen.blit(otherdoge, (X, height/2 - R.center[1]))
                pygame.display.update()
                if pygame.QUIT in [e.type for e in pygame.event.get()]:
                    break

#~~~~~~~~~~~~~~~~~~~~~~~~~Setting up the Doge Sprite~~~~~~~~~~~~~~~~~~~~~~~~~~~~

class Doge(pygame.sprite.Sprite):

    def __init__(self):
        pygame.sprite.Sprite.__init__(self)

        #the skin variable is what determines how the doge looks (what image gets loaded)
        global skin
        skin = "run1.png"
        self.image = pygame.image.load(skin).convert_alpha()
        self.rect = self.image.get_rect()

        #Orientation
        global Orient
        Orient = 0

        #Trying to animate a gif
        global blip
        blip = 0
        global image_changing_score
        image_changing_score = 0

    def update(self):

        #trying to animate a gif + Dealing with orientation
        global Orient
        global blip
        blip += 1
        global image_changing_score
        if blip == 5:
            image_changing_score += 1
            blip = 0
        if image_changing_score % 2 == 0 and Orient == 0:
            skin = "moving?.png"
        if image_changing_score % 2 == 0 and Orient == 1:
            skin = "flippedmoving?.png"
        if image_changing_score % 2 != 0 and Orient == 0:
            skin = "run1.png"
        if image_changing_score % 2 != 0 and Orient == 1:
            skin = "flippedrun1.png"
        self.image = pygame.image.load(skin).convert_alpha()

        #Exiting via a mouse click on the doge
        x,y = pygame.mouse.get_pos()
        if x >= self.rect.x and x <= self.rect.x + 139:
            Mouse_Location_X = True
        if x < self.rect.x or x > self.rect.x + 139:
            Mouse_Location_X = False
        if y >= self.rect.y and y <= self.rect.y + 92:
            Mouse_Location_Y = True
        if y < self.rect.y or y > self.rect.y + 92:
            Mouse_Location_Y = False
        if pygame.mouse.get_pressed()[0] and Mouse_Location_X == True and Mouse_Location_Y == True:
            pygame.quit()

        #No flying
        if self.rect.y < 340:
            self.rect.y += 20

        #Moving
        if pygame.key.get_pressed()[pygame.K_RIGHT] and self.rect.x < 1170:
            if score == 0:
                self.rect.x += 10
            if score != 0 and score <= 4:
                self.rect.x += score*10
            if score > 4:
                self.rect.x += 60
            Orient = 0
        if pygame.key.get_pressed()[pygame.K_LEFT] and self.rect.x > 0:
            if score == 0:
                self.rect.x -= 10
            if score != 0 and score <= 4:
                self.rect.x -= score*10
            if score > 4:
                self.rect.x -= 60
            Orient = 1
        if pygame.key.get_pressed()[pygame.K_UP] and self.rect.y > 340:
            if score == 0:
                self.rect.y -= 10
            if score != 0 and score <= 4:
                self.rect.y -= score*10
            if score > 4:
                self.rect.y -= 60
        if pygame.key.get_pressed()[pygame.K_DOWN] and self.rect.y < 640:
            if score == 0:
                self.rect.y += 10
            if score != 0 and score <= 4:
                self.rect.y += score*10
            if score > 4:
                self.rect.y += 60

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Doritos~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

class Doritos(pygame.sprite.Sprite):
    def __init__(self):
        pygame.sprite.Sprite.__init__(self)
        self.image = pygame.image.load("Doritos.png").convert_alpha()
        self.rect = self.image.get_rect()

    def it_be_spawning(self):
        self.rect.x = random.randrange(1,1170)
        self.rect.y = random.randrange(340,640)

    def oh_no_it_be_eaten(self):
        global score
        if pygame.sprite.collide_mask(doge, doritos):
            pygame.mixer.Sound.play(random.choice(sounds))
            self.rect.x = random.randrange(1,1170)
            self.rect.y = random.randrange(340,640)
            score += 1

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~MLG Elmo~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

class MLG_Elmo(pygame.sprite.Sprite):
    def __init__(self):
        pygame.sprite.Sprite.__init__(self)
        self.image = pygame.image.load("Elmo_a_coma_for_ya.png").convert_alpha()
        self.rect = self.image.get_rect()

    def did_elmo_get_ya(self):
        if self.rect.x > doge.rect.x:
            self.rect.x += -1*score
        if self.rect.x < doge.rect.x:
            self.rect.x += 1*score
        if self.rect.y > doge.rect.y:
            self.rect.y += -1*score
        if self.rect.y < doge.rect.y:
            self.rect.y += 1*score
        if pygame.sprite.collide_mask(self, doge):
            game_over()

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~The Sun~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

class sun(pygame.sprite.Sprite):
    def __init__(self):
        self.image = pygame.image.load("Sun.png").convert_alpha()
        self.rect = self.image.get_rect()

    def the_sun_turns(self):
        if self.rect.x > doge.rect.x:
            self.rect.x += -3
        if self.rect.x < doge.rect.x:
            self.rect.x += 3
        if self.rect.y > doge.rect.y:
            self.rect.y += -3
        if self.rect.y < doge.rect.y:
            self.rect.y += 3
        if pygame.sprite.collide_mask(self, doge):
            game_over()

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Restart Button~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

class revive_button(pygame.sprite.Sprite):
    def __init__(self):
        self.image = pygame.image.load("revive_button_untouched.png").convert_alpha()
        self.rect = self.image.get_rect()

    def sensing_click(self):
        x,y = pygame.mouse.get_pos()
        if x >= width/2-50 and x <= width/2+50:
            Mouse_Location_X = True
        if x < width/2-50 or x > width/2+50:
            Mouse_Location_X = False
        if y >= 570 and y <= 620:
            Mouse_Location_Y = True
        if y < 570 or y > 620:
            Mouse_Location_Y = False

        #button functions
        if pygame.mouse.get_pressed()[0] and Mouse_Location_X == True and Mouse_Location_Y == True:
            Level_1()
        if Mouse_Location_X == True and Mouse_Location_Y == True:
            self.image = pygame.image.load("revive_button_touched.png").convert_alpha()
        else:
            self.image = pygame.image.load("revive_button_untouched.png").convert_alpha()

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Title Screen~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def GameIntro():
    pygame.mixer.music.play(-1)
    intro_doge = pygame.image.load('doge.jpg')
    r = intro_doge.get_rect()
    direction = 2
    X = width/2 - r.center[0]

    window = pygame.display.set_mode((width,height))
    pygame.display.set_caption("MUCH WOW")

    intro_happening = True
    while intro_happening:
        window.blit(pygame.image.load('background.jpg'),(0,0))

        #DOGE
        window.blit(intro_doge, (X, -100))

        #Moving Doge
        X += direction
        if X >= width - r.width or X <= 0:
            direction *= -1

        #Title
        ffont = pygame.font.SysFont("freeserif", 190, bold = 1)
        ttextSurface = ffont.render("MUCH WOW", 1, pygame.Color(255,255,255))
        tt = ttextSurface.get_rect()
        window.blit(ttextSurface, (width/2 - tt.width/2 + 10, height/15))
        pygame.display.update()

        #Buttons
        def W_button():
            Font = pygame.font.SysFont("freesansbold", 100, bold = 1)
            TextSurface = Font.render("PLAY", 1, pygame.Color(255,255,255))
            T = TextSurface.get_rect()
            window.blit(TextSurface, (width/2 - T.width/2, height/2 - T.height/2))
            pygame.display.update()
        def B_button():
            Font = pygame.font.SysFont("freesansbold", 100, bold = 1)
            TextSurface = Font.render("PLAY", 1, pygame.Color(0,0,0))
            T = TextSurface.get_rect()
            window.blit(TextSurface, (width/2 - T.width/2, height/2 - T.height/2))
            pygame.display.update()

        #Mouse Location
        x,y = pygame.mouse.get_pos()
        if x >= width/2-50 and x <= width/2+50:
            Mouse_Location_X = True
        if x < width/2-50 or x > width/2+50:
            Mouse_Location_X = False
        if y >= height/2-30 and y <= height/2+30:
            Mouse_Location_Y = True
        if y < height/2-30 or y > height/2+30:
            Mouse_Location_Y = False

        #button functions
        if pygame.mouse.get_pressed()[0] and Mouse_Location_X == True and Mouse_Location_Y == True:
            Level_1()

        elif Mouse_Location_X == True and Mouse_Location_Y == True:
            B_button()

        else:
            W_button()

        No_Leaving()
        pygame.display.update()

#~~~~~~~~~~~~~~~~~~Level 1 (I'll add more later, hopefully...)~~~~~~~~~~~~~~~~~~

def Level_1():
    intro_happening = False
    global score
    score = 0
    Level1 = pygame.display.set_mode((width,height))

    #Set Doge location
    global doge
    doge = Doge()
    doge.rect.x = width/2-doge.rect.center[0]
    doge.rect.y = height/2-doge.rect.center[1]

    #Spawn Doritos
    global doritos
    doritos = Doritos()
    doritos.it_be_spawning()

    #Spawning in the Sun
    global sun_lol
    sun_lol = sun()
    sun_lol.rect.x = 1080
    sun_lol.rect.y = 100

    #Elmo's on his way!
    global elmo
    elmo = MLG_Elmo()
    elmo.rect.x = 50
    elmo.rect.y = 50

    #this is jsut a variable so that elmo screeches on entry
    random_var = 0

    global game_in_play
    game_in_play = True
    while game_in_play:

        #Background
        Level1.blit(pygame.image.load('Background_lol.jpg').convert(),(0,0))

        #keeping score
        font = pygame.font.SysFont("freeserif", 72, bold = 1)
        textSurface = font.render(str(score), 1,
                                  pygame.Color(0,0,0))
        Level1.blit(textSurface,(50, 50))

        #Create Doge Sprite
        doge.update()
        Level1.blit(doge.image, doge.rect)

        #Spawn the Doritos
        doritos.oh_no_it_be_eaten()
        Level1.blit(doritos.image, doritos.rect)

        #The Sun
        if score >= 10:
            sun_lol.the_sun_turns()
        Level1.blit(sun_lol.image, sun_lol.rect)

        #The Triggering of MLG Elmo
        elmo.did_elmo_get_ya()
        if score >= 1:
            if random_var == 0:
                pygame.mixer.Sound.play(screech)
                random_var = 1
            Level1.blit(elmo.image, elmo.rect)

        #No leaving Mechanic
        No_Leaving()

        #Update screen
        pygame.display.update()

#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Game Over~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def game_over():
    pygame.mixer.Sound.stop(triple)
    pygame.mixer.Sound.stop(boomheadshot)
    pygame.mixer.Sound.stop(screech)
    pygame.mixer.Sound.stop(damnson)
    pygame.mixer.Sound.play(gotcha)
    global game_in_play
    game_in_play = False

    #The Revive Button
    button = revive_button()
    button.rect.x = width/2-50
    button.rect.y = 570
    while True:
        No_Leaving()
        window2 = pygame.display.set_mode((width,height))

        #picture of dead doge
        dd = pygame.image.load("dead_doge.png").convert_alpha()
        dd_dimensions = dd.get_rect()
        window2.blit(dd, (width/2 - dd_dimensions.height/2, height/2 - dd_dimensions.height/2))

        #The game over message
        Font = pygame.font.SysFont("freesansbold", 100, bold = 1)
        number_of_stolen_doritos = str(score)
        TextSurface = Font.render(f"You stole {number_of_stolen_doritos} doritos!", 1, pygame.Color(255,255,255))
        T = TextSurface.get_rect()
        window2.blit(TextSurface, (width/2 - T.width/2, 50))

        #The game over message cont.
        lelFont = pygame.font.SysFont("freesansbold", 50, bold = 1)
        lelTextSurface = lelFont.render("But died...", 1, pygame.Color(255,25,25))
        lelT = lelTextSurface.get_rect()
        window2.blit(lelTextSurface, (width/2 - lelT.width/2, 150))

        #The revive Button
        button.sensing_click()
        window2.blit(button.image, button.rect)

        #updating the window
        pygame.display.update()

while __name__ == '__main__':
    GameIntro()


#! /usr/bin/env python3

"""
	python3 has many standards data strctures inbuilt and sometimes these are very useful.
"""

#deque -> doubly ended queue.

#insertion and removal operations are faster.
# deque -> remove O(1)
# python list -> remove O(n)

from collections import deque
q = deque()

q.append('eat')
q.append('sleep')
q.append('code')

q   # output -> deque(['eat', 'sleep', 'code'])

q.popleft() # deque -> ['sleep', 'code']


#  Copyright (c) maiot GmbH 2020. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at:
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
#  or implied. See the License for the specific language governing
#  permissions and limitations under the License.

import os

import pytorch_lightning as pl
import torch
from pytorch_lightning import Trainer
from torch.nn import functional as F

from zenml.steps.trainer import TorchFeedForwardTrainer
from zenml.steps.trainer import utils
from zenml.utils import path_utils

class MyPyTorchLightningTrainer(TorchFeedForwardTrainer):
    """
    PyTorch Lightning trainer
    """

    def run_fn(self):
        train_split_patterns = [self.input_patterns[split] for split in
                                self.split_mapping[utils.TRAIN_SPLITS]]
        train_dataset = self.input_fn(train_split_patterns)

        eval_split_patterns = [self.input_patterns[split] for split in
                               self.split_mapping[utils.EVAL_SPLITS]]
        eval_dataset = self.input_fn(eval_split_patterns)

        class LitModel(pl.LightningModule):
            def __init__(self):
                super().__init__()
                self.l1 = torch.nn.Linear(8, 64)
                self.layer_out = torch.nn.Linear(64, 1)

            def forward(self, x):
                x = torch.relu(self.l1(x))
                x = self.layer_out(x)
                return x

            def training_step(self, batch, batch_idx):
                x, y, z = batch
                x_batch = torch.cat(list(x.values()), dim=-1)
                y_t = torch.cat(list(y.values()), dim=-1)
                y_hat = self(x_batch)
                loss = F.binary_cross_entropy_with_logits(y_hat, y_t)
                tensorboard_logs = {'train_loss': loss}
                return {'loss': loss, 'log': tensorboard_logs}

            def configure_optimizers(self):
                return torch.optim.Adam(self.parameters(), lr=0.001)

            def train_dataloader(self):
                return train_dataset

            def validation_step(self, batch, batch_idx):
                x, y, z = batch
                x_batch = torch.cat(list(x.values()), dim=-1)
                y_t = torch.cat(list(y.values()), dim=-1)
                y_hat = self(x_batch)
                return {
                    'val_loss': F.binary_cross_entropy_with_logits(y_hat, y_t)}

            def validation_epoch_end(self, outputs):
                avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
                tensorboard_logs = {'val_loss': avg_loss}
                return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}

            def val_dataloader(self):
                return eval_dataset

        model = LitModel()

        # most basic trainer, uses good defaults
        trainer = Trainer(
            default_root_dir=self.log_dir,
            max_epochs=self.epochs,
        )
        trainer.fit(model)

        path_utils.create_dir_if_not_exists(self.serving_model_dir)
        if path_utils.is_remote(self.serving_model_dir):
            temp_model_dir = '__temp_model_dir__'
            temp_path = os.path.join(os.getcwd(), temp_model_dir)
            if path_utils.is_dir(temp_path):
                raise PermissionError('{} is used as a temp path but it '
                                      'already exists. Please remove it to '
                                      'continue.')
            trainer.save_checkpoint(os.path.join(temp_path, 'model.cpkt'))
            path_utils.copy_dir(temp_path, self.serving_model_dir)
            path_utils.rm_dir(temp_path)
        else:
            trainer.save_checkpoint(
                os.path.join(self.serving_model_dir, 'model.ckpt'))

        # test
        for split in self.split_mapping[utils.TEST_SPLITS]:
            pattern = self.input_patterns[split]
            test_dataset = self.input_fn([pattern])
            test_results = self.test_fn(model, test_dataset)
            utils.save_test_results(test_results, self.output_patterns[split])


import numpy as np

import jax

import pytest

from scico.linop import DFT
from scico.random import randn
from scico.test.linop.test_linop import adjoint_AAt_test, adjoint_AtA_test

class TestDFT:
    def setup_method(self, method):
        self.key = jax.random.PRNGKey(12345)

    @pytest.mark.parametrize("input_shape", [(32,), (32, 48)])
    @pytest.mark.parametrize("pad_output", [True, False])
    @pytest.mark.parametrize("jit", [False, True])
    def test_eval(self, input_shape, pad_output, jit):
        if pad_output:
            output_shape = (48, 64)[: len(input_shape)]
        else:
            output_shape = None

        x, key = randn(input_shape, dtype=np.complex64, key=self.key)
        F = DFT(input_shape=input_shape, output_shape=output_shape, jit=jit)
        Fx = F @ x

        # In the future we can compare against snp.fft.fftn,
        # but at present it does not support the "s" argument
        if output_shape is None:
            s = None
        else:
            s = np.array(output_shape)

        np_result = np.fft.fftn(x.copy(), s=s).astype(np.complex64)

        np.testing.assert_allclose(Fx, np_result, rtol=1e-4)

    @pytest.mark.parametrize("input_shape", [(32,), (32, 48)])
    @pytest.mark.parametrize("pad_output", [True, False])
    @pytest.mark.parametrize("jit", [False, True])
    def test_adjoint(self, input_shape, pad_output, jit):
        if pad_output:
            output_shape = (48, 64)[: len(input_shape)]
        else:
            output_shape = None

        F = DFT(input_shape=input_shape, output_shape=output_shape, jit=jit)
        adjoint_AtA_test(F, self.key)
        adjoint_AAt_test(F, self.key)

    @pytest.mark.parametrize("input_shape", [(32,), (32, 48)])
    @pytest.mark.parametrize("pad_output", [True, False])
    @pytest.mark.parametrize("truncate", [True, False])
    def test_inv(self, input_shape, pad_output, truncate):
        if pad_output:
            output_shape = (48, 64)[: len(input_shape)]
        else:
            output_shape = None

        F = DFT(input_shape=input_shape, output_shape=output_shape)

        y, key = randn(F.output_shape, dtype=np.complex64, key=self.key)

        Fi_y = F.inv(y, truncate=truncate)
        if truncate:
            assert Fi_y.shape == input_shape
            np_result = np.fft.ifftn(y.copy())

            for i, s in enumerate(input_shape):
                np_result = np.take(np_result, indices=np.r_[:s], axis=i)
            np.testing.assert_allclose(Fi_y.copy(), np_result, rtol=1e-4)
        else:
            np_result = np.fft.ifftn(y.copy())
            np.testing.assert_allclose(Fi_y.copy(), np_result, rtol=1e-4)

    def test_length_mismatch(self):
        input_shape = (32, 48)
        output_shape = (32,)
        with pytest.raises(ValueError):
            F = DFT(input_shape=input_shape, output_shape=output_shape)


from pathlib import Path
import coloredlogs
import datetime
import sys
import logging

def setup_logging(name, level=logging.INFO):
    ts = datetime.datetime.utcnow().strftime('%Y%m%d_%H-%M-%S')
    log_path = Path(f'../logs/irc_bot_{ts}.log')
    log_path.parent.mkdir(exist_ok=True)
    logging.basicConfig(
        level=level,
        format='[{%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
        handlers=[
            logging.FileHandler(filename=log_path),
            logging.StreamHandler(sys.stdout)
        ]
    )
    coloredlogs.install(level=level)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from copy import deepcopy

import matplotlib.pyplot as plt
import numpy as np
from scipy.linalg import norm

from scipy.io import loadmat

from pyvib.common import db
from pyvib.forcing import multisine
from pyvib.frf import covariance
from pyvib.nlss import NLSS
from pyvib.nonlinear_elements import (NLS, Nonlinear_Element,
                                      Tanhdryfriction, Unilatteralspring)
from pyvib.signal import Signal
from pyvib.subspace import Subspace
from pyvib.fnsi import FNSI

"""This tutorial shows the work flow of modeling a single input single output
(SISO) polynomial nonlinear state-space (PNLSS) model.

The tutorial shows how to
1) generate a multisine signal
2) compute the response of a SISO PNLSS model to the multisine input
3) estimate a nonparametric linear model from the input and noisy output data
4) estimate a parametric linear state-space model on the nonparametric model
5) estimate the parameters in the full PNLSS model

Translation of the tutorial provided with the matlab PNLSS program[1]_

[1]_
http://homepages.vub.ac.be/~ktiels/pnlss.html
"""

# save figures to disk
savefig = False
add_noise = False
weight = False
p = 2

## Generate data from true model ##
# Construct model to estimate
A = np.array([[0.73915535, -0.62433133],[0.6247377, 0.7364469]])
B = np.array([[0.79287245], [-0.34515159]])
C = np.array([[0.71165154, 0.34917771]])
D = np.array([[0.04498052]])
if p == 2:
    C = np.vstack((C,C))
    D = np.vstack((D,0.1563532))

Ffull = np.array([[-0.00867042, -0.00636662]])
Efull = np.array([[1.88130305e-01, -2.70291900e-01],
                  [-5.35196110e-01,-3.66250013e-01]])

if p == 1:
    Wy = [1]
    Wt = [1]
elif p ==2:
    Wt = [0,1]

F = np.array([])
nly = None

# for identification
tahn1 = Tanhdryfriction(eps=0.1, w=Wt)
nlx = [tahn1]
E = 1e0*Efull[:,:len(nlx)]

true_model = FNSI(A, B, C, D, E, F)
true_model.add_nl(nlx=nlx, nly=nly)

# excitation signal
RMSu = 0.05   # Root mean square value for the input signal
npp = 2048    # Number of samples
R = 3         # Number of phase realizations (one for validation and one for
              # testing)
P = 3         # Number of periods
kind = 'Odd'  # 'Full','Odd','SpecialOdd', or 'RandomOdd': kind of multisine
m = D.shape[1]         # number of inputs
p = C.shape[0]         # number of outputs
fs = 1        # normalized sampling rate
Ntr = 2
if True:
    # get predictable random numbers. https://dilbert.com/strip/2001-10-25
    np.random.seed(10)
    # shape of u from multisine: (R,P*npp)
    u, lines, freq, t = multisine(N=npp, P=P, R=R, lines=kind, rms=RMSu)
    # if multiple input is required, this will copy u m times

    # Transient: Add one period before the start of each realization. To generate
    # steady state data.
    T1 = np.r_[npp*Ntr, np.r_[0:(R-1)*P*npp+1:P*npp]]
    _, yorig, _ = true_model.simulate(u.ravel(), T1=T1)
    print(norm(yorig))
    u = u.reshape((R,P,npp)).transpose((2,0,1))[:,None]  # (npp,m,R,P)
    y = yorig.reshape((R,P,npp,p),order='C').transpose((2,3,0,1))

    #y = yorig.reshape((R,P,npp)).transpose((2,0,1))[:,None]
    # or in F order:
    # y2 = yorig.reshape((npp,P,R,p),order='F').transpose((0,3,2,1))

    # Add colored noise to the output. randn generate white noise
    if add_noise:
        np.random.seed(10)
        noise = 1e-3*np.std(y[:,-1,-1]) * np.random.randn(*y.shape)
        # Do some filtering to get colored noise
        noise[1:-2] += noise[2:-1]
        y += noise

    ## START of Identification ##
    # partitioning the data. Use last period of two last realizations.
    # test for performance testing and val for model selection
    utest = u[:,:,-1,-1]
    ytest = y[:,:,-1,-1]
    uval = u[:,:,-2,-1]
    yval = y[:,:,-2,-1]
    # all other realizations are used for estimation
    uest = u[...,0,:][:,:,None]
    yest = y[...,0,:][:,:,None]
    # noise estimate over periods. This sets the performace limit for the estimated
    # model
    covY = covariance(yest)
    Pest = yest.shape[-1]

    # create signal object
    sig = Signal(uest,yest,fs=fs)
    sig.lines = lines
    # plot periodicity for one realization to verify data is steady state
    #sig.periodicity()
    # Calculate BLA, total- and noise distortion. Used for subspace identification
    sig.bla()
    # average signal over periods. Used for training of PNLSS model
    um, ym = sig.average()

# model orders and Subspace dimensioning parameter
nvec = [2,3]
maxr = 5

if 'linmodel' not in locals() or True:
    Rest = yest.shape[2]
    T1 = np.r_[npp*Ntr, np.r_[0:(Rest-1)*npp+1:npp]]

    linmodel = FNSI()
    linmodel.set_signal(sig)
    linmodel.estimate(n=2, r=5, weight=weight)

    linmodel.transient(T1)
    # linmodel._cost_normalize = 1
    linmodel.optimize(weight=weight)

    fnsi1 = FNSI()
    fnsi1.set_signal(sig)
    fnsi1.add_nl(nlx=nlx)
    fnsi1.estimate(n=2, r=5, weight=weight)
    fnsi1.transient(T1)
    #fnsi1.optimize(lamb=100, weight=weight, nmax=25)

    print(f"Best subspace model, n, r: {linmodel.n}, {linmodel.r}")
    # linmodel_orig = linmodel

if False:  # dont scan subspace
    linmodel = Subspace(sig)
    # get best model on validation data
    models, infodict = linmodel.scan(nvec, maxr, weight=weight)
    l_errvec = linmodel.extract_model(yval, uval)
    # or estimate the subspace model directly
    linmodel.estimate(2, 5, weight=weight)  # best model, when noise weighting is used
    linmodel.optimize(weight=weight)
    print(f"Best subspace model, n, r: {linmodel.n}, {linmodel.r}")

# linmodel = deepcopy(linmodel_orig)
# estimate PNLSS
# transient: Add one period before the start of each realization. Note that
# this is for the signal averaged over periods
Rest = yest.shape[2]
T1 = np.r_[npp*Ntr, np.r_[0:(Rest-1)*npp+1:npp]]

nlx2 = NLS([tahn1])
nly2 = None

model = NLSS(linmodel)
model._cost_normalize = 1
model.add_nl(nlx=nlx2, nly=nly2)
model.set_signal(sig)
model.transient(T1)
model.optimize(lamb=100, weight=weight, nmax=25)

#raise SystemExit(0)

# get best model on validation data. Change Transient settings, as there is
# only one realization
nl_errvec = model.extract_model(yval, uval, T1=npp*Ntr)

models = [linmodel, model, fnsi1]
descrip = [type(mod).__name__ for mod in models]
descrip = tuple(descrip)  # convert to tuple for legend concatenation in figs
# simulation error
val = np.empty((*yval.shape, len(models)))
est = np.empty((*ym.shape, len(models)))
test = np.empty((*ytest.shape, len(models)))
for i, model in enumerate(models):
    test[...,i] = model.simulate(utest, T1=npp*Ntr)[1]
    val[...,i] = model.simulate(uval, T1=npp*Ntr)[1]
    est[...,i] = model.simulate(um, T1=T1)[1]

# convenience inline functions
stack = lambda ydata, ymodel: \
    np.concatenate((ydata[...,None], (ydata[...,None] - ymodel)),axis=2)
rms = lambda y: np.sqrt(np.mean(y**2, axis=0))
est_err = stack(ym, est)  # (npp*R,p,nmodels)
val_err = stack(yval, val)
test_err = stack(ytest, test)
noise = np.abs(np.sqrt(Pest*covY.squeeze()))
print(f"err for models {descrip}")
print(f'rms error noise: {rms(noise)}\tdb: {db(rms(noise))} ')
print(f'rms error est:  \n{rms(est_err)}   \ndb: \n{db(rms(est_err))}')
print(f'rms error val:  \n{rms(val_err)}   \ndb: \n{db(rms(val_err))}')
print(f'rms error test: \n{rms(test_err)}  \ndb: \n{db(rms(test_err))}')

## Plots ##
# store figure handle for saving the figures later
figs = {}

# linear and nonlinear model error; plot for each output
for pp in range(p):
    plt.figure()
    plt.plot(est_err[:,pp])
    plt.xlabel('Time index')
    plt.ylabel('Output (errors)')
    plt.legend(('Output',) + descrip)
    plt.title(f'Estimation results p:{pp}')
    figs['estimation_error'] = (plt.gcf(), plt.gca())

    # result on validation data
    N = len(yval)
    freq = np.arange(N)/N*fs
    plottime = val_err
    plotfreq = np.fft.fft(plottime, axis=0)/np.sqrt(N)
    nfd = plotfreq.shape[0]
    plt.figure()
    plt.plot(freq[lines], db(plotfreq[lines,pp]), '.')
    plt.plot(freq[lines], db(np.sqrt(Pest*covY[lines,pp,pp].squeeze() / N)), '.')
    plt.xlabel('Frequency')
    plt.ylabel('Output (errors) (dB)')
    plt.legend(('Output',) + descrip + ('Noise',))
    plt.title(f'Validation results p:{pp}')
    figs['val_data'] = (plt.gcf(), plt.gca())

    # result on test data
    N = len(ytest)
    freq = np.arange(N)/N*fs
    plottime = test_err
    plotfreq = np.fft.fft(plottime, axis=0)/np.sqrt(N)
    nfd = plotfreq.shape[0]
    plt.figure()
    plt.plot(freq[:nfd//2], db(plotfreq[:nfd//2,pp]), '.')
    plt.plot(freq[:nfd//2], db(np.sqrt(Pest*covY[:nfd//2,pp,pp].squeeze() / N)), '.')
    plt.xlabel('Frequency')
    plt.ylabel('Output (errors) (dB)')
    plt.legend(('Output',) + descrip + ('Noise',))
    plt.title(f'Test results p:{pp}')
    figs['test_data'] = (plt.gcf(), plt.gca())

    try:
        # BLA plot. We can estimate nonlinear distortion
        # total and noise distortion averaged over P periods and M realizations
        # total distortion level includes nonlinear and noise distortion
        plt.figure()
        # When comparing distortion(variance, proportional to power) with
        # G(propertional to amplitude(field)), there is two definations for dB:
        # dB for power: Lp = 10 log10(P).
        # dB for field quantity: Lf = 10 log10(F²)
        # Alternative calc: bla_noise = db(np.abs(sig.covGn[:,pp,pp])*R, 'power')
        # if the signal is noise-free, fix noise so we see it in plot
        bla_noise = db(np.sqrt(np.abs(sig.covGn[:,pp,pp])*R))
        bla_noise[bla_noise < -150] = -150
        bla_tot = db(np.sqrt(np.abs(sig.covG[:,pp,pp])*R))
        bla_tot[bla_tot < -150] = -150

        plt.plot(freq[lines], db(np.abs(sig.G[:,pp,0])))
        plt.plot(freq[lines], bla_noise,'s')
        plt.plot(freq[lines], bla_tot,'*')
        plt.xlabel('Frequency (Hz)')
        plt.ylabel('magnitude (dB)')
        plt.title(f'Estimated BLA p: {pp}')
        plt.legend(('BLA FRF','Noise Distortion','Total Distortion'))
        plt.gca().set_ylim(bottom=-150)
        figs['bla'] = (plt.gcf(), plt.gca())
    except:
        pass

# optimization path for PNLSS
plt.figure()
plt.plot(db(nl_errvec))
imin = np.argmin(nl_errvec)
plt.scatter(imin, db(nl_errvec[imin]))
plt.xlabel('Successful iteration number')
plt.ylabel('Validation error [dB]')
plt.title('Selection of the best model on a separate data set')
figs['pnlss_path'] = (plt.gcf(), plt.gca())

# subspace plots
#figs['subspace_optim'] = linmodel.plot_info()
#figs['subspace_models'] = linmodel.plot_models()

if savefig:
    for k, fig in figs.items():
        fig = fig if isinstance(fig, list) else [fig]
        for i, f in enumerate(fig):
            f[0].tight_layout()
            f[0].savefig(f"fig/tutorial_{k}{i}.pdf")

plt.show()

"""
Workable parameters
-------------------
RMSu = 0.05
Ntr = 5
nlx = NLS([Tanhdryfriction(eps=0.1, w=[1])])
E = 1e-1*Efull[:,:len(nlx.nls)]

----
RMSu = 0.05
Ntr = 5

gap = 0.25
nlx = NLS([Unilatteralspring(gap=gap, w=[1])])
E = 1e-1*Efull[:,:len(nlx.nls)]
----

RMSu = 0.05
Ntr = 5
E = Efull[:,:2]
nlx = NLS([poly2y, poly1y])  #, poly3])

nlx2 = NLS([poly1y,poly3y,poly2x,poly2y])  #,poly3])
nly2 = None

====
p = 2

if p == 2:
    C = np.vstack((C,C))
    D = np.vstack((D,0.1563532))

E = Efull
F = Ffull
nlx = NLS([Pnlss(degree=[2,3], structure='full')])
nly = NLS([Pnlss(degree=[2,3], structure='full')])
----

elif p ==2:
    Wy = np.array([[1,0],[0,1]])
    exp1 = [2,1]
    exp2 = [2,2]
    exp3 = [3,1]

nly = None
nlx = NLS([poly2y, poly1y])
E = Efull[:,:len(nlx.nls)]

nlx2 = NLS([poly1y,poly3y,poly2x,poly2y])
"""