import pandas as pd
from numeric_predictors.predictions import simple_linear_regression
import matplotlib.pyplot as plt
import math
import random
import numpy as np
from typing import List
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, mean_squared_error

def get_random_chunk(df: pd.DataFrame, steps: int, step_size: int) -> pd.DataFrame:
    first_ts = min(df.index)
    max_ts = max(df.index)
    period_size = steps * step_size
    not_after = max_ts - period_size
    beginning = random.randint(first_ts, not_after)
    end = beginning + period_size
    return df[(beginning <= df.index) & (df.index < end)]

def angle_of_change(cx: float, cy: float, ex: float, ey: float) -> float:
    dy = ey - cy
    dx = ex - cx
    theta = math.atan(dy / dx)
    return theta * 180 / math.pi

def determine_confusion(fca: float, aca: float):
    threshold = 0.0
    if aca > threshold:
        a = "increase"
    elif -threshold <= aca and aca <= threshold:
        a = "no_change"
    else:
        a = "decrease"
    if fca > threshold:
        f = "increase"
    elif -threshold <= fca and fca <= threshold:
        f = "no_change"
    else:
        f = "decrease"
    return f, a

def operate_with_sln(df: pd.DataFrame, steps: int, step_sizes: List[int]):
    list_of_results = []
    rmse_list = []
    confusion_list = []
    possible_change_forecasts = ["no_change", "increase", "decrease"]
    for step_size in step_sizes:
        local_price_forecast_results = pd.DataFrame(
            columns=["forecast", "actual", "timestamp"]
        )
        local_change_forecast_results = pd.DataFrame(columns=["forecast", "actual"])
        df = get_random_chunk(df, steps, step_size)
        chunk_1s_resample = df
        chunk_1s_resample["timestamp"] = chunk_1s_resample.index
        chunk_1s_resample["datetime"] = pd.to_datetime(df.index)
        chunk_1s_resample = df.set_index("datetime").resample("1s").mean()
        chunk_1s_resample = chunk_1s_resample.set_index(
            "timestamp"
        )  # this DF will give you the actual measurement but at the 1-second-step-mark
        init_ts = min(df.index)
        first_ts = init_ts
        last_ts = max(df.index)
        # steps = math.floor((last_ts-init_ts) / step_size) - 1
        for i in range(0, steps):
            print(f"{i}/{steps}")
            next_edge = init_ts + step_size
            next_future_edge = next_edge + step_size
            sln_datapoints = df[(init_ts < df.index) & (df.index < next_edge)]
            forecast_time = df[
                (next_edge < df.index) & (df.index < next_future_edge)
            ].index
            # print(sln_datapoints.head())
            # print(forecast_time)
            if sln_datapoints.empty or forecast_time.empty:
                init_ts = next_edge
                continue
            forecast_vals = simple_linear_regression(
                x_learn=sln_datapoints.index.values,
                y_learn=sln_datapoints["price"].values,
                x_predict=np.array([next_future_edge]),
            )
            present_vals = simple_linear_regression(
                x_learn=sln_datapoints.index.values,
                y_learn=sln_datapoints["price"].values,
                x_predict=np.array([next_edge]),
            )
            # Price forecast
            local_price_forecast_results.loc[i + 1, "forecast"] = forecast_vals[0]
            local_price_forecast_results.loc[i, "actual"] = present_vals[0]
            local_price_forecast_results.loc[i, "timestamp"] = next_edge
            # Increase/decrease (change) forecast
            if i != 0:
                try:
                    forecast_change_angle = angle_of_change(
                        cx=next_edge,
                        cy=local_price_forecast_results.loc[i, "forecast"],
                        ex=init_ts,
                        ey=local_price_forecast_results.loc[i - 1, "actual"],
                    )
                    actual_change_angle = angle_of_change(
                        cx=next_edge,
                        cy=local_price_forecast_results.loc[i, "actual"],
                        ex=init_ts,
                        ey=local_price_forecast_results.loc[i - 1, "actual"],
                    )
                    forecast_change, actual_change = determine_confusion(
                        forecast_change_angle, actual_change_angle
                    )
                    local_change_forecast_results.loc[i, "forecast"] = forecast_change
                    local_change_forecast_results.loc[i, "actual"] = actual_change
                except KeyError as e:
                    pass
            # plt.scatter(
            #     next_edge,
            #     present_vals[0],
            #     color="green"
            #     )
            # plt.scatter(
            #     next_future_edge,
            #     forecast_vals[0],
            #     color="blue"
            # )
            init_ts = next_edge
    print(local_price_forecast_results.sort_index().head())
    plt.plot(
        local_price_forecast_results["timestamp"],
        local_price_forecast_results["actual"],
        color="green",
    )
    plt.plot(
        local_price_forecast_results["timestamp"],
        local_price_forecast_results["forecast"],
        color="blue",
    )
    plt.vlines(
        x=range(first_ts, next_edge, step_size),
        ymin=min(local_price_forecast_results["forecast"]) - 50,
        ymax=max(local_price_forecast_results["forecast"]) + 50,
        colors="black",
    )

    plt.figure(2)
    plt.scatter(
        local_price_forecast_results["forecast"], local_price_forecast_results["actual"]
    )
    confusion_mtx = confusion_matrix(
        y_true=local_change_forecast_results["actual"],
        y_pred=local_change_forecast_results["forecast"],
        labels=possible_change_forecasts,
    )
    confusion_mtx_disp = ConfusionMatrixDisplay(
        confusion_matrix=confusion_mtx, display_labels=possible_change_forecasts
    )
    confusion_mtx_disp.plot()
    nc_nc, nc_i, nc_d, i_nc, i_i, i_d, d_nc, d_i, d_d = confusion_mtx.ravel()
    price_change_accuracy = (nc_nc + i_i + d_d) / (
        nc_nc + nc_i + nc_d + i_nc + i_i + i_d + d_nc + d_i + d_d
    )
    print(f"Accuracy: {price_change_accuracy*100}%")
    plt.show()

df1 = pd.read_csv("../../../predictors/data/btc_prices_eur.csv", sep=";")

# df["datetime"] = pd.to_datetime(df["timestamp"], unit="s")
# print(df.head())
# df = df.set_index("datetime")
# df = df.resample("5T").mean()
# print(df.head())

df1 = df1.set_index("timestamp")
operate_with_sln(df=df1, steps=5000, step_sizes=[60 * 10])

# print(angle_of_change(cx=5, cy=-5, ex=0, ey=0))


from tabulate import tabulate
class MucPhi:
    def __init__(self, db):
        self.db = db

    def them_muc_phi(self, nam, hoc_phi):
        query = """
        INSERT INTO MucPhi (nam, hoc_phi)
        VALUES (%s, %s)
        """
        self.db.execute_query(query, (nam, hoc_phi))
        self.xem_thong_tin()

    def sua_muc_phi(self, nam, hoc_phi):
        query = "UPDATE MucPhi SET hoc_phi=%s WHERE nam=%s"
        if (hoc_phi):
            self.db.execute_query(query, (hoc_phi, nam))
            self.xem_thong_tin(nam)
        else:
            print("Không có thông tin nào được chỉnh sửa")

    def xoa_muc_phi(self, nam):
        query = "DELETE FROM MucPhi WHERE nam=%s"
        self.db.execute_query(query, (nam,))

    def lay_hoc_phi(self, nam):
        query = "SELECT hoc_phi FROM MucPhi WHERE nam=%s"
        results = self.db.fetch_query(query, (nam,))
        return results[0][0] if results else None

    def muc_phi_ton_tai(self, nam):
        query = "SELECT COUNT(*) FROM MucPhi WHERE nam=%s"
        results = self.db.fetch_query(query, (nam,))
        return results[0][0] > 0

    def lay_hoc_phi_theo_ky(self, ky_hoc):
        # Tách kỳ học để lấy năm
        nam_hoc = ky_hoc.split('-')[0]

        # Truy vấn học phí dựa trên năm
        query = "SELECT hoc_phi FROM MucPhi WHERE nam = %s"
        result = self.db.fetch_query(query, (nam_hoc,))

        # Kiểm tra nếu có kết quả
        if result:
            return result[0]
        else:
            return None

    def xem_thong_tin(self, nam=None):
        query = ""
        results = ""
        if nam == None:
            query = """
                SELECT * FROM MucPhi

            """
            results = self.db.fetch_query(query, ())
        else:
            query = """
                SELECT * FROM MucPhi where nam=%s

            """
            results = self.db.fetch_query(query, (nam,))
        headers = ["Năm", "Học Phí"]
        print( tabulate(results, headers=headers, tablefmt="grid", floatfmt=(",.0f")))


from django.urls import path,re_path,include
from django.conf.urls import include, url
from django.urls.conf import re_path
from . import views
from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    path('',views.home, name='home'),
    re_path('project/(\d+)', views.get_project_by_id, name='project_results'),
    path('new/project', views.new_project, name='new-project'),
    path('api/projects/', views.ProjectsList.as_view()),
    path('api/profile/', views.ProfileList.as_view()),
    path('ratings/', include('star_ratings.urls', namespace='ratings')),
    path('search/', views.search_projects, name='search_results'),
    path('accounts/profile/', views.user_profiles, name='profile'),
    path('registration/', views.registerPage, name="registration"),
    path('login/', views.loginPage, name="login"),
    path('logout/', views.logoutUser, name="logout"),
]
if settings.DEBUG:
    urlpatterns+= static(settings.MEDIA_URL, document_root = settings.MEDIA_ROOT)


from absl import app, flags, logging
from absl.flags import FLAGS
import os
import tensorflow as tf

from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard

from modules.models import ArcFaceModel
from modules.losses import SoftmaxLoss
from modules.utils import set_memory_growth, load_yaml, get_ckpt_inf,generatePermKey
from losses.angular_margin_loss import arcface_loss,cosface_loss,sphereface_loss
from losses.euclidan_distance_loss import triplet_loss,triplet_loss_vanila,contrastive_loss,triplet_loss_omoindrot
import modules.dataset as dataset
from tensorflow import keras
import modules

flags.DEFINE_string('cfg_path', './configs/iom_res50.yaml', 'config file path')
flags.DEFINE_string('gpu', '0', 'which gpu to use')
flags.DEFINE_enum('mode', 'fit', ['fit', 'eager_tf'],
                  'fit: model.fit, eager_tf: custom GradientTape')

# modules.utils.set_memory_growth()

def main(_):
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu

    logger = tf.get_logger()
    logger.disabled = True
    logger.setLevel(logging.FATAL)
    # set_memory_growth()
    logging.info("os.environ['CUDA_VISIBLE_DEVICES']: "+ os.environ['CUDA_VISIBLE_DEVICES'])

    cfg = load_yaml(FLAGS.cfg_path)
    model = ArcFaceModel(size=cfg['input_size'],
                         backbone_type=cfg['backbone_type'],
                         num_classes=cfg['num_classes'],
                         head_type=cfg['head_type'],
                         embd_shape=cfg['embd_shape'],
                         w_decay=cfg['w_decay'],
                         training=True, cfg=cfg)
    model.summary(line_length=80)
    for layer in model.layers:
        if layer.name == 'resnet50':
            layer.summary(line_length=80)
    if cfg['train_dataset']:
        logging.info("load ms1m dataset.")
        dataset_len = cfg['num_samples']
        steps_per_epoch = dataset_len // cfg['batch_size']
        train_dataset = dataset.load_tfrecord_dataset(
            cfg['train_dataset'], cfg['batch_size'], cfg['binary_img'],
            is_ccrop=cfg['is_ccrop'])
    else:
        logging.info("load fake dataset.")
        steps_per_epoch = 1
        train_dataset = dataset.load_fake_dataset(cfg['input_size'])
        # (x_train1, y_train1), (x_test1, y_test1) = keras.datasets.cifar10.load_data()
        # from keras.preprocessing.image import ImageDataGenerator
        # datagen = ImageDataGenerator(rescale=1. / 255,
        #                              shear_range=0.2,
        #                              zoom_range=0.2,
        #                              horizontal_flip=True)
        # train_dataset = datagen.flow(x_train1, y_train1, batch_size=cfg['batch_size'])

    learning_rate = tf.constant(cfg['base_lr'])
    optimizer = tf.keras.optimizers.SGD(
        learning_rate=learning_rate, momentum=0.9, nesterov=True)
    loss_fn = SoftmaxLoss() #############################################
    ckpt_path = tf.train.latest_checkpoint('./checkpoints/' + cfg['sub_name'])
    if ckpt_path is not None:
        print("[*] load ckpt from {}".format(ckpt_path))
        model.load_weights(ckpt_path)
        epochs, steps = get_ckpt_inf(ckpt_path, steps_per_epoch)
    else:
        print("[*] training from scratch.")
        epochs, steps = 1, 1
    model.summary(line_length=80)
    if FLAGS.mode == 'eager_tf':
        # Eager mode is great for debugging
        # Non eager graph mode is recommended for real training
        summary_writer = tf.summary.create_file_writer(
            './logs/' + cfg['sub_name'])

        train_dataset = iter(train_dataset)

        while epochs <= cfg['epochs']:
            inputs, labels = next(train_dataset)

            with tf.GradientTape() as tape:
                logist = model(inputs, training=True)
                # print(logist)
                if cfg['head_type'] == 'IoMHead':
                    reg_loss = tf.cast(tf.reduce_sum(model.losses),tf.double)
                else:
                    reg_loss = tf.reduce_sum(model.losses)
                pred_loss = loss_fn(labels, logist)
                total_loss = pred_loss + reg_loss

            grads = tape.gradient(total_loss, model.trainable_variables)
            optimizer.apply_gradients(zip(grads, model.trainable_variables))

            if steps % 5 == 0:
                verb_str = "Epoch {}/{}: {}/{}, loss={:.2f}, lr={:.4f}"
                print(verb_str.format(epochs, cfg['epochs'],
                                      steps % steps_per_epoch,
                                      steps_per_epoch,
                                      total_loss.numpy(),
                                      learning_rate.numpy()))

                with summary_writer.as_default():
                    tf.summary.scalar(
                        'loss/total loss', total_loss, step=steps)
                    tf.summary.scalar(
                        'loss/pred loss', pred_loss, step=steps)
                    tf.summary.scalar(
                        'loss/reg loss', reg_loss, step=steps)
                    tf.summary.scalar(
                        'learning rate', optimizer.lr, step=steps)

            if steps % cfg['save_steps'] == 0:
                print('[*] save ckpt file!')
                model.save_weights('checkpoints/{}/e_{}_b_{}.ckpt'.format(
                    cfg['sub_name'], epochs, steps % steps_per_epoch))

            steps += 1
            epochs = steps // steps_per_epoch + 1
    else:
        model.compile(optimizer=optimizer, loss=loss_fn)

        mc_callback = ModelCheckpoint(
            'checkpoints/' + cfg['sub_name'] + '/e_{epoch}_b_{batch}.ckpt',
            save_freq=cfg['save_steps'] * cfg['batch_size'], verbose=1,
            save_weights_only=True)
        tb_callback = TensorBoard(log_dir='logs/',
                                  update_freq=cfg['batch_size'] * 5,
                                  profile_batch=0)
        tb_callback._total_batches_seen = steps
        tb_callback._samples_seen = steps * cfg['batch_size']
        callbacks = [mc_callback, tb_callback]

        model.fit(train_dataset,
                  epochs=cfg['epochs'],
                  steps_per_epoch=steps_per_epoch,
                  callbacks=callbacks,
                  initial_epoch=epochs - 1)

    print("[*] training done!")

if __name__ == '__main__':
    app.run(main)


#!/usr/bin/python3
from scripts.deployment.base import ContractDeployment
from constants import ContractName
from brownie import Configuration
from .mixins.transfer_ownership import TransferOwnershipMixin

class ConfigurationDeployment(ContractDeployment, TransferOwnershipMixin):
    contract_name = ContractName.CONFIGURATION
    contract_class = Configuration

    def validate(self):
        errors = super().validate()
        if not self.setting.GFN_CONFIGURATION_OWNER_ADDRESS:
            errors.append(
                "Please setup env: 'GFN_CONFIGURATION_OWNER_ADDRESS'")
        return errors

    def get_owner(self):
        return self.setting.GFN_CONFIGURATION_OWNER_ADDRESS

    def deploy(self):
        registry_instance = self.get_registry_instance()

        print(f"==> Deploying {self.contract_name} .....")
        configuration = self.contract_class.deploy(
            self.setting.GFN_DEPLOYER_ADDRESS,
            self.setting.GFN_NFT_HOLDER_ADDRESS,
            registry_instance.address,
            self.setting.TXN_SENDER
        )

        return configuration

    def transfer_contract_owner(self):
        print(f'==> Transferring Owner of {self.contract_name} '
              f'to {self.get_owner()}')
        self.contract_instance.transferOwnership(
            self.get_owner(),
            self.setting.TXN_SENDER
        )


# -*- coding: utf-8 -*-

from scipy.stats import pearsonr
import numpy as np

#Edit the "pathtolocal", which should be the same as in LSTrAP_Lite_download.py and LSTrAP_Lite_TrimMatrix.py

#PATHS
pathtolocal = '/path/to/local'
matrix_file = open(pathtolocal + '/matrix_selected.tsv', "r")

"""
pcc_cal(geneID, cutoff = 0.7, n_size = 51)
   Function to calculate the pcc value of query gene against all other genes in the matrix.
   Input: geneID (query gene)
   Arguments cutoff (cutoff for PCC value to be considered); n_size (size of immediate neighbourhood)

   The functions takes in a matrix and calculates the PCC value of the query gene against all other genes.
   Additional parameters such as PCC cutoff value and neighbourhood size can be specified.

   Output: None

   Files generated: 1) List of genes pairs and their PCC values sorted according to the PCC value in descending order.
					2) PCC network of the gene pairs from the file described above
"""

def pcc_cal(geneID, cutoff = 0.7, n_size = 51):
	neighbourhood_file = open(pathtolocal  +'/' + geneID.replace('|','_') + '_nbh.txt', 'w+') #creates PCC output file. Replaces possible illegal characters
	dict_val = {} #container to store gene : [tpm values]
	gene_list = [] #records the genes in the matrix
	matrix_file.readline() #skips header of matrix file
	for line in matrix_file: #converts matrix to dictionary where key refers to gene and values refer to the tpm values across the various samples
		content = line.rstrip('\n').split('\t')
		key = content[0]
		values = np.array(content[1:]).astype(np.float)
		gene_list.append(key)
		if key in dict_val:
			dict_val[key].append(values)
		else:
			dict_val[key]=values
	if '' in dict_val:
		dict_val.pop('')

	#Calculates PCC values all possible combinations of gene pairs and writes the gene pairs to output_file
	gene_val = [] #container for gene pairs with PCC higher than cutoff defined (default = 0.7)
	counter = 0
	#Calculates PCC value for each gene against query gene
	for item in dict_val:
		pcc_val, p_value = pearsonr(dict_val.get(geneID), dict_val.get(item))
		counter+=1
		if counter % 10000 == 0:
			print('Calculated PCC for ' + str(counter) + ' out of ' + str(len(gene_list)) + " genes")
		if pcc_val >= cutoff:
			gene_val.append([geneID, item, pcc_val])
	sorted_gene_val = sorted(gene_val, key = lambda x: x[2], reverse = True) #sorts genes according to the pcc value
	#If there are more gene pairs than the defined neighbourhood size cutoff, only write the ones included in the cutoff. Else, write everything in sorted_gene_val
	if len(sorted_gene_val) >n_size:
		for i in range(n_size):
			neighbourhood_file.write(sorted_gene_val[i][0] + "\t" + sorted_gene_val[i][1] + "\t" + str(sorted_gene_val[i][2]) + "\n")
	else:
		for z in sorted_gene_val:
			neighbourhood_file.write(z[0] + "\t" + z[1] + "\t" + str(z[2]) + "\n")
	neighbourhood_file.close()
	print("PCC neighbourhood calculated for: " + geneID + "\n")
	print("Neighbourhood file generated at: " + pathtolocal  +'/' + geneID.replace('|','_') + '_nbh.txt\n')

	"""Construction of PCC network"""
	neighbourhood_file = open(pathtolocal  +'/' + geneID.replace('|','_') + '_nbh.txt', 'r') #creates PCC output file. Replaces possible illegal characters
	network_file = open(pathtolocal  +'/' + geneID.replace('|','_') + '_nw.txt', 'w+') #creates PCC network file. Replaces possible illegal characters
	nw_gene_list = [x.split("\t")[1] for x in neighbourhood_file]
	network_size = 0
	for i in range(len(nw_gene_list)):
		network_size+=i
	counter2 = 0
	ngene_val = []
	"""Calculates PCC values for al possible combinations of gene pairs and writes the gene pairs that have PCC val ?= 0.7 to network_file"""
	for gene in range(len(nw_gene_list)):
		for gene2 in range(gene):
			npcc_val, np_value = pearsonr(dict_val.get(nw_gene_list[gene]), dict_val.get(nw_gene_list[gene2]))
			ngene_val.append([nw_gene_list[gene], nw_gene_list[gene2], npcc_val])
			counter2+=1
			if counter2 % 1000 == 0:
				print('Calculated PCC for ' + str(counter2) + ' out of ' + str(network_size) + ' gene pairs')
	nsorted_gene_val = sorted(ngene_val, key = lambda x: x[2], reverse = True) #sorts genes according to the pcc value
	for i in nsorted_gene_val:
		network_file.write(i[0] + "\t" + i[1] + "\t" + str(i[2]) + "\n")
	neighbourhood_file.close()
	network_file.close()
	print("PCC network calculated for all gene pairs\n")
	print("Network file generated at: " + pathtolocal  +'/' + geneID.replace('|','_') + '_nw.txt')

user = input("Enter gene of interest, PCC cutoff (default = 0.7) [optional] and network size (default = 51) [optional]. eg. AT1G01720,0.5,80: ")
print("\n")

#While the user gives an empty input, keeps asking for the user to give a valid input
while user == '':
	user = input("Enter gene of interest and PCC cutoff (default = 0.7) eg. AT1G01720,0.5,80: ")
#Checks if user specified arguments. If the user specified arguments, check for the number of arguments user specified to split the input accordingly.
#If no arguments are specified, takes the query gene and run the calculation
if ',' in user:
	if len(user.split(',')) == 2:
		x = user.split(",")[0]
		y = float(user.strip('\n').split(",")[1])
		pcc_cal(x,y)
	elif len(user.split(',')) == 3:
		x = user.split(",")[0]
		y = float(user.strip('\n').split(",")[1])
		z = int(user.strip('\n').split(",")[2])
		pcc_cal(x,y,z)
else:
	pcc_cal(user.strip('\n'))

matrix_file.close()


import logging

from django.conf import settings
from django.db import migrations

from metadata.migration_util import models

logger = logging.getLogger("metadata")

def change_custom_report_aggregation_datasource(apps, *args, **kwargs):
    # 获取APP models
    for model_name in list(models.keys()):
        models[model_name] = apps.get_model("metadata", model_name)

    data_id = 1100014
    # 实际influxdb对应的数据库名
    database = "datalink_stats"
    # data_id对应的名称
    data_name = f"{settings.AGGREGATION_BIZ_ID}_{database}"
    # 结果表id
    old_table_id = "bkmonitorbeat_statistics.__default__"
    table_id = f"{database}.__default__"
    # 来源标签
    source_label = "datalink"

    # 修改data_name和source_label
    models["DataSource"].objects.filter(bk_data_id=data_id).update(
        data_name=data_name,
        source_label=source_label,
        data_description="init data_source for %s" % data_name,
    )
    # 修改table_id
    table_id_models = [
        "DataSourceResultTable",
        "ResultTable",
        "ResultTableField",
        "InfluxDBStorage",
        "TimeSeriesGroup",
        "ResultTableOption",
    ]
    for name in table_id_models:
        models[name].objects.filter(table_id=old_table_id).update(
            table_id=table_id,
        )

class Migration(migrations.Migration):
    dependencies = [
        ("metadata", "0127_merge_20220901_1544"),
    ]

    operations = [
        migrations.RunPython(change_custom_report_aggregation_datasource, reverse_code=migrations.RunPython.noop),
    ]


from pyramid.config import Configurator

def main(global_config, **settings):
    """Build the pyramid WSGI App."""
    with Configurator(settings=settings) as config:
        config.add_route("notify_v1", "/v1/notification")
        config.add_route("prom_metrics", "/metrics")
        config.scan(".views")

        app = config.make_wsgi_app()
        return app


#! /usr/bin/python

import numpy as np
import matplotlib.pyplot as plt
import os,sys
import keras

latent_dim = 100
model_name = 'models/wgan_240_240_50001_Jeans_G.h5'

def model_test():
    r, c = 5, 5

    model = keras.models.load_model(model_name)
    model.summary()

    noise = np.random.normal(0, 1, (r * c, latent_dim))
    gen_imgs = model.predict(noise)

    # Rescale images 0 - 1
    gen_imgs = 0.5 * gen_imgs + 0.5

    fig, axs = plt.subplots(r, c)
    cnt = 0
    for i in range(r):
        for j in range(c):
            axs[i,j].imshow(gen_imgs[cnt, :,:,:])
            axs[i,j].axis('off')
            cnt += 1
    save_path = 'images/Check'
    if not os.path.exists(save_path):
        os.mkdir(save_path)
    fig_name = "%s.png" % (model_name.split('/')[1].split('.')[0])
    print (fig_name)
    if os.path.exists(sys.path[0]+"/"+fig_name):
        os.remove(sys.path[0]+"/"+fig_name)
    fig.savefig(save_path+'/'+fig_name)
    plt.close()

if __name__ == "__main__":
    model_test()


from PySide6.QtCore import (QSize, Qt)
from PySide6.QtGui import QIcon
from PySide6.QtWidgets import (QFrame, QHBoxLayout, QPushButton, QVBoxLayout)
from PySide6.QtCore import (Slot, Signal)
import pyqtgraph
import data_models as dms
import numpy as np
import copy

class PlotWidget(QFrame):

    sig_data_updated = Signal()
    sig_line_region_picked = Signal()
    sig_set_enable = Signal(bool)

    def __init__(self, parent=None):
        super().__init__(parent=parent)

        self.setFrameShape(QFrame.NoFrame)
        self.setFrameShadow(QFrame.Raised)
        self.setLineWidth(1)

        self.data_model = dms.DataModel()
        self.region_values = None
        self.line_value = None

        plt_win = pyqtgraph.GraphicsLayoutWidget()
        self.plt_species = plt_win.addPlot(title="Raw & Decomposed Data")  # PlotItem
        self.plt_model = plt_win.addPlot(title="Raw & Modeled Data")
        plt_win.nextRow()
        self.plt_error = plt_win.addPlot(title="Residuals")
        self.plt_import = plt_win.addPlot(title="Imported Data")

        self.icon_check = QIcon()
        self.icon_check.addFile(u":/Icon/Resources/Icons/check_circle_FILL0_wght400_GRAD0_opsz48.svg",
                                     QSize(), QIcon.Normal, QIcon.Off)
        self.icon_region = QIcon()
        self.icon_region.addFile(u":/Icon/Resources/Icons/horizontal_distribute_FILL0_wght400_GRAD0_opsz48.svg",
                                     QSize(), QIcon.Normal, QIcon.Off)
        self.icon_cancel = QIcon()
        self.icon_cancel.addFile(u":/Icon/Resources/Icons/cancel_FILL0_wght400_GRAD0_opsz48.svg",
                                 QSize(), QIcon.Normal, QIcon.Off)

        self.pb_region = QPushButton(u"Region")
        self.pb_region.setCheckable(True)
        self.pb_region.setIcon(self.icon_region)
        self.pb_region.setStyleSheet(u"background-color: rgb(249, 240, 107);")
        self.pb_region.setIconSize(QSize(16, 16))

        self.pb_set_region = QPushButton("Apply")
        self.pb_set_region.setEnabled(False)
        self.pb_set_region.setIcon(self.icon_check)
        self.pb_set_region.setIconSize(QSize(16, 16))

        lyt_pb = QVBoxLayout()
        lyt_pb.setContentsMargins(0, 0, 0, 0)
        lyt_pb.setSpacing(1)
        lyt_pb.addStretch(1)
        lyt_pb.addWidget(self.pb_region)
        lyt_pb.addWidget(self.pb_set_region)

        pen = pyqtgraph.mkPen(color='y', width=2)
        self.curve_import = self.plt_import.plot(pen=pen)  # PlotDataItem
        self.curve_trim = self.plt_species.plot(pen=pen)
        self.curve_raw = self.plt_model.plot(pen=pen)

        pen = pyqtgraph.mkPen(color='w', width=3)
        self.region_picker_import = pyqtgraph.LinearRegionItem(pen=pen)
        self.line_picker_trim = pyqtgraph.InfiniteLine(pen=pen, movable=True)
        pen.setStyle(Qt.DotLine)
        self.region_picker_trim = pyqtgraph.LinearRegionItem(pen=pen)
        # self.picker_raw.setZValue(-10)

        layout = QHBoxLayout()
        layout.setSpacing(1)
        layout.setContentsMargins(2, 2, 2, 2)
        layout.addWidget(plt_win)
        layout.addLayout(lyt_pb)
        self.setLayout(layout)

        self.region_picker_import.sigRegionChanged.connect(self.slt_update_trim)
        self.pb_region.clicked.connect(self.slt_update_region)
        self.pb_set_region.clicked.connect(self.slt_set_region)
        self.plt_error.sigRangeChanged.connect(self.slt_error_view_range)

    @Slot(object)
    def slt_update_trim(self):
        [min_val, max_val] = self.region_picker_import.getRegion()
        x_raw = self.data_model.data.x_raw
        ind = np.logical_and(x_raw >= min_val, x_raw <= max_val)
        x_temp = x_raw[ind]
        y_temp = self.data_model.data.y_raw[ind]
        self.curve_trim.setData(x_temp, y_temp)
        self.curve_raw.setData(x_temp, y_temp)

    @Slot(bool)
    def slt_update_region(self, checked):
        # self.pb_reg.setIconSize(QSize(16, 16))
        if checked:
            self.pb_region.setText("Cancel")
            self.pb_region.setIcon(self.icon_cancel)
            self.pb_region.setStyleSheet(u"background-color: rgb(246, 97, 81);")
            self.pb_set_region.setStyleSheet(u"background-color: rgb(143, 240, 164);")
            self.pb_set_region.setEnabled(True)
            self._pick_region_import(1)
        else:
            self.pb_region.setText("Region")
            self.pb_region.setIcon(self.icon_region)
            self.pb_region.setStyleSheet(u"background-color: rgb(249, 240, 107);")
            self.pb_set_region.setStyleSheet(u"")
            self.pb_set_region.setEnabled(False)
            self._pick_region_import(-1)

    @Slot()
    def slt_set_region(self):
        self.pb_region.setText("Region")
        self.pb_region.setChecked(False)
        self.pb_region.setIcon(self.icon_region)
        self.pb_region.setStyleSheet(u"background-color: rgb(249, 240, 107);")
        self.pb_set_region.setStyleSheet(u"")
        self.pb_set_region.setEnabled(False)
        self._pick_region_import(0)

    @Slot(object, object)
    def slt_error_view_range(self, arg1, arg2):
        text_item = None
        for item in self.plt_error.items:
            if isinstance(item, pyqtgraph.TextItem):
                text_item = item
                break
        if text_item is None:
            return
        xr, yr = arg2
        dx = xr[1] - xr[0]
        dy = yr[1] - yr[0]
        pad = 0.1
        px = xr[1] - dx * pad
        py = yr[1] - dy * pad
        text_item.setPos(px, py)
        text_item.updateTextPos()

    def _pick_region_import(self, state: int):
        if state == 1:  # connect picker
            min_x = np.min(self.data_model.data.x_trim)
            max_x = np.max(self.data_model.data.x_trim)
            self.region_picker_import.setRegion([min_x, max_x])
            self.plt_import.addItem(self.region_picker_import)
            self.sig_set_enable.emit(False)
        elif state == -1:  # cancel and close piker
            self.plt_import.removeItem(self.region_picker_import)
            self.curve_trim.setData(self.data_model.data.x_trim, self.data_model.data.y_trim)
            self.curve_raw.setData(self.data_model.data.x_trim, self.data_model.data.y_trim)
            self.sig_set_enable.emit(True)
        elif state == 0:  # accept and close picker
            data = self.curve_trim.getData()
            self.data_model.data.x_trim = data[0]
            self.data_model.data.y_trim = data[1]
            self.plt_import.removeItem(self.region_picker_import)
            self.sig_data_updated.emit()
            self.sig_set_enable.emit(True)

    def pick_line_region_trim(self, state: int, line: float, x1: float, x2: float):
        if state == 1:  # connect picker
            if x1 == -1 and x2 == -1 and line == -1:
                vr = self.plt_species.viewRange()[0]
                x1, x2 = vr[0], vr[1]
                dx = (x2 - x1) * 0.05
                x1 += dx
                x2 -= dx
                line = 0.5 * (x1 + x2)
            self.region_picker_trim.setRegion([x1, x2])
            self.line_picker_trim.setValue(line)
            self.plt_species.addItem(self.region_picker_trim)
            self.plt_species.addItem(self.line_picker_trim)
        elif state == 0:  # accept and close picker
            self.region_values = self.region_picker_trim.getRegion()
            self.line_value = self.line_picker_trim.value()
            self.plt_species.removeItem(self.region_picker_trim)
            self.plt_species.removeItem(self.line_picker_trim)
            self.sig_line_region_picked.emit()

    def _plot_species(self, index=None):
        for i in range(len(self.data_model.model)):
            xx, yy = self.data_model.model[i].get_xy()
            if xx is not None and yy is not None:
                curve = self.plt_species.plot(pen=pyqtgraph.mkPen(color='magenta', width=2))
                curve.setData(xx, yy)
        if index is not None:
            xx, yy = self.data_model.model[index].get_xy()
            if xx is not None and yy is not None:
                brush = pyqtgraph.mkBrush(color='magenta')
                # brush.setStyle(Qt.BrushStyle.BDiagPattern)
                brush.setStyle(Qt.BrushStyle.DiagCrossPattern)
                pen = pyqtgraph.mkPen(color='magenta', width=2)
                curve = self.plt_species.plot(pen=pen, fillLevel=0)
                curve.setBrush(brush)
                curve.setData(xx, yy)
            else:
                if self.data_model.model[index].type == dms.Types.EXP:
                    return
                x_1 = self.data_model.model[index].gauss.center.min
                x_2 = self.data_model.model[index].gauss.center.value
                x_3 = self.data_model.model[index].gauss.center.max
                pen = pyqtgraph.mkPen(color='cyan', width=1, style=Qt.DotLine)
                curve_1 = pyqtgraph.InfiniteLine(pen=pen, movable=False)
                self.plt_species.addItem(curve_1)
                curve_1.setValue(x_1)
                pen.setStyle(Qt.SolidLine)
                curve_2 = pyqtgraph.InfiniteLine(pen=pen, movable=False)
                self.plt_species.addItem(curve_2)
                curve_2.setValue(x_2)
                pen.setStyle(Qt.DotLine)
                curve_3 = pyqtgraph.InfiniteLine(pen=pen, movable=False)
                self.plt_species.addItem(curve_3)
                curve_3.setValue(x_3)

    def plot_data(self, data_model: dms.DataModel, index=None):
        self.data_model = copy.deepcopy(data_model)
        self.plt_import.clear()
        self.plt_species.clear()
        self.plt_model.clear()
        self.plt_error.clear()
        self.plt_import.addItem(self.curve_import)
        self.plt_species.addItem(self.curve_trim)
        self.plt_model.addItem(self.curve_raw)
        self.curve_import.setData(self.data_model.data.x_raw, self.data_model.data.y_raw)
        self.curve_trim.setData(self.data_model.data.x_trim, self.data_model.data.y_trim)
        self.curve_raw.setData(self.data_model.data.x_trim, self.data_model.data.y_trim)
        if self.data_model.data.y_model is not None:
            curve_m = self.plt_model.plot(pen=pyqtgraph.mkPen(color='magenta', width=2))
            curve_r = self.plt_error.plot(pen='g')
            curve_m.setData(self.data_model.data.x_trim, self.data_model.data.y_model)
            curve_r.setData(self.data_model.data.x_trim, self.data_model.data.residual)
            rmsd = np.sqrt(np.mean(self.data_model.data.residual ** 2))
            px = np.max(self.data_model.data.x_trim)
            py = np.max(self.data_model.data.residual)
            text_item = pyqtgraph.TextItem(f"RMSD = {rmsd: .6f}", pyqtgraph.mkColor("white"), anchor=(1, 0))
            self.plt_error.addItem(text_item)
            text_item.setPos(px, py)
        self._plot_species(index)

    def get_data(self):
        return copy.deepcopy(self.data_model.data)

    def get_region_values(self):
        return self.region_values

    def get_line_value(self):
        return self.line_value


# !pip install open_clip_torch

import open_clip
# import tensorflow as tf
# import tensorflow_hub as hub
import torch
import gensim.downloader
from tqdm import tqdm

class USEEncode:
    def __init__(self, model=None, prompt=None):
        self.prefix, self.suffix = '', ''
        if prompt:
            self.prefix, self.suffix = prompt.split('{}')
        if model is None:
            model = "https://tfhub.dev/google/universal-sentence-encoder/4"

        self.model = hub.load(model)

    def encode(self, df):
        prompted = (self.prefix + df.color + self.suffix).values
        return torch.FloatTensor(self.model(prompted).numpy())

class CLIPEncode:
    def __init__(self, model=None, prompt=None):
        self.prefix, self.suffix = '', ''
        if prompt:
            self.prefix, self.suffix = prompt.split('{}')
        if model is None:
            model = "ViT-B-32"
        self.clip, _, _ = open_clip.create_model_and_transforms(
        model, pretrained="openai")

        self.tokenizer = open_clip.get_tokenizer(model)

    def encode(self, df):
        batch_size = 200
        prompted = (self.prefix + df.color + self.suffix).values
        with torch.no_grad():
            encodings = []
            for i in tqdm(range(0, len(prompted), batch_size)):
                tokenized = self.tokenizer(prompted[i:i+batch_size])
                encodings.append(self.clip.encode_text(tokenized))
        return torch.vstack(encodings)

class W2VEncode:
    def __init__(self, model=None, prompt=None):
        """
        Available Models:
         'fasttext-wiki-news-subwords-300',
         'conceptnet-numberbatch-17-06-300',
         'word2vec-ruscorpora-300',
         'word2vec-google-news-300',
         'glove-wiki-gigaword-50',
         'glove-wiki-gigaword-100',
         'glove-wiki-gigaword-200',
         'glove-wiki-gigaword-300',
         'glove-twitter-25',
         'glove-twitter-50',
         'glove-twitter-100',
         'glove-twitter-200',
        """
        if model is None:
            model = 'glove-wiki-gigaword-50'
        self.vectorizer = gensim.downloader.load(model)

    def encode(self, df):
        return torch.FloatTensor(self.vectorizer[df.color.values])


"""
Energy data NLP and prediction
------------------------------------------------------------------
Author: Simon (Hyungjun) Park

"""

"""
For the categorization of the predictions with natural language processing, we take the following steps:
    1. Remove unknown characters and classify each sentence according to energy type.
    2. Extract token related to target categories (e.g., "price") from each sentence.
    3. Conduct several methods of syntactic dependency parse. First, examine ancestors of each tokens.
    If we can't find any direction words, then check verb ancestors' children. Then, we can't find anything,
     try tokens' children.
    4. Compare the keywords (i.e.,ancestors or children of the verb or tokens' children) with direction words
     (e.g., "increase", "down" etc) to catch the predictions for each target and count the number of matched
      words. Also, we take negative sentence words into consideration.
For limitation, since we can't detect every dependency by simply using tokens and their ancestors or
children, some direction word are missed. Also, because of the shortage of reference about some energy type
(e.g., solar, wind), there are some NaNs in our final output dataframe.

"""

import PyPDF2
import pandas as pd
import requests
import os
import re
import spacy
import pyprind
import numpy as np

nlp = spacy.load("en_core_web_sm")

class ReadPDF():

    def __init__(self, years, path = os.getcwd()):
        self.years = years
        self.path = path
        self.textinpage = self.read_pdf()

    def get_pdf(self):
        """
        Download PDF files from EIA

        """
        if not any([os.path.exists(os.path.join(self.path,'aeo{}.pdf'.format(y))) for y in self.years]):
            print('Downloading data... (overwriting)')
            for y in self.years:
                dl_url = r'https://www.eia.gov/outlooks/aeo/pdf/aeo{}.pdf'.format(y)
                r = requests.get(dl_url)
                file = dl_url.split('/')[-1]
                with open(os.path.join(self.path, file), 'wb') as pfile:
                    pfile.write(r.content)
            print('PDF(s) for', self.years, 'downloaded in your working directory.')
        else:
            print('PDF(s) for', self.years, 'already exist(s) in your working directory.')

    def read_pdf(self):
        """
        Read the PDFs
        (Reference: https://pythonhosted.org/PyPDF2/)

        Returns
        ---------
        pdf_textinpage: dictionary
            The value is text data for each year from PDF.
            The key is year (i.e., 2019, 2018)

        """
        self.get_pdf()

        pdf_textinpage = {}
        for y in self.years:
            with open(os.path.join(self.path,'aeo{}.pdf'.format(y)), 'rb') as ifile:
                pdf = PyPDF2.PdfFileReader(ifile)

                print(y)
                print('Number of pages:', pdf.numPages)
                pbar = pyprind.ProgBar(pdf.numPages)

                pages = []
                for p in range(pdf.numPages):
                    page = pdf.getPage(p)
                    text = page.extractText()
                    pages.append(text)
                    pbar.update()
            pdf_textinpage[y] = pages
        print('\n')
        return pdf_textinpage

class ForecastPDF():

    def __init__(self, pdf_file, energy_dict, target_dict, predict_dict, pdf_index, start = None, end = None):
        self.pdf = pdf_file[pdf_index][start:end]
        self.pdf_index = pdf_index
        self.ene_dict = energy_dict
        self.trgt_dict = target_dict
        self.prdt_dict = predict_dict
        self.resultdf = self.df_builder()

    def cleaner(self, text):
        """
        Remove unknown characters and rearrage line break

        Input
        -------
        text: list
            text data for each year from PDF.

        Return
        -------
        text: list
            text data without unknown characters

        """
        text = re.sub('\n|Š','', text)
        text = re.sub('Ł',' ', text)
        return text

    def token_generator(self, target_words, sents):
        """
        Generate token according to the categories of target variable

        Input
        -------
        target_term: list
            words related to target term.
        sents: list
            Sentences mentioned energy type we are interested in

        Return
        -------
        tokens: list of spacy.tokens.token

        """
        tokens = []
        for s in sents:
            token =  [t for t in s if any([k in t.string for k in target_words])]
            tokens.extend(token)
        return tokens

    def detector(self, keys):
        """
        Detect whether related words appear in prediction word dictionary and count the number.
        If the list of keywords contain negative words (i.e., no or not), inverse the direction of
        prediction.

        Input
        -------
        keys: list
            Token's ancestors or children of token's ancestor or token's children.

        Return
        -------
        result: 1d-array

        """
        result = []
        words_list = [[w.lemma_.strip() for w in k] for k in keys]

        if any([[w for w in words if w in ['no', 'not']] for words in words_list]):
            direction = list(prdt_dict.keys())[::-1]
        else:
            direction = list(self.prdt_dict.keys())

        for d in direction:
            judge = [[w for w in words if w in self.prdt_dict[d]] for words in words_list]
            judge = [l for l in judge if len(l) > 0]
            result.append(len(judge))
        if all([j == 0 for j in result]):
            return True, None
        else:
            return False, np.array(result)

    def predicator(self, tokens):
        """
        Try several syntactic dependency parse method to find words related to target word

        Input
        -------
        tokens: list of spacy.tokens.token

        Return
        -------
        result: 1d-array

        """
        ancestors = [list(t.ancestors) for t in tokens]
        anc_flat = [a for anc in ancestors for a in anc]

        key_words_set = [ ancestors,
                         [list(a.children) for a in anc_flat if a.pos_ == 'VERB'],
                         [list(t.children) for t in tokens]]
        for keys in key_words_set:
            trial = self.detector(keys)
            if trial[0]:
                continue
            else:
                return trial[1]
                break
        return np.array([0,0,0])

    def classifier(self, document, tg):
        """
        Integrate all functions to summarize the document

        Input
        -------
        document: text data
            Scraping from PDF
        tg: string
            target words

        Return
        -------
        results: array-like
            The matrix of precited direction for each energy type about specific targe term

        """
        results = np.zeros([len(self.ene_dict), len(self.prdt_dict)])
        for i, page in enumerate(document):
            text = self.cleaner(page)
            doc = nlp(text)

            type_sents = {}
            for e in self.ene_dict.keys():
                type_sents[e] = [sent for sent in doc.sents if any([e in sent.string for e in self.ene_dict[e]])]

            by_page = np.zeros((0, len(self.prdt_dict)))
            for e in type_sents.keys():
                tokens = self.token_generator(self.trgt_dict[tg], type_sents[e])
                by_sent = self.predicator(tokens)
                by_page =  np.r_[by_page, by_sent.reshape(1,-1)]

            results += by_page
        return results

    def df_builder(self):
        """
        Aggregate the results of classification and generate dataframe

        Return
        ------
        final_output: Data frame

        """
        final_output = pd.DataFrame(index = [])
        for tg in self.trgt_dict.keys():
            result_by_page = self.classifier(self.pdf, tg)
            df_by_target = pd.DataFrame(result_by_page, index = self.ene_dict.keys(), columns = self.prdt_dict.keys())
            result = df_by_target.apply(lambda x: "" if x.sum() == 0 else x.idxmax(axis = 1), axis = 1)
            final_output = pd.concat([final_output, result], axis=1, sort=False)
        final_output.columns = self.trgt_dict.keys()
        print('File:', self.pdf_index)
        return  print(final_output, '\n')

'''
Set Parameters
'''

years = [2019,2018]
path = os.path.dirname(os.path.realpath(__file__)) #Use os.getcwd() instead if not running the whole script

trgt_dict = {'Price'    :['price','cost'],
             'Emissions':['emission', 'emit','CO2'],
             'Produce'  :['production', 'produce', 'yield', 'generate', 'generation'],
             'Trade'    :['export', 'import', 'exporter', 'importer','trade']}

prdt_dict = {'Up'      : ['increase', 'increasingly', 'up','upper' 'high','raise', 'grow','growth','rise'],
             'Unchange': ['unchange', 'stable', 'same'],
             'Down'    : ['decrease','decline','down', 'low','lower','loss', 'fall','reduce']}

ene_dict = {'Coal'    :['coal'],
            'Nuclear' :['nuclear'],
            'Wind'    :['wind'],
            'Solar'   :['solar'],
            'Oil'     :['oil','petroleum']}

'''
Execute

'''

rawpdfs = ReadPDF(years, path)
pdfs = rawpdfs.textinpage

forecast19 = ForecastPDF(pdfs, ene_dict, trgt_dict, prdt_dict, pdf_index=2019, start=5, end=13)
forecast18 = ForecastPDF(pdfs, ene_dict, trgt_dict, prdt_dict, pdf_index=2018, start=2, end=15)


from django.shortcuts import render
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor

def home(request):
    return render(request,'home.html')
def predict(request):
    return render(request,'predict.html')
def result(request):
    df = pd.read_excel(r'C:\Users\Gaayana\OneDrive\Desktop\DiabetesPrediction\DiabetesPrediction\DiabetesPrediction\dataset1.xlsx').drop(['Name','Gender'], axis = 1)
    # Convert data types
    df = df.astype({
        'Age': 'int64',
        'Ambient Temperature': 'int64',
        'Ambient Humidity': 'int64',
        'Skin temperature': 'float64',
        'Heart rate': 'int64',
        'SpO2': 'int64',
        'Mean_bp': 'float64',
        'Galvanic skin response': 'int64',
        'Blood glucose level': 'int64'
    })

    # Rename columns
    df.columns = ['age', 'ambient_temp', 'ambient_humidity', 'skin_temp', 'heart_rate', 'spo2', 'mean_bp', 'gsr', 'glucose']

    # Remove outliers using the IQR method
    Q1 = df.quantile(0.25)
    Q3 = df.quantile(0.75)
    IQR = Q3 - Q1
    df = df[~((df < (Q1 - 1.5 * IQR)) | (df > (Q3 + 1.5 * IQR))).any(axis=1)]

    # Split the data into training and testing sets
    X_train, X_test, y_train, y_test = train_test_split(df.drop('glucose', axis=1),
                                                        df['glucose'],
                                                        test_size=0.2,
                                                        random_state=42)

    # create the GradientBoostingRegressor model
    gbr = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, max_depth=3, random_state=42)

    # train the model on the training data
    gbr.fit(X_train, y_train)

    #fetch values
    val1 = float(request.GET['n1'])
    val2 = float(request.GET['n2'])
    val3 = float(request.GET['n3'])
    val4 = float(request.GET['n4'])
    val5 = float(request.GET['n5'])
    val6 = float(request.GET['n6'])
    val7 = float(request.GET['n7'])
    val8 = float(request.GET['n8'])

    #prediction
    pred = gbr.predict([[val1,val2,val3,val4,val5,val6,val7,val8]])

    #display result
    return render(request,'predict.html', {"result2":pred})


#!/usr/local/python-3.5.2/bin/python3.5

import os, sys, time, threading
import signal, uuid, argparse, socket

from config_class import *
#from db_class import *
from sqlite_class import *
from io_class import *

#config_file = os.path.dirname(os.path.realpath(sys.argv[0])) + "/dbtranscode.conf"
parser = argparse.ArgumentParser()
cf = os.path.dirname(os.path.realpath(sys.argv[0])) + "/dbtranscode.conf"
parser.add_argument('-m', action='store', dest='module', help='seeker | coder', default='seeker')
parser.add_argument('-c', action='store', dest='configfile', help='Config File', default=cf)
parser.add_argument('--version', action='version', version='%(prog)s 0.1')

results = parser.parse_args()
module = results.module
config_file = results.configfile

CONFIG = db_config(config_file)
SQL_CONNECTION = sqlite_DB(CONFIG.sql_db, CONFIG.sql_table)
IO = db_IO(CONFIG.logfile)
maxTranscodingSlots = 3

signal.signal(signal.SIGTERM, IO.handler)
signal.signal(signal.SIGINT, IO.handler)

IO.checkIoFolder(CONFIG.outfolder, CONFIG.watchfolder, CONFIG.formats.items())
SQL_CONNECTION.createTable()

IO.writelog('dbtranscode-' + module + '[' + socket.gethostname() + '] started.')

try:
    loop = 0
    while True:

        if(module == "seeker"):
            wf_array = os.listdir(CONFIG.watchfolder)
            for wf in wf_array:
                for file in os.listdir(CONFIG.watchfolder + "/" + wf):
                    fullPathFile = CONFIG.watchfolder + "/" + wf + "/" + file
                    if(IO.file_is_locked(fullPathFile)):
                        continue
                    if(IO.get_extension(file).lower() in CONFIG.extensions):
                        if (not SQL_CONNECTION.check_entry(fullPathFile)):
                            SQL_CONNECTION.add_entry(fullPathFile,wf)
                            IO.writelog('add '+ fullPathFile + ' to Queue')
            if(loop == 2):
                SQL_CONNECTION.db_clean()
                loop = 0

        elif(module == "coder"):
            nextjob = SQL_CONNECTION.next_job()
            if(nextjob):
                if(threading.activeCount() <= maxTranscodingSlots):
                    nextjob_id = nextjob['post_id']
                    SQL_CONNECTION.update_job(nextjob_id, "1","", 0,'0')
                    fCount = IO.getFrameCount(nextjob['filename'],CONFIG.mediainfo)
                    ff_command = CONFIG.gen_ff_cmd(IO, nextjob['filename'], nextjob['format'])
                    rewrap = []
                    if(nextjob['format'] == "avcintra100"):
                        try:
                            rewrap.append(CONFIG.bmxtranswrap + " -t op1a --afd 8 --ard-zdf-hdf -o \"" + CONFIG.outfolder + "/" + os.path.splitext(IO.get_filename(nextjob['filename']))[0] + "_.mxf\"" + " \"" +  CONFIG.outfolder + "/" + os.path.splitext(IO.get_filename(nextjob['filename']))[0] + ".mxf\"")
                            rewrap.append("/usr/bin/rm \"" + CONFIG.outfolder + "/" + os.path.splitext(IO.get_filename(nextjob['filename']))[0] + ".mxf\"")
                        except:
                            pass
                    else:
                        rewrap = []

                    try:
                        t = threading.Thread(target=IO.runCMD, args=[ff_command,fCount,SQL_CONNECTION,nextjob_id, nextjob['filename'], rewrap])
                        t.start()
                        IO.writelog('start ' + nextjob['filename'] + ' transcode to ' + nextjob['format'] )
                    except Error as e:
                        print ("Error %d: %s" % (e.args[0], e.args[1]))

        else:
            sys.exit(2)
        time.sleep(2)
        loop += 1

finally:
    time.sleep(1)
    IO.writelog('dbtranscoder ' + module + ' stopped.')
    print ("exit")


# 530. Minimum Absolute Difference in BST
# Given the root of a BST, return the minimum absolute difference between the values of any two different nodes in the tree.
from typing import Optional

class TreeNode:
    def __init__(self, val, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class BST:
    def getMinimumDifference(self, root: Optional[TreeNode]) -> int:
        def dfs(node):
            if not node:
                return
            dfs(node.left)
            values.append(node.val)
            dfs(node.right)

        values = []
        dfs(root)
        ans = float("inf")
        for i in range(1, len(values)):
            ans = min(ans, values[i] - values[i - 1])
        return ans

one = TreeNode(1)
three = TreeNode(3)
five = TreeNode(5)
two = TreeNode(2, one, three)
four = TreeNode(4, two, five)  # root

bst = BST()
print(bst.getMinimumDifference(four))  # Output: 1


import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import precision_score, recall_score, precision_recall_curve
import nltk
# nltk.download('punkt')
# nltk.download('stopwords')
# nltk.download('SnowballStemmer')
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import SnowballStemmer
import string

#Making a dataframe

df = pd.read_csv('labeled.csv', sep=',')
df['toxic'] = df['toxic'].apply(int)
train_df, test_df = train_test_split(df, test_size=500)
#Text Preprocessing

snowball = SnowballStemmer(language = 'russian')
rswords = stopwords.words('russian')

def tokenize(sentence: str, remove_swords: bool=True):
    tokens_begin = word_tokenize (sentence, language = 'russian')
    tokens_wout_punct = [i for i in tokens_begin if i not in string.punctuation]
    tokens_wout_punct_and_rswords = [i for i in tokens_wout_punct if i not in rswords]
    tokens = [snowball.stem(i) for i in tokens_wout_punct_and_rswords]
    return tokens

vectorizer = TfidfVectorizer (tokenizer = lambda x: tokenize(x))
features = vectorizer.fit_transform(train_df['comment'])
#Model which is learning on a vectors
# model = LogisticRegression(random_state=0)
# model.fit(features, train_df['toxic'])
#Model which is learning on a sentences
model_pipeline = Pipeline([
    ('vectorizer', TfidfVectorizer (tokenizer = lambda x: tokenize(x))),
    ('model', LogisticRegression(random_state=0))
])
model_pipeline.fit(train_df['comment'], train_df['toxic'])

#Metrics

print(precision_score(y_true=test_df['toxic'], y_pred=model_pipeline.predict(test_df['comment'])))
print(recall_score(y_true=test_df['toxic'], y_pred=model_pipeline.predict(test_df['comment'])))

flag=True
while flag:
    print('Your comment')
    com = str(input())
    if model_pipeline.predict([com])==0:
        print("It's ok")
    else:
        print("It's not ok")
    print('Continue? y/n')
    ans = input()
    if ans == 'n':
        flag = False


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 23 20:35:18 2019

@author: ai
"""
import torch.utils.data as data
import cv2
import os
import os.path
import numpy as np
import torch

def has_file_allowed_extension(filename, extensions):
    """Checks if a file is an allowed extension.

    Args:
        filename (string): path to a file

    Returns:
        bool: True if the filename ends with a known image extension
    """
    filename_lower = filename.lower()
    return any(filename_lower.endswith(ext) for ext in extensions)

def find_classes(dir):
    classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))]
    classes.sort()
    class_to_idx = {classes[i]: i for i in range(len(classes))}
    return classes, class_to_idx

def make_dataset(dir, class_to_idx, extensions):
    images = []
    dir = os.path.expanduser(dir)
    for target in sorted(os.listdir(dir)):
        d = os.path.join(dir, target)
        if not os.path.isdir(d):
            continue

        for root, _, fnames in sorted(os.walk(d)):
            for fname in sorted(fnames):
                if has_file_allowed_extension(fname, extensions):
                    path = os.path.join(root, fname)
                    item = (path, class_to_idx[target])
                    images.append(item)

    return images
IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png']
class Dataset_car_stanford(data.Dataset):
    """A generic data loader where the samples are arranged in this way: ::

        root/class_x/xxx.ext
        root/class_x/xxy.ext
        root/class_x/xxz.ext

        root/class_y/123.ext
        root/class_y/nsdf3.ext
        root/class_y/asd932_.ext

    Args:
        root (string): Root directory path.
        loader (callable): A function to load a sample given its path.
        extensions (list[string]): A list of allowed extensions.
        transform (callable, optional): A function/transform that takes in
            a sample and returns a transformed version.
            E.g, ``transforms.RandomCrop`` for images.
        target_transform (callable, optional): A function/transform that takes
            in the target and transforms it.

     Attributes:
        classes (list): List of the class names.
        class_to_idx (dict): Dict with items (class_name, class_index).
        samples (list): List of (sample path, class_index) tuples
    """

    def __init__(self, root,  transform=None, target_transform=None):
        self.root = root
        classes, class_to_idx=find_classes(self.root)
        samples = make_dataset(root, class_to_idx, IMG_EXTENSIONS)
        if len(samples) == 0:
            raise(RuntimeError("Found 0 files in subfolders of: " + root + "\n"
                               "Supported extensions are: " + ",".join(IMG_EXTENSIONS)))

        self.loader = cv_loader
        self.samples = samples

        self.transform = transform
        self.target_transform = target_transform

    def __getitem__(self, index):
        """
        Args:
            index (int): Index

        Returns:
            tuple: (sample, target) where target is class_index of the target class.
        """
        path, target = self.samples[index]
        sample = self.loader(path)
        if self.transform is not None:
            sample = self.transform(sample)
        if self.target_transform is not None:
            target = self.target_transform(target)

        return to_float_tensor(sample), target

    def __len__(self):
        return len(self.samples)

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        fmt_str += '    Root Location: {}\n'.format(self.root)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        tmp = '    Target Transforms (if any): '
        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

class Test_loader(data.Dataset):
    """Face Landmarks dataset."""

    def __init__(self, root_dir, transform=None):
        """
        Args:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.loader = cv_loader
        self.root_dir = root_dir
        self.image_files_list = [s for s in os.listdir(self.root_dir)]
        self.transform = transform

    def __len__(self):
        return len(self.image_files_list)

    def __getitem__(self, idx):
        img_name = os.path.join(self.root_dir, self.image_files_list[idx])
        label = np.zeros((196,))
        image = self.loader(img_name)
        image = self.transform(image)

        return to_float_tensor(image), label, self.image_files_list[idx]

IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif']

def cv_loader(path):
    img=cv2.imread(path,1).astype(np.uint8)
    #img = cv2.resize(img, (128, 128), interpolation=cv2.INTER_LINEAR)
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    return img
def to_float_tensor(img):
    return torch.from_numpy(np.moveaxis(img, -1, 0)).float()

def save_checkpoint(state, is_best, filename):
    if is_best:
        print ('Saved best!')
        torch.save(state, filename)
def load_checkpoint(checkpoint_path, model, optimizer):
    state = torch.load(checkpoint_path)
    model.load_state_dict(state['model'])
    #optimizer.load_state_dict(state['optimizer'])
    print('model loaded from %s' % checkpoint_path)


#https://github.com/Bjarten/early-stopping-pytorch/blob/master/pytorchtools.py

from tqdm import tqdm
import torch
import torch.nn as nn
import numpy as np

from torchmetrics.classification import MulticlassPrecision
from torchmetrics.classification import MulticlassRecall

# Gradient scaler for amp (Mixed Precision)
scaler = torch.cuda.amp.GradScaler()

def train(model, train_loader, optimizer, criterion, device):
    model.train()
    train_loss = 0
    correct = 0
    total = 0
    loss_sum = 0
    len_loader = len(train_loader)
    p_tqdm = tqdm(train_loader)
    for batch_idx, (data, target) in enumerate(p_tqdm):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        #with torch.cuda.amp.autocast():
        #    output = model(data)
        #    loss = criterion(output, target)
        #scaler.scale(loss).backward()
        #scaler.step(optimizer)
        #scaler.update()

        output = model(data)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()
        _, predicted = output.max(1)
        total += target.size(0)
        correct += predicted.eq(target).sum().item()
        # update loss in each batch in progress bar
        train_loss = loss.item()
        loss_sum += loss.item()
        p_tqdm.set_postfix_str(
            f"Train Loss: {train_loss:.3f} | Train Acc: {100.*correct/total:.3f}")

    return loss_sum/len_loader, correct, total

def test(model, test_loader, criterion, device):
    model.eval()
    test_loss = 0
    correct = 0
    total = 0
    loss_sum = 0
    len_loader = len(test_loader)

    recall = MulticlassRecall(num_classes=4).to(device)
    precision = MulticlassPrecision(num_classes=4).to(device)
    prec, rec = [], []

    with torch.no_grad():
        p_tqdm = tqdm(test_loader)
        for batch_idx, (data, target) in enumerate(p_tqdm):
            data, target = data.to(device), target.to(device)
            output = model(data)
            loss = criterion(output, target)
            test_loss = loss.item()
            loss_sum += loss.item()
            _, predicted = output.max(1)
            rec.append(recall(predicted, target))
            prec.append(precision(predicted, target))
            total += target.size(0)
            correct += predicted.eq(target).sum().item()
            p_tqdm.set_postfix_str(
                f"Test Loss: {test_loss:.3f} | Test Acc: {100.*correct/total:.3f}")

    return loss_sum/len_loader, correct, total, sum(rec)/len(test_loader), sum(prec)/len(test_loader)

def get_probs_preds(model, loader, device):
    predicted = torch.tensor([])
    output_probs = torch.tensor([])
    with torch.no_grad():
        for data in loader:
            images, labels = data
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)
            outputs = nn.Softmax(dim=1)(outputs)
            _, predicted_new = torch.max(outputs.data, 1)
            predicted_new = predicted_new.cpu()
            predicted = torch.cat((predicted, predicted_new), 0)
            output_probs = torch.cat((output_probs, outputs.data.cpu()), 0)
    return predicted, output_probs

def get_right_labels(loader):
    correct_labels = []
    for data in loader:
        _, labels = data
        correct_labels.append(labels)
    correct_labels = torch.cat(correct_labels, 0)
    correct_labels = correct_labels.numpy()
    return correct_labels

class EarlyStopping:
    """Early stops the training if validation loss doesn't improve after a given patience."""
    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):
        """
        Args:
            patience (int): How long to wait after last time validation loss improved.
                            Default: 7
            verbose (bool): If True, prints a message for each validation loss improvement.
                            Default: False
            delta (float): Minimum change in the monitored quantity to qualify as an improvement.
                            Default: 0
            path (str): Path for the checkpoint to be saved to.
                            Default: 'checkpoint.pt'
            trace_func (function): trace print function.
                            Default: print
        """
        self.patience = patience
        self.verbose = verbose
        self.counter = 0
        self.best_score = None
        self.early_stop = False
        self.val_loss_min = np.Inf
        self.delta = delta
        self.path = path
        self.trace_func = trace_func
    def __call__(self, val_loss, model):

        score = -val_loss

        if self.best_score is None:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
        elif score < self.best_score + self.delta:
            self.counter += 1
            self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')
            if self.counter >= self.patience:
                self.early_stop = True
        else:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
            self.counter = 0

    def save_checkpoint(self, val_loss, model):
        '''Saves model when validation loss decrease.'''
        if self.verbose:
            self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')
        torch.save(model.state_dict(), self.path)
        self.val_loss_min = val_loss


import NetworkTrain
import CellFromIllustrator
import os
import argparse
import scipy.ndimage
import numpy as np
from PIL import Image
import time
import shutil
from Constants import SIZE, RADIUS

class OutputFileType:
    def __init__(self):
        self.svg = False
        self.csv = False
        self.jpg = False

    def any(self):
        return self.svg or self.csv or self.jpg

def get_next_number(folder):
    i = 0
    while True:
        i += 1
        if not os.path.exists(os.path.join(folder, "%02d.npz" % i)):
            break
    return os.path.join(folder, "%02d" % i)

def detect_local_maxima(arr, mask=None):
    scale = 4
    dilated = scipy.ndimage.morphology.grey_dilation(arr, size=(scale, scale))
    eroded = scipy.ndimage.morphology.grey_erosion(arr, size=(scale, scale))
    x1 = arr == dilated
    x2 = arr > eroded
    x = np.logical_and(x1, x2)
    if mask is not None:
        x = x * mask
    x[:RADIUS, :] = False
    x[-RADIUS:, :] = False
    x[:, :RADIUS] = False
    x[:, -RADIUS:] = False
    return np.where(x)

def detect_local_minima(arr, mask=None):
    scale = 4
    dilated = scipy.ndimage.morphology.grey_dilation(arr, size=(scale, scale))
    eroded = scipy.ndimage.morphology.grey_erosion(arr, size=(scale, scale))
    x1 = arr == eroded
    x2 = arr < dilated - .01
    x = np.logical_and(x1, x2)
    if mask is not None:
        x = x * mask
    x[:RADIUS, :] = False
    x[-RADIUS:, :] = False
    x[:, :RADIUS] = False
    x[:, -RADIUS:] = False
    return np.where(x)

def local_maxima_generate_points(arr, mask=None, find_maxima=True):
    float_arr = arr.astype(np.float32)
    scale = 1
    fuzzy = scipy.ndimage.gaussian_filter(float_arr, sigma=scale)
    if find_maxima:
        maxima = np.array(detect_local_maxima(fuzzy, mask)).T
    else:
        maxima = np.array(detect_local_minima(fuzzy, mask)).T
    assert maxima.shape[0] > 0
    X = np.zeros(shape=(maxima.shape[0], 1, SIZE, SIZE), dtype=np.float32)
    for i in range(maxima.shape[0]):
        x, y = maxima[i, :]
        X[i, 0, :, :] = arr[x - RADIUS:x + RADIUS, y - RADIUS:y + RADIUS]
    return X, maxima

class CreatePNG():
    def __init__(self, output_bouton_png_name, artboard_size_xy, downscale=1.0):
        self.output_name = output_bouton_png_name
        self.x = artboard_size_xy[0]
        self.y = artboard_size_xy[1]
        self.arr = np.zeros(shape=(self.x, self.y), dtype=np.bool)
        self.radius = 5
        self.symbol_shape = np.zeros((self.radius * 2 + 1, self.radius * 2 + 1), dtype=np.bool)
        self.downscale = downscale
        for x in range(self.radius * 2 + 1):
            for y in range(self.radius * 2 + 1):
                if (x - self.radius) ** 2 + (y - self.radius) ** 2 <= self.radius ** 2:
                    self.symbol_shape[x, y] = True

    def add_symbol(self, location_xy):
        try:
            y, x = location_xy[0], location_xy[1]
            self.arr[x - self.radius: x + self.radius + 1, y - self.radius: y + self.radius + 1] = \
                np.logical_or(self.arr[x - self.radius: x + self.radius + 1, y - self.radius: y + self.radius + 1], self.symbol_shape)
        except ValueError as e:
            pass

    def output(self):
        image_array = np.zeros((self.arr.shape[0], self.arr.shape[1], 4), dtype=np.uint8)
        image_array[:, :, 0] = 255  # Convert to image array
        image_array[:, :, 1] = 255 - 255 * self.arr
        image_array[:, :, 2] = 255 - 255 * self.arr
        image_array[:, :, 3] = self.arr * 255
        image = Image.fromarray(image_array, 'RGBA')
        image = image.resize((int(self.arr.shape[1] * self.downscale), int(self.arr.shape[0] * self.downscale)), Image.BILINEAR)
        image.save(self.output_name)

class CreateSVG():
    def __init__(self, output_svgname, artboard_size_xy, input_image):
        self.output_svgname = output_svgname
        x = artboard_size_xy[1]
        y = artboard_size_xy[0]
        self.strings = ["""
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="%s" height="%s" viewBox="0 0 %s %s">
  <defs>
    <style>
      .cls-1 {
        isolation: isolate;
      }
    </style>
    <symbol id="Bouton" data-name="Bouton" viewBox="0 0 1 1">
      <circle cx="0.5" cy="0.5" r="0.5"/>
    </symbol>
  </defs>
  <image class="cls-1" x="0.5" y="0.5" width="%s" height="%s" xlink:href="%s"/>
    """ % (x, y, y, x, y, x, input_image)]

    def add_symbol(self, location_xy):
        self.strings.append(
            """  <use width="1" height="1" transform="translate(%s %s)" xlink:href="#Bouton"/>
            """ % (location_xy[0], location_xy[1]))

    def output(self):
        self.strings.append(\
"""</svg>""")
        string = "".join(self.strings)
        with open(self.output_svgname, 'w+') as f:
            f.write(string)

def generate_border_and_mask(mask_file_path):
    if mask_file_path is None:
        return None, None
    else:
        mask = CellFromIllustrator.file_to_array(mask_file_path)
        mask = mask > 128
        border = np.bitwise_and(mask, np.invert(scipy.ndimage.morphology.binary_erosion(mask)))
        return mask, border

def count_section(directory, model, oft):
    input_mask = os.path.join(directory, "Mask.jpg")
    output_image = os.path.join(directory, "Image.jpg")
    output_svg = os.path.join(directory, "Boutons.svg")
    output_png = os.path.join(directory, "Boutons.png")
    output_csv = os.path.join(directory, "Boutons.csv")
    arr = None
    start_time = time.time()
    if os.path.exists(output_image):
        arr = CellFromIllustrator.file_to_array(output_image).astype(np.float32)
    else:
        files = [f for f in sorted(os.listdir(directory)) if
                 os.path.isfile(os.path.join(directory, f)) and (f.endswith(".tif") or f.endswith("png")) and "IMAGE" in f]
        if len(files) == 0:  # If exported through CellSens
            for r, d, f in os.walk(directory, topdown=False):
                for file in f:
                    if file.endswith(".tif") and "EFI" in file:
                        print(os.path.join(r, file))
                        arr = CellFromIllustrator.file_to_array(os.path.join(r, file)).astype(np.float32)
                        os.remove(os.path.join(r, file))
                if r != directory:
                    os.rmdir(r)
        else:
            try:
                arrs = []
                for file in files:  # If exported through VS-ASW
                    arrs.append(CellFromIllustrator.file_to_array(os.path.join(directory, file)).astype(np.float32))
                    os.remove(os.path.join(directory, file))
                if arrs[0].shape[0] > arrs[0].shape[1]:
                    arr = np.concatenate(arrs, axis=1)
                else:
                    arr = np.concatenate(arrs, axis=0)
            except OSError:
                print("Images for %s do not exist" % directory)
                return
        print("Working on %s" % output_image)
    arr = (arr - arr.min()) / (arr.max() - arr.min())
    im = Image.fromarray(arr * 255).convert("L")
    im.save(output_image)
    if oft.any():
        mask = None
        if os.path.exists(input_mask):
            mask, _ = generate_border_and_mask(input_mask)
        print("Working on %s" % output_svg)
        print("Load Time: %.2f" % (time.time() - start_time))
        start_time = time.time()
        X, COMs = local_maxima_generate_points(arr, mask, find_maxima=False)
        print("Proposal Time: %.2f" % (time.time() - start_time))
        start_time = time.time()
        output = model.predict(X)
        print("ML Time: %.2f" % (time.time() - start_time))
        start_time = time.time()
        true_cells = COMs[output[:, 0] > 0.95, :]
        if oft.svg and not os.path.exists(output_svg):
            svg = CreateSVG(output_svg, arr.shape, output_image)
            for i in range(true_cells.shape[0]):
                svg.add_symbol(location_xy=(true_cells[i, 1], true_cells[i, 0]))
            svg.output()
        if oft.jpg:
            jpg = CreatePNG(output_png, arr.shape, downscale=0.3)
            for i in range(true_cells.shape[0]):
                jpg.add_symbol(location_xy=(true_cells[i, 1], true_cells[i, 0]))
            jpg.output()
        if oft.csv:
            str = ""
            for i in range(true_cells.shape[0]):
                str += "%s,%s%s" % (true_cells[i, 1], true_cells[i, 0], os.linesep)
            with open(output_csv, 'w') as f:
                f.write(str)
        print("Output Time: %.2f" % (time.time() - start_time))

def count_brain(brain_directory, model, oft):
    for directory in sorted(os.listdir(brain_directory)):
        directory = os.path.join(brain_directory, directory)
        if os.path.isdir(directory):
            count_section(directory, model, oft)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="This program uses the pretrained NN to search for cells or boutons")
    parser.add_argument("-d", "--directory", type=str, help="Brain level directory")
    parser.add_argument("-o", "--order", type=str, help="Order file for multiple brains")
    parser.add_argument("-f", "--folder", type=str, help="File for one section from one brain")
    parser.add_argument("-j", "--jpg", action="store_true", help="Output jpg file Boutons.tif")
    parser.add_argument("-s", "--svg", action="store_true", help="Output svg file Boutons.svg")
    parser.add_argument("-c", "--csv", action="store_true", help="Output csv file Boutons.csv")
    args = parser.parse_args()
    Image.MAX_IMAGE_PIXELS = None
    model = NetworkTrain.load_json_model("Boutons")
    oft = OutputFileType()
    oft.svg, oft.jpg, oft.csv = args.svg, args.jpg, args.csv
    if args.directory is not None:
        count_brain(args.directory, model, oft)
    elif args.order is not None:
        dir_name = os.path.dirname(args.order)
        with open(args.order) as f:
            for line in f:
                line = os.path.join(dir_name, line.strip())
                count_brain(line, model, oft)
    elif args.folder is not None:
        count_section(args.folder, model, oft)


import matplotlib.pyplot as plt
import numpy as np
import xml.etree.ElementTree as ET
import copy
from math import sqrt
from matplotlib.patches import Polygon
from io import BytesIO
from app.models.table import Table

class Analisis:
    def __init__(self, tabla: Table, clase):
        self.tabla = copy.deepcopy(tabla)
        self.nombre_clase = clase
        self.clase = self.obtiene_atributo(clase) # La clase para este conjunto de datos (puede cambiar a petición del usuario)
        self.index_clase = self.obtiene_index(self.clase)
        self.tipos_numericos = ("integer", "number", "float")
        self.tipos_categoricos = ("enum", "boolean", "categorical")
        self.tipos_clase = ("enum", "categorical", "boolean")
        self.tipos_numericos_itr = (type(int()), type(float()))
        self.tipos_categoricos_itr = (type(""),)
        self.atr_clase = self.obtiene_atr_clase() # Obtenemos los atributos categóricos que podíamos utilizar como clases
        self.atributos = self.tabla.properties.props['attributes']
        # self.tabla_dep_tot = self.obtiene_tabla_dep() # Tabla depurada en su totalidad
        # (sin registros con valores faltantes o fuera del dominio para ningún atributo de la tabla)
        # self.tabla_remp_tot = self.obtiene_tabla_remp() # Tabla reemplazada en su totalidad
        #  (con los valores faltantes reemplazados para todos los atributos de la tabla)

    # Obtenemos los atributos categóricos que podríamos utilizar como clases
    def obtiene_atr_clase(self):
        atributos_clase = []
        for atr in self.tabla.properties.propsAttr:
            if atr['type'] in self.tipos_clase:
                # Sabemos que el atributo actual es categórico, lo agregamos a la lista
                atributos_clase.append(atr)
        # Si la lista contiene elementos la devolvemos, si no, devolvemos None
        return atributos_clase if atributos_clase != [] else None

    # Obtenemos el indice del atributo recibido como parametro, en caso de existir
    def obtiene_index(self, atributo):
        temp = list(self.tabla.properties.props['attributes'])
        indice = 0
        for item in temp:
            if item['name'] == atributo['name']:
                break
            indice = indice + 1
        return indice

    def obtiene_dominio(self, atributo):
        if(not atributo['type'] in self.tipos_categoricos):
            return None
        if(atributo['type'] == 'boolean'):
            # Los atributos boolean, aunque sabemos que son categoricos, no tendrán clases como tal
            return ('False', 'True')
        # Indice del atributo especificado
        indice = self.obtiene_index(atributo)
        return tuple(self.tabla.properties.propsAttr[indice]['classes'])

    # Obtenemos un atributo por nombre
    def obtiene_atributo(self, nombre):
        for atr in self.tabla.properties.propsAttr:
            if atr['name'] == nombre:
                return atr # Encontramos el atributo, lo devolvemos
        return None # No encontramos el atributo, devolvemos none

    def val_fuera_dominio(self):
        return None

    def val_faltantes(self):
        faltantes = []
        for reg in self.tabla.data:
            # Iteramos en cada uno de los registros del conjunto de datos
            for indice in range(1, len(reg)):
                if not reg[indice - 1]:
                    # Si el dato está vacío agregamos todo el registro al arreglo de valores faltantes
                    faltantes.append(reg)
        return faltantes

    def obtiene_tabla_dep(self):
        tabla_aux = copy.deepcopy(self.tabla)
        eliminar = []
        # Obtenemos todos los registros con valores fuera del dominio
        fuera_dominio = self.val_faltantes()
        # Obtenemos todos los registros con valores faltantes
        # Concatenamos registros faltantes con registros fuera de dominio
        eliminar = fuera_dominio
        # Eliminamos dichos registros de la tabla y devolvemos dicha tabla
        for registro in eliminar:
            tabla_aux.remove(registro)
        return tabla_aux

class AnalisisUni(Analisis):
    def __init__(self, tabla, atributo, clase):
        super().__init__(tabla, clase)
        # Información atributo a analizar del conjunto de datos
        self.atributo = atributo
        self.indice = self.obtiene_index(atributo)
        self.nombre = atributo['name']
        self.tipo = atributo['type']
        self.dominio = self.obtiene_dominio(self.atributo)
        self.datos_crud = self.obtiene_datos_crudos()
        # Estadísticas de registros con datos faltantes
        self.faltantes = self.val_faltantes()
        self.cant_faltantes = len(self.faltantes)
        self.porc_faltantes = f'{round(self.cant_faltantes * 100 / len(tabla.data), 2)}%'
        # Estadísticas de registros con datos fuera del dominio
        self.fuera_dominio = self.val_fuera_dominio()
        self.cant_fuera_dominio = len(self.fuera_dominio)
        self.porc_fuera_dominio = f'{round(self.cant_fuera_dominio * 100 / len(tabla.data), 2)}%'
        # Tabla con valores depurados (sin faltantes ni fuera de domino)
        self.tabla_dep = self.depura_tabla()
        # Estadísticas básicas
        self.media = self.obtiene_media()
        self.mediana = self.obtiene_mediana()
        self.moda = self.obtiene_moda()
        self.desviacion_estandar = self.obtiene_desviacion()

    def depura_tabla(self):
        # Eliminamos de la lista de registros los registros con valores vacíos
        aux = list(self.tabla.data)
        for vacio in self.faltantes:
            aux.remove(vacio)
        # Eliminamos de la lista los registros con valores fuera de dominio
        for fuera in self.fuera_dominio:
            if fuera in aux:
                aux.remove(fuera)
        return aux

    # Obtención de datos crudos, los devolveremos en una lista
    def obtiene_datos_crudos(self):
        datos = []
        for reg in self.tabla.data:
            valor = reg[self.indice]
            datos.append(valor if valor else 0)
        return datos

    # Obtención de datos discrimiados por clase, los devolveremos en un diccionario
    # o en una lista, dependiendo de lo que se necesite
    def obtiene_datos_disc(self, clase, regresa_tipo = 'lista'):
        # Obtenemos el index de la clase en el conjunto de datos
        index_clase = self.obtiene_index(clase)
        datos = {}
        etiquetas = []
        # Obtenemos el dominio de la clase especificada
        dominio_clase = self.obtiene_dominio(clase)
        for reg in self.tabla_dep:
            clase = reg[index_clase]
            # No incuiremos en el conjunto de datos los que tienen un valor de clase fuera del dominio
            if clase in dominio_clase:
                if not clase in datos:
                    datos[f'{clase}'] = [] # Inicializamos la lista en el diccionario
                    etiquetas.append(clase) # Agregamos la etiqueta nueva
                datos[f'{clase}'].append(reg[self.indice])
        if regresa_tipo == 'diccionario':
            # La llamada espera un diccionario de listas
            return datos, etiquetas
        elif regresa_tipo == 'lista':
            # La llamada espera una lista de listas
            datos_lista = []
            for entrada in datos.values():
                datos_lista.append(entrada)
            return datos_lista, etiquetas
        # Cualquier valor para regresa_tipo no válido devolverá None
        return None

    # Obtención de datos clasificados para presentación en istográma, esto es, número de
    # instancias por clase (solo válido para atributos categóricos)
    # los devolveremos en un diccionario o en una lista
    # dependiendo de lo que se necesite
    def obtiene_datos_isto(self):
        datos = [] # Número de datos para cada clase encontrada en el atributo
        pos_val = 0 # El número de posibles valores
        for reg in self.tabla_dep:
            clase = reg[self.indice]
            if clase in self.dominio:
                if not clase in datos:
                    pos_val += 1
                datos.append(clase)
        return datos, pos_val

    def muestra_box_plot(self, clase):
        # El box plot solamente podrá ser creado para atributos numéricos
        if(not self.atributo['type'] in str(self.tipos_numericos)):
            return None
        datos = [] # Los datos que utilizaremos para llenar el box plot
        fig, axs = plt.subplots() # Creamos un subplot y una figura
        if clase:
            # Diccionario con los datos discriminado por la clase especificada
            datos, etiquetas = self.obtiene_datos_disc(clase, 'lista')
            axs.boxplot(datos, labels = etiquetas) # Cargamos los datos crudos en el box plot
        else:
            datos = self.datos_crud
            axs.boxplot(datos) # Cargamos los datos crudos en el box plot
        axs.set_title(f'Box plot - {self.nombre}')
        plt.show()

    def muestra_istograma(self):
        # El istograma solamente podrá ser creado para atributos categóricos
        if (not self.atributo['type'] in self.tipos_categoricos):
            return None
        # Código obtenido del siguiente enlace
        # https://matplotlib.org/3.2.1/gallery/statistics/hist.html?highlight=isto
        datos, pos_val = self.obtiene_datos_isto()
        fig, axs = plt.subplots(1, 1, sharey=True, tight_layout=True)

        # Podemos configurar el número de contenedores por medio del atributo 'bins'
        axs.hist(datos, pos_val)
        plt.show()

    def obtiene_media(self):
        # La media solo la obtendremos para valores numéricos
        if( not self.atributo['type'] in str(self.tipos_numericos)):
            return None

        total = 0
        n_instancias =  len(self.tabla.data)

        for reg in self.tabla.data:
            if reg[self.indice]:
                total = total + int(reg[self.indice])

        return round(total / n_instancias, 2)

    def obtiene_mediana(self):
        # La mediana solo la obtendremos para valores numéricos
        if( not self.atributo['type'] in str(self.tipos_numericos)):
            return None
        # Obtenemos el valor mínimo y máximo para el atributo dado
        minimo = min(self.tabla_dep, key=lambda x: x[self.indice])
        minimo = minimo[self.indice]
        maximo = max(self.tabla_dep, key=lambda x: x[self.indice])
        maximo = maximo[self.indice]
        return round((maximo + minimo) / 2, 2)

    def obtiene_moda(self):
        # La moda solo la obtendremos para valores numéricos
        if(not self.atributo['type'] in str(self.tipos_numericos)):
            return None
        frecuencias = {}
        for registro in self.tabla_dep:
            if(registro[self.indice] not in frecuencias.keys()):
                # Si el diccionario aún no tiene esta cantidad agregada la agregamos
                frecuencias[f'{registro[self.indice]}'] = 0
            frecuencias[f'{registro[self.indice]}'] += 1
        # Devolvemos el valor con mas instancias en el conjunton de datos
        return max(frecuencias)

    def obtiene_desviacion(self):
        # La desviación estandar solo la obtendremos para valores numéricos
        if (not self.tipo in str(self.tipos_numericos)):
            return None
        res = 0
        for reg in self.tabla_dep:
            res += pow(reg[self.indice] - self.media, 2)
        # Dividimos la suma de las desviaciones entre el numero de instancias menos uno
        # (se trata de una muestra) y obtenemos raiz cuadrada
        res = res / len(self.tabla.data) - 1
        res = sqrt(res)
        return round(res, 2)

    # Método que recivirá un string con el nombre de un atributo,
    # obtendrá las instancias con valores faltantes para dicho atributo en
    # el conjunto de datos y devolverá un diccionario con dichas instancias
    def val_faltantes(self):
        # Obtenemos los valores faltantes en el indice encontrado
        faltantes = []
        for reg in self.tabla.data:
            # Iteramos en cada uno de los registros del conjunto de datos
            if not reg[self.indice]:
                # Si el dato está vacío agregamos todo el registro al arreglo de valores faltantes
                faltantes.append(reg)
        return faltantes

    def val_fuera_dominio(self):
        # Obtenemos el tipo de dato, el rango en caso de tratarse de un atributo numérico
        # y el dominio en caso de tratarse de uno categórico, obtendremos esta información de
        # tabla.properties.propsAttr
        fuera_dominio = []
        if (self.tipo in str(self.tipos_numericos)):
            # Nos encontramos con un atributo numérico
            for reg in self.tabla.data:
                # Checaremos que cada instancia del atributo sea del tipo requerido
                if(not type(reg[self.indice]) in self.tipos_numericos_itr):
                    # El dato no es del tipo correcto, lo agregamos a la lista de
                    # registros fuera del dominio
                    fuera_dominio.append(reg)
                # Checaremos que cada instancia se encuentre dentro del rango permitido
                # PendientesSaul!!!

        elif (self.tipo in str(self.tipos_categoricos)):
            # Nos encontramos con un atributo categórico
            # Checaremos que el valor se ecuentre dentro del dominio permitido
            for reg in self.tabla.data:
                # Checamos que cada instancia del atributo sea del tipo requerido
                if(not type(reg[self.indice]) in self.tipos_categoricos_itr):
                    # El dato no se del tipo correcto
                    fuera_dominio.append(reg)
                else:
                    if(not reg[self.indice] in str(self.dominio)):
                        # El valor no se encuentra dentro del dominio
                        fuera_dominio.append(reg)
        # Devolveremos la lista de instancias con valores fuera del dominio
        return fuera_dominio

class AnalisisBi(Analisis):
    def __init__(self, tabla, an1, an2, clase):
        super().__init__(tabla, clase)
        self.analisis_1 = an1 # El objeto de analisis univariable uno
        self.analisis_2 = an2 # El objeto de analisis univariable dos

# class Zero_R():
#     def __init__(self, tabla, clase, Analisis):
#         self.tablas_frecuencia = Tabla_frecuencias.tablas_frecuencia(Analisis, clase)
#         self