from collections import namedtuple

import cerberus
import torch

from ._utils import static_vars

LEARNING_RATE_IO_DESCRIPTION_NAME = "__learning_rate"
ALL_FINITE_IO_DESCRIPTION_NAME = "__all_finite"
LOSS_SCALE_INPUT_IO_DESCRIPTION_NAME = "__loss_scale_input_name"
GRADIENT_ACCUMULATION_IO_DESCRIPTION_NAME = "__gradient_accumulation_name"

class _ORTTrainerModelDesc:
    def __init__(self, model_desc):
        # Keep a copy of original input for debug
        self._original = dict(model_desc)

        # Global counter used to validate occurrences of 'is_loss=True' whithin 'model_desc.outputs'
        #   A stateless validator is used for each tuple, but validation accross the whole list of tuple is needed
        #       because just one 'is_loss=True' is allowed withing 'model_desc.outputs' list of tuples
        _model_desc_outputs_validation.loss_counter = 0

        # Used for logging purposes
        self._main_class_name = self.__class__.__name__

        # Validates user input
        self._validated = dict(self._original)
        validator = cerberus.Validator(MODEL_DESC_SCHEMA)
        self._validated = validator.validated(self._validated)
        if self._validated is None:
            raise ValueError(f"Invalid model_desc: {validator.errors}")

        # Normalize inputs to a list of namedtuple(name, shape)
        self._InputDescription = namedtuple("InputDescription", ["name", "shape"])
        self._InputDescriptionTyped = namedtuple("InputDescriptionTyped", ["name", "shape", "dtype"])
        for idx, input in enumerate(self._validated["inputs"]):
            self._validated["inputs"][idx] = self._InputDescription(*input)

        # Normalize outputs to a list of namedtuple(name, shape, is_loss)
        self._OutputDescription = namedtuple("OutputDescription", ["name", "shape", "is_loss"])
        self._OutputDescriptionTyped = namedtuple(
            "OutputDescriptionTyped", ["name", "shape", "is_loss", "dtype", "dtype_amp"]
        )
        for idx, output in enumerate(self._validated["outputs"]):
            if len(output) == 2:
                self._validated["outputs"][idx] = self._OutputDescription(*output, False)
            else:
                self._validated["outputs"][idx] = self._OutputDescription(*output)

        # Hard-code learning rate, all_finite descriptors
        self.learning_rate = self._InputDescriptionTyped(LEARNING_RATE_IO_DESCRIPTION_NAME, [1], torch.float32)

        # Convert dict in object
        for k, v in self._validated.items():
            setattr(self, k, self._wrap(v))

    def __repr__(self):
        """Pretty representation for a model description class"""

        pretty_msg = "Model description:\n"

        # Inputs
        inputs = []
        for i_desc in self.inputs:
            if isinstance(i_desc, self._InputDescription):
                inputs.append(f"(name={i_desc.name}, shape={i_desc.shape})")
            elif isinstance(i_desc, self._InputDescriptionTyped):
                inputs.append(f"(name={i_desc.name}, shape={i_desc.shape}, dtype={i_desc.dtype})")
            else:
                raise ValueError(f"Unexpected type {type(i_desc)} for input description")

        pretty_msg += "\nInputs:"
        for idx, item in enumerate(inputs):
            pretty_msg += f"\n\t{idx}: {item}"

        # Outputs
        outputs = []
        for o_desc in self.outputs:
            if isinstance(o_desc, self._OutputDescription):
                outputs.append(f"(name={o_desc.name}, shape={o_desc.shape})")
            elif isinstance(o_desc, self._OutputDescriptionTyped):
                outputs.append(
                    f"(name={o_desc.name}, shape={o_desc.shape}, dtype={o_desc.dtype}, dtype_amp={o_desc.dtype_amp})"
                )
            else:
                raise ValueError(f"Unexpected type {type(o_desc)} for output description")
        pretty_msg += "\nOutputs:"
        for idx, item in enumerate(outputs):
            pretty_msg += f"\n\t{idx}: {item}"

        # Learning rate
        if self.learning_rate:
            pretty_msg += "\nLearning rate: "
            pretty_msg += (
                f"(name={self.learning_rate.name}, shape={self.learning_rate.shape}, dtype={self.learning_rate.dtype})"
            )

        # Mixed precision
        if getattr(self, ALL_FINITE_IO_DESCRIPTION_NAME, None) or getattr(
            self, LOSS_SCALE_INPUT_IO_DESCRIPTION_NAME, None
        ):
            pretty_msg += "\nMixed Precision:"
            if getattr(self, ALL_FINITE_IO_DESCRIPTION_NAME, None):
                pretty_msg += "\n\tis gradients finite: "
                pretty_msg += (
                    f"(name={self.all_finite.name}, shape={self.all_finite.shape}, dtype={self.all_finite.dtype})"
                )
            if getattr(self, LOSS_SCALE_INPUT_IO_DESCRIPTION_NAME, None):
                pretty_msg += "\n\tloss scale input name: "
                pretty_msg += f"(name={self.loss_scale_input.name}, shape={self.loss_scale_input.shape}, dtype={self.loss_scale_input.dtype})"

        # Gradient Accumulation steps
        if self.gradient_accumulation:
            pretty_msg += "\nGradient Accumulation: "
            pretty_msg += f"(name={self.gradient_accumulation.name}, shape={self.gradient_accumulation.shape}, dtype={self.gradient_accumulation.dtype})"

        return pretty_msg

    def add_type_to_input_description(self, index, dtype):
        """Updates an existing input description at position 'index' with 'dtype' type information

        Args:
            index (int): position within 'inputs' description
            dtype (torch.dtype): input data type
        """

        assert isinstance(index, int) and index >= 0, "input 'index' must be a positive int"
        assert isinstance(dtype, torch.dtype), "input 'dtype' must be a torch.dtype type"
        existing_values = (*self.inputs[index],)
        if isinstance(self.inputs[index], self._InputDescriptionTyped):
            existing_values = (*existing_values[:-1],)
        self.inputs[index] = self._InputDescriptionTyped(*existing_values, dtype)

    def add_type_to_output_description(self, index, dtype, dtype_amp=None):
        """Updates an existing output description at position 'index' with 'dtype' type information

        Args:
            index (int): position within 'inputs' description
            dtype (torch.dtype): input data type
            dtype_amp (torch.dtype, default is None): input data type for evaluation with mixed precision
        """

        assert isinstance(index, int) and index >= 0, "output 'index' must be a positive int"
        assert isinstance(dtype, torch.dtype), "output 'dtype' must be a torch.dtype type"
        assert dtype_amp is None or isinstance(
            dtype_amp, torch.dtype
        ), "output 'dtype_amp' must be either None or torch.dtype type"
        existing_values = (*self.outputs[index],)
        if isinstance(self.outputs[index], self._OutputDescriptionTyped):
            existing_values = (*existing_values[:-2],)
        self.outputs[index] = self._OutputDescriptionTyped(*existing_values, dtype, dtype_amp)

    @property
    def gradient_accumulation(self):
        return getattr(self, GRADIENT_ACCUMULATION_IO_DESCRIPTION_NAME, None)

    @gradient_accumulation.setter
    def gradient_accumulation(self, name):
        self._add_output_description(
            self, name, [1], False, torch.bool, None, GRADIENT_ACCUMULATION_IO_DESCRIPTION_NAME, ignore_duplicate=True
        )

    @property
    def all_finite(self):
        return getattr(self, ALL_FINITE_IO_DESCRIPTION_NAME, None)

    @all_finite.setter
    def all_finite(self, name):
        self._add_output_description(
            self, name, [1], False, torch.bool, None, ALL_FINITE_IO_DESCRIPTION_NAME, ignore_duplicate=True
        )

    @property
    def loss_scale_input(self):
        return getattr(self, LOSS_SCALE_INPUT_IO_DESCRIPTION_NAME, None)

    @loss_scale_input.setter
    def loss_scale_input(self, name):
        self._add_input_description(
            self, name, [], torch.float32, LOSS_SCALE_INPUT_IO_DESCRIPTION_NAME, ignore_duplicate=True
        )

    def _add_input_description(self, node, name, shape, dtype=None, attr_name=None, ignore_duplicate=False):
        """Add a new input description into the node object

        If 'dtype' is specified, a typed input description namedtuple(name, shape, dtype) is created.
        Otherwise an untyped input description namedtuple(name, shape) is created instead.

        Args:
            node (list or object): node to append input description to. When 'node' is 'self.inputs',
                a new input description is appended to the list.
                Otherwise, a new input description is created as an attribute into 'node' with name 'attr_name'
            name (str): name of input description
            shape (list): shape of input description
            dtype (torch.dtype): input data type
            attr_name (str, default is None): friendly name to allow direct access to the output description
            ignore_duplicate (bool, default is False): silently skips addition of duplicate inputs
        """

        assert isinstance(name, str) and len(name) > 0, "'name' is an invalid input name"
        not_found = True
        if not ignore_duplicate:
            if id(node) == id(self.inputs):
                not_found = all([name not in i_desc.name for i_desc in node])
                assert not_found, f"'name' {name} already exists in the inputs description"
            else:
                not_found = attr_name not in dir(self)
                assert not_found, f"'attr_name' {attr_name} already exists in the 'node'"
        elif not not_found:
            return
        assert isinstance(shape, list) and all(
            [(isinstance(dim, int) or (isinstance(dim, str) and len(dim) > 0)) for dim in shape]
        ), "'shape' must be a list of int or str with length at least 1"
        assert dtype is None or isinstance(dtype, torch.dtype), "'dtype' must be either None or a torch.dtype type"
        if dtype:
            new_input_desc = self._InputDescriptionTyped(name, shape, dtype)
        else:
            new_input_desc = self._InputDescription(name, shape)

        if id(node) == id(self.inputs):
            self.inputs.append(new_input_desc)
        else:
            assert isinstance(attr_name, str) and len(attr_name) > 0, "Invalid 'attr_name'"
            setattr(node, attr_name, new_input_desc)

    def _add_output_description(
        self, node, name, shape, is_loss, dtype=None, dtype_amp=None, attr_name=None, ignore_duplicate=False
    ):
        """Add a new output description into the node object as a tuple

        When (name, shape, is_loss, dtype) is specified, a typed output description is created
        Otherwise an untyped output description (name, shape, is_loss) is created instead

        Args:
            node (list or object): node to append output description to. When 'node' is 'self.outputs',
                a new output description is appended to the list.
                Otherwise, a new output description is created as an attribute into 'node' with name 'attr_name'
            name (str): name of output description
            shape (list): shape of output description
            is_loss (bool): specifies whether this output is a loss
            dtype (torch.dtype): input data type
            dtype_amp (torch.dtype, default is None): input data type for evaluation with mixed precision.
            attr_name (str, default is None): friendly name to allow direct access to the output description
            ignore_duplicate (bool, default is False): silently skips addition of duplicate outputs
        """

        assert isinstance(name, str) and len(name) > 0, "'name' is an invalid output name"
        assert isinstance(shape, list) and all(
            [(isinstance(dim, int) or (isinstance(dim, str) and len(dim) > 0)) for dim in shape]
        ), "'shape' must be a list of int or str with length at least 1"
        assert isinstance(is_loss, bool), "'is_loss' must be a bool"

        not_found = True
        if not ignore_duplicate:
            if id(node) == id(self.outputs):
                not_found = all([name not in o_desc.name for o_desc in node])
                assert not_found, f"'name' {name} already exists in the outputs description"
                assert (
                    all([not o_desc.is_loss for o_desc in node]) if is_loss else True
                ), "Only one 'is_loss' is supported at outputs description"
            else:
                not_found = attr_name not in dir(self)
                assert not_found, f"'attr_name' {attr_name} already exists in the 'node'"
        elif not not_found:
            return

        assert dtype is None or isinstance(dtype, torch.dtype), "'dtype' must be either None or a torch.dtype type"
        if dtype:
            new_output_desc = self._OutputDescriptionTyped(name, shape, is_loss, dtype, None)
        else:
            new_output_desc = self._OutputDescription(name, shape, is_loss)

        if id(node) == id(self.outputs):
            self.outputs.append(new_output_desc)
        else:
            assert isinstance(attr_name, str) and len(attr_name) > 0, "Invalid 'attr_name'"
            setattr(node, attr_name, new_output_desc)

    def _wrap(self, v):
        """Add 'v' as self's attribute to allow direct access as self.v"""
        if isinstance(v, (list)):
            return type(v)([self._wrap(v) for v in v])
        elif isinstance(
            v,
            (
                self._InputDescription,
                self._InputDescriptionTyped,
                self._OutputDescription,
                self._OutputDescriptionTyped,
            ),
        ):
            return v
        elif isinstance(v, (tuple)):
            return type(v)([self._wrap(v) for v in v])
        elif isinstance(v, (dict, int, float, bool, str)):
            return _ORTTrainerModelDescInternal(self._main_class_name, v) if isinstance(v, dict) else v
        else:
            raise ValueError(
                f"Unsupported type for model_desc ({v})."
                "Only int, float, bool, str, list, tuple and dict are supported"
            )

class _ORTTrainerModelDescInternal(_ORTTrainerModelDesc):
    r"""Internal class used by ONNX Runtime training backend for input validation

    NOTE: Users MUST NOT use this class in any way!
    """

    def __init__(self, main_class_name, model_desc):
        # Used for logging purposes
        self._main_class_name = main_class_name

        # Convert dict in object
        for k, v in dict(model_desc).items():
            setattr(self, k, self._wrap(v))

def _model_desc_inputs_validation(field, value, error):
    r"""Cerberus custom check method for 'model_desc.inputs'

    'model_desc.inputs' is a list of tuples.
    The list has variable length, but each tuple has size 2

    The first element of the tuple is a string which represents the input name
    The second element is a list of shapes. Each shape must be either an int or string.
        Empty list represents a scalar output

    Validation is done within each tuple to enforce the schema described above.

    Example:

        .. code-block:: python

            model_desc['inputs'] = [('input1', ['batch', 1024]),
                                    ('input2', [])
                                    ('input3', [512])]
    """

    if not isinstance(value, tuple) or len(value) != 2:
        error(field, "must be a tuple with size 2")
    if not isinstance(value[0], str):
        error(field, "the first element of the tuple (aka name) must be a string")
    if not isinstance(value[1], list):
        error(field, "the second element of the tuple (aka shape) must be a list")
    else:
        for shape in value[1]:
            if not isinstance(shape, str) and not isinstance(shape, int) or isinstance(shape, bool):
                error(field, "each shape must be either a string or integer")

@static_vars(loss_counter=0)
def _model_desc_outputs_validation(field, value, error):
    r"""Cerberus custom check method for 'model_desc.outputs'

    'model_desc.outputs' is a list of tuples with variable length.
    The first element of the tuple is a string which represents the output name
    The second element is a list of shapes. Each shape must be either an int or string.
        Empty list represents a scalar output
    The third element is optional and is a flag that signals whether the output is a loss value

    Validation is done within each tuple to enforce the schema described above, but also
    throughout the list of tuples to ensure a single 'is_loss=True' occurrence.

    Example:

        .. code-block:: python

            model_desc['outputs'] = [('output1', ['batch', 1024], is_loss=True),
                                     ('output2', [], is_loss=False)
                                     ('output3', [512])]
    """

    if not isinstance(value, tuple) or len(value) < 2 or len(value) > 3:
        error(field, "must be a tuple with size 2 or 3")
    if len(value) == 3 and not isinstance(value[2], bool):
        error(field, "the third element of the tuple (aka is_loss) must be a boolean")
    elif len(value) == 3:
        if value[2]:
            _model_desc_outputs_validation.loss_counter += 1
        if _model_desc_outputs_validation.loss_counter > 1:
            error(field, "only one is_loss can bet set to True")
    if not isinstance(value[0], str):
        error(field, "the first element of the tuple (aka name) must be a string")
    if not isinstance(value[1], list):
        error(field, "the second element of the tuple (aka shape) must be a list")
    else:
        for shape in value[1]:
            if not isinstance(shape, str) and not isinstance(shape, int) or isinstance(shape, bool):
                error(field, "each shape must be either a string or integer")

# Validation schema for model description dictionary
MODEL_DESC_SCHEMA = {
    "inputs": {
        "type": "list",
        "required": True,
        "minlength": 1,
        "schema": {"check_with": _model_desc_inputs_validation},
    },
    "outputs": {
        "type": "list",
        "required": True,
        "minlength": 1,
        "schema": {"check_with": _model_desc_outputs_validation},
    },
}


import glob

from django.conf import settings

from ..device.models import Device
from ..train.models import CNNModel
from ..utils.aws_s3 import read_params_from_s3
from ..utils.storage import read_params_from_storage

def predict_image_path(image_path):
    client = Device.objects.get(id=settings.CLIENT_ID)
    model_path = client.current_model_path
    if settings.USE_AWS_STORAGE:
        params = read_params_from_s3(model_path)
    else:
        params = read_params_from_storage(model_path)
    model = CNNModel(num_channels=client.num_channels, num_classes=client.num_classes)
    model.set_weights(params)
    res = model.predict([image_path])
    print(res)
    return res

def validate(img_path_regex):
    """
    - img_path_regex: vd: image_path = 'src/train/Dataset/Test/benign/*.jpg'
    """
    breast_img = glob.glob(img_path_regex, recursive=True)
    client = Device.objects.get(id=settings.CLIENT_ID)
    model_path = client.current_model_path
    if settings.USE_AWS_STORAGE:
        params = read_params_from_s3(model_path)
    else:
        params = read_params_from_storage(model_path)
    model = CNNModel(num_channels=client.num_channels, num_classes=client.num_classes)
    model.set_weights(params)
    res = model.predict(breast_img)
    print(res)
    print("total: ", len(res))
    print("total_0: ", len([i for i in res if i == 0]))
    print("total_1: ", len([i for i in res if i == 1]))


from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_iris
from web_app.models import User , Tweet , db
# Will use the picke to load the file
import pickle
import os

#Path where the model will be saved
MODEL_PATH = os.path.join(os.path.dirname(__file__), os.path.pardir, "predictive_models", "saved_model.pkl")

# used to train the iris and save it
def train_model_and_save():
    print("Training the model")
    X, y = load_iris(return_X_y=True)
    classifier = LogisticRegression(random_state=49, max_iter=150)
    classifier.fit(X, y)

    # Saving the model
    with open(MODEL_PATH, "wb") as model:
        pickle.dump(classifier, model)

    return classifier

# Will return the trained model. Will read it to a
# file
def load_model():
    # Will load the model from the file
    with open(MODEL_PATH, "rb") as model:
        the_model = pickle.load(model)

    return the_model

# This is the method that will be used to train the
# for the twitter prediction
# The user1 and user2 are from the data base
def train_twitter_model(user1, user2):

    classifier = LogisticRegression(random_state=49, solver="lbfgs", multi_class="multinomial")
    # will get the embeddings and the screen name for the
    # two different  users
    embeddings = []
    userList = []

    tweets1 = user1.tweets
    tweets2 = user2.tweets

    # Getting the embedding from the tweets
    for tweet in tweets1:
        userList.append(user1.screen_name)
        embeddings.append(tweet.embedding)

    for tweet in tweets2:
        userList.append(user2.screen_name)
        embeddings.append(tweet.embedding)

    classifier.fit(embeddings, userList)
    # returning the model to do the prediction
    return classifier

if __name__ == "__main__":
    train_model_and_save()

    load_model()


import pygame
import sys
class Main:
    def __init__(self, scr_size):
        pygame.init()
        pygame.freetype.init()
        self.size = scr_size
        self.screen = pygame.display.set_mode(self.size)
        self.clock = pygame.time.Clock()
        self.process_stopped = False
        self.score = 0

    def quit(self, e):
        if e.type == pygame.QUIT:
            self.process_stopped = True
            sys.exit(0)
        if e.type == pygame.KEYDOWN:
            if e.key == pygame.K_ESCAPE:
                self.process_stopped = True
                sys.exit(0)


import sys
import os
from data.config import Config
from data.config import cfg
# 不要删除/home/zqr/data/test/下的目录
if __name__ == '__main__':
    # 用来整理整个流程
    # videosFolder 存放视频数据位置
    # imagesFolder 视频帧存放位置
    # opticalFolder 存放光流文件位置
    # dataInfoPath 数据文件存放位置
    # videosBboxPath 存放视频的bbox
    # imagesFolder_160 存放裁剪成160×160的视频帧
    # resultFolder 存放关键帧结果
    # optFilesFolder 存放原始视频光流图
    # optFilesFolder_160 存放resize成160×160光流图
    # if len(sys.argv) != 6:
    #     print("you should give enough param")
    #     sys.exit(1)
    # videosFolder = sys.argv[1]
    # imagesFolder = sys.argv[2]
    # opticalFolder = sys.argv[3]
    # dataInfoPath = sys.argv[4]
    # videosBboxPath = sys.argv[5]
    # imagesFolder_160 = sys.argv[6]
    # resultFolder = sys.argv[7]
    videosFolder = "/home/zqr/data/test/test_video"
    imagesFolder = "/home/zqr/data/test/test_img"
    opticalFolder = "/home/zqr/codes/data/opticalFiles"
    dataInfoPath = "/home/zqr/codes/data/data_info.txt"
    videosBboxPath = "/home/zqr/data/test/videosBbox.txt"
    imagesFolder_160 = "/home/zqr/codes/data/glofdb_frame_no_resize"
    resultFolder = "/home/zqr/codes/data/resultFolder"
    optFilesFolder_160 = "/home/zqr/codes/data/optFromOri_160"
    optFilesFolder = "/home/zqr/codes/data/optOri"

    # # step1: 切帧
    runStatus = os.system(
        'python3 getFrames.py')
    if runStatus != 0:
        print("切帧流程错误")
        sys.exit(1)
    print("切帧流程完成")

    # # step2：FPN 检测bouldingbox和球杆关键点
    runStatus = os.system('python3 getInfo.py')
    if runStatus != 0:
        print("检测候选框，球杆关键点，流程错误")
        sys.exit(2)
    print("检测候选框，球杆关键点，流程完成")

    # # step3: MSPN 检测人体关键点（候选框是人体候选框，目前主要是根据bbox大小来判断筛选出击球人）
    os.chdir("/home/zqr/codes/MSPN/exps/mspn.2xstg.coco")
    runStatus = os.system('python3 present.py')
    if runStatus != 0:
        print("检测人体关键点流程错误")
        sys.exit(3)
    print("检测人体关键点流程完成")

    # # # step4:整合关键点
    # os.chdir("/home/zqr/codes/MyGolfDB")
    # runStatus = os.system('python3 process_keypoints_data.py')
    # if runStatus != 0:
    #     print("整合关键点流程错误")
    #     sys.exit(2)
    # print("整合关键点流程完成")

    # # step3: 根据bbox 将图片重构为dim×dim(这一步待定吧)
    # os.chdir("/home/zqr/codes/MyGolfDB")
    # runStatus = os.system('python3 resizeFrame.py')
    # if runStatus != 0:
    #     print("resize流程错误")
    #     sys.exit(3)
    # print("resize流程完成")

    # step4: 光流法提取特征(flag是用来打开resize开关的，dim需要在光流proj的config里面配置一下，这样做的话就是在原图提光流特征，然后resize)
    # 这里没有完应该是根据bbox resize比较好吧
    os.chdir("/home/zqr/codes/pytorch-liteflownet")
    runStatus = os.system('python3 run.py {} {} {} {} {}'.format(
        cfg.TEST_IMGS_DIR, cfg.TEST_OPT_DIR, cfg.OPT_RESIZE_FLAG, cfg.TEST_BBOX_INFO_PATH,cfg.TEST_VIDEO_PAHT))
    if runStatus != 0:
        print("光流法提取特征错误")
        sys.exit(4)
    print("光流法提取特征完成")

    # step5: 提取关键帧(具体的路径都已经在disply_dataloader里面配置好了，直接使用就可以了)
    # 8帧
    os.chdir("/home/zqr/codes/MyGolfDB")
    runStatus = os.system(
        'python3 disply.py {}'.format(True))
    if runStatus != 0:
        print("关键帧8帧提取错误")
        sys.exit(5)
    print("关键帧8帧提取完成")

    # 13帧
    runStatus = os.system(
        'python3 disply.py {}'.format(False))
    if runStatus != 0:
        print("关键帧13帧提取错误")
        sys.exit(5)
    print("关键帧13帧提取完成")

    runStatus = os.system('python3 merge_result.py')
    if runStatus != 0:
        print("merge错误")
        sys.exit(5)
    print("merge完成")

    # step6:后处理功能

    # step7:在提取的关键帧图片上加关键点
    runStatus = os.system('python3 visualize.py')
    if runStatus != 0:
        print("13帧关键点错误")
        sys.exit(7)
    print("13帧关键点完成")

    # step8:动作评价


# -*- coding: utf-8 -*-
from django.conf.urls import patterns, include, url
from django.contrib import admin
admin.autodiscover()

# import views
from asset.views import *
from opsa.views import *
from installed.views import *
from deploy.views import *
from config.views import *
from api.views import *
from message.views import *

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'opsa.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    #url(r'^opsa/admin/', include(admin.site.urls)),
    #url(r'^login', login, name='login'),
    #url(r'^accounts/login/$', login, name='login'),
    url(r'^logout', logout, name='logout'),
    url(r'^$', index, name='index'),
    url(r'^asset/host_list/$', host_list, name='host_list'),
    url(r'^asset/add_host/$', host_list_manage, name='add_host'),
    url(r'^asset/delete_host/$', host_list_manage, name='delete_host'),
    url(r'^asset/host_manage/(?P<id>\d+)/$', host_list_manage, name='host_manage'),
    url(r'^asset/server_asset/$', server_asset_list, name='server_asset_list'),
    url(r'^asset/server_get/$', get_server_asset, name='get_server_asset'),
    url(r'^asset/grains_get/$', get_host_grains, name='get_host_grains'),
    url(r'^asset/hosts_autget/$', autoget_host_list, name='autoget_host_list'),
    url(r'^asset/device_list/$', network_device_list, name='network_device_list'),
    url(r'^asset/device_add/$', network_device_discovery, name='add_device'),
    url(r'^asset/idc_list/$', idc_asset_list, name='idc_asset_list'),
    url(r'^asset/add_idc/$', idc_asset_manage, name='add_idc'),
    url(r'^asset/listhostgroups/$', ListHostgroups, name='listhostgroups'),
    url(r'^asset/edithostgroups/(?P<ID>\d+)/$', EditHostgroups, name='edithostgroups'),
    url(r'^asset/deletehostgroups/(?P<ID>\d+)/$', DeleteHostgroups,name='deletehostgroups'),
    url(r'^asset/addhostgroups/$', AddHostgroups, name='addhostgroups'),
    url(r'^install/install_list/$', system_install_list, name='install_list'),
    url(r'^install/install_status/$', system_install_status, name='install_status'),
    url(r'^install/install_manage/(?P<id>\d+)/$', system_install_managed, name='install_manage'),
    url(r'^install/add_install/$', system_install_managed, name='add_install'),
    url(r'^install/install_logs/$', install_logs, name='install_logs'),
    url(r'^install/system_install/$',system_install, name='system_install'),
    url(r'^install/ipmi_c/$',ipmi_c, name='ipim_c'),
    url(r'^install/install_record/$',system_install_record, name='install_record'),
    url(r'^install/install_finish/$',system_install_finish, name='install_finish'),
    url(r'^install/install_init/$', deploy_init_status, name='install_init'),
    url(r'^deploy/job_list/$', salt_job_list, name='job_list'),
    url(r'^deploy/minions_status/$', host_list, name='minions_status'),
    url(r'^deploy/job_result/$', salt_job_results, name='job_result'),
    url(r'^deploy/key_list/$', salt_key_list, name='key_list'),
    url(r'^deploy/key_delete/$', salt_delete_key, name='delete_key'),
    url(r'^deploy/key_accept/$', salt_accept_key, name='accept_key'),
    url(r'^deploy/module_deploy/$', module_deploy, name='module_deploy'),
    url(r'^deploy/remote_execution/$', remote_execution, name='remote_execution'),
    url(r'^config/config_api_add/$', config_info, name='config_api_add'),
    url(r'^config/config_api_list/$', config_api_list, name='config_api_list'),
    url(r'^config/config_os_list/$', config_os_list, name='config_os_list'),
    url(r'^config/get_os_list/$', get_os_list, name='get_os_list'),
    url(r'^config/config_api_add/(?P<id>\d+)/$', config_info, name='config_api_add'),
    url(r'^api/install_status_collect/',install_status_collect, name='install_status_collect'),
    url(r'^message/mesg_list/',mesg_list, name='mesg_list'),
    url(r'^accounts/',include('UserManage.urls' )),
    url(r'^hosts/',include('hostmanage.urls' )),
)


from pydriller import Modification,ModificationType
from pydriller import Commit
from os import path
import src.mining.CodeFeaturesMiners.constants_and_configs as constants_and_configs

from src.mining.CodeFeaturesMiners.AbstractMiner import AbstractMiner

class FileLevelFeatureMiner(AbstractMiner):
    def __init__(self, feature_classes) -> None:
        super().__init__(feature_classes)

    def mine_commit(self, repo_full_name, changeFile: Modification, history, index):
        self.features = {}
        self._mine_labels(changeFile, history[index], repo_full_name)
        if 'basic' in self.feature_classes:
            self._mine_basic(changeFile)
        if 'code_analysis' in self.feature_classes:
            self._mine_code_analysis(changeFile)
        if 'bag_of_words' in self.feature_classes:
            self._mine_bag_of_words(changeFile)
        if 'history' in self.feature_classes and history != None and index != None:
            self._mine_history_based(changeFile, history, index)
        return self.features

    def _mine_labels(self, changeFile: Modification, commit:Commit, repo_full_name):
        path = changeFile.new_path if changeFile.new_path != None else changeFile.old_path
        self._add_feature('label_sha', commit.hash)
        self._add_feature('label_commit_date', commit.committer_date)
        self._add_feature('label_repo_full_name', repo_full_name)
        self._add_feature('label_filepath', path.lower())

    def _mine_basic(self, changeFile: Modification):
        filename = changeFile.filename.lower()
        _, file_extension = path.splitext(filename)
        file_extension = file_extension.lower()

        has_ecosystem_files = {k:False for k in constants_and_configs.code_extensions}
        for ecosystem in constants_and_configs.code_extensions:
            for extension in constants_and_configs.code_extensions[ecosystem]:
                if file_extension == extension:
                    has_ecosystem_files[ecosystem] = True
                    break

        for k,v in has_ecosystem_files.items():
            self._add_feature(f'has_{k}_code', v)

        is_add = changeFile.change_type == ModificationType.ADD
        self._add_feature('is_add', is_add)
        is_rename = changeFile.change_type == ModificationType.RENAME
        self._add_feature('is_rename', is_rename)
        is_delete = changeFile.change_type == ModificationType.DELETE
        self._add_feature('is_delete', is_delete)
        is_modify = changeFile.change_type == ModificationType.MODIFY
        self._add_feature('is_modify', is_modify)

        removed_lines_count = changeFile.removed if changeFile.removed != None else 0
        self._add_feature('removed_lines_count', removed_lines_count)
        added_lines_count = changeFile.added if changeFile.added != None else 0
        self._add_feature('added_lines_count', added_lines_count)
        changed_lines_count = removed_lines_count + added_lines_count
        self._add_feature('changed_lines_count', changed_lines_count)
        removed_lines_ratio = removed_lines_count/changed_lines_count if changed_lines_count>0 else float('nan')
        self._add_feature('removed_lines_ratio', removed_lines_ratio)
        added_lines_ratio = added_lines_count/changed_lines_count if changed_lines_count>0 else float('nan')
        self._add_feature('added_lines_ratio', added_lines_ratio)

        if 'added' in changeFile.diff_parsed and 'deleted' in changeFile.diff_parsed:
            added_lines_set = set([x[0] for x in changeFile.diff_parsed['added']])
            removed_lines_set = set([x[0] for x in changeFile.diff_parsed['deleted']])
            modified_lines_count = len(added_lines_set.intersection(removed_lines_set))
            self._add_feature('modified_lines_count', modified_lines_count)
            modified_lines_ratio = modified_lines_count / changed_lines_count if changed_lines_count>0 else float('nan')
            self._add_feature('modified_lines_ratio', modified_lines_ratio)

        file_size = float("nan")
        if changeFile.source_code != None:
            file_size  = len(changeFile.source_code)
        elif changeFile.source_code_before != None:
            file_size  = len(changeFile.source_code_before)
        self._add_feature('file_size', file_size)

    def _mine_code_analysis(self, changeFile: Modification):
        if not changeFile.language_supported:
            return

        changed_methods_count = len(changeFile.changed_methods)
        self._add_feature('changed_methods_count', changed_methods_count)
        total_methods_count = len(changeFile.methods)
        self._add_feature('total_methods_count', total_methods_count)
        file_complexity = changeFile.complexity if changeFile.complexity != None else 0
        self._add_feature('file_complexity', file_complexity)
        file_nloc = changeFile.nloc if changeFile.nloc != None else 0
        self._add_feature('file_nloc', file_nloc)
        file_token_count = changeFile.token_count if changeFile.token_count != None else 0
        self._add_feature('file_token_count', file_token_count)

        max_method_token_count = 0
        avg_method_token_count = 0
        max_method_complexity = 0
        avg_method_complexity = 0
        max_method_nloc = 0
        avg_method_nloc = 0
        max_method_parameter_count = 0
        avg_method_parameter_count = 0
        keywords_in_method_names = {k:0 for k in constants_and_configs.better_security_keywords}

        for method in changeFile.changed_methods:
            method_name_lower = method.long_name.lower()
            max_method_token_count = max(max_method_token_count, method.token_count)
            avg_method_token_count = avg_method_token_count + method.token_count

            max_method_complexity = max(max_method_complexity, method.complexity)
            avg_method_complexity = avg_method_complexity + method.complexity

            max_method_nloc = max(max_method_nloc, method.nloc)
            avg_method_nloc = avg_method_nloc + method.nloc

            max_method_parameter_count = max(max_method_parameter_count, len(method.parameters))
            avg_method_parameter_count = avg_method_parameter_count + len(method.parameters)

            for k in constants_and_configs.better_security_keywords:
                if k in method_name_lower:
                    keywords_in_method_names[k] +=1

        self._add_feature('max_method_token_count', max_method_token_count)
        self._add_feature('max_method_complexity', max_method_complexity)
        self._add_feature('max_method_nloc', max_method_nloc)
        self._add_feature('max_method_parameter_count', max_method_parameter_count)

        save_file_changed_method_count = changed_methods_count if changed_methods_count > 0 else 1
        self._add_feature('avg_method_token_count', avg_method_token_count/save_file_changed_method_count)
        self._add_feature('avg_method_complexity', avg_method_complexity/save_file_changed_method_count)
        self._add_feature('avg_method_nloc', avg_method_nloc/save_file_changed_method_count)
        self._add_feature('avg_method_parameter_count', avg_method_parameter_count/save_file_changed_method_count)

        for k, v in keywords_in_method_names.items():
            self._add_feature(f'methods_with_{k}_count', v)

    def _mine_bag_of_words(self,changeFile: Modification):
        filename = changeFile.filename.lower()
        path = changeFile.new_path if changeFile.new_path != None else changeFile.old_path
        path = path.lower()

        test_in_filename = 'test' in filename
        self._add_feature('test_in_filename', test_in_filename)
        test_in_path = 'test' in path
        self._add_feature('test_in_path', test_in_path)

        diff_scaped = self._escape_separators(changeFile.diff.lower())

        file_content = ''
        if changeFile.source_code != None:
            file_content  = self._escape_separators(changeFile.source_code.lower())
        elif changeFile.source_code_before != None:
            file_content  = self._escape_separators(changeFile.source_code_before.lower())

        for k in constants_and_configs.better_security_keywords:
            if k in diff_scaped:
                self._add_feature(f'{k}_in_patch', True)
            else:
                self._add_feature(f'{k}_in_patch', False)
            if k in file_content:
                self._add_feature(f'{k}_in_file_content', True)
            else:
                self._add_feature(f'{k}_in_file_content', False)

    def _mine_history_based(self, changeFile, history, index):
        safe_path = changeFile.new_path if changeFile.new_path != None else changeFile.old_path

        changes_to_file_in_prev_50_commits = 0
        is_file_recently_added = False
        for offset in range(-50, -1):
            i = index + offset
            if i < 0:
                continue
            if i >= len(history):
                continue

            for mod in history[i].modifications:
                if safe_path == mod.new_path or safe_path == mod.old_path:
                    changes_to_file_in_prev_50_commits += 1
                    if mod.change_type == ModificationType.ADD:
                        is_file_recently_added = True

        self._add_feature('changes_to_file_in_prev_50_commits', changes_to_file_in_prev_50_commits)
        self._add_feature('is_file_recently_added', is_file_recently_added)

        changes_to_file_in_next_50_commits = 0
        is_file_recently_removed = False
        for offset in range(1, 50):
            i = index + offset
            if i < 0:
                continue
            if i >= len(history):
                continue

            for mod in history[i].modifications:
                if safe_path == mod.new_path or safe_path == mod.old_path:
                    changes_to_file_in_next_50_commits += 1
                    if mod.change_type == ModificationType.DELETE:
                        is_file_recently_removed = True

        self._add_feature('changes_to_file_in_next_50_commits', changes_to_file_in_next_50_commits)
        self._add_feature('is_file_recently_removed', is_file_recently_removed)


from django.db.models import Q
from django.db.models.functions import Lower
from django.urls import reverse_lazy
from poll.models import Poll
from accomplishment.serializers import AccomplishmentSerializer
from uniklinik.mixins import DatatablesMixin

class PollDatatables(DatatablesMixin):
    queryset = Poll.objects.all()
    serializer_class = AccomplishmentSerializer

    def __init__(self):
        super().__init__()
        self.records_total = None
        self.page_number = None
        self.page_size = None

    def get_filtered_queryset(self):
        self.filter_by_search_value()
        return self.queryset

    def filter_by_search_value(self):
        whole_search_value = self.request.GET.get("search[value]")
        search_values = whole_search_value.split(" ")
        for search_value in search_values:
            if search_value != "" and search_value is not None:
                self.queryset = self.queryset.filter(
                    Q(Q(title__icontains=search_value))).distinct()

    def get_ordered_queryset(self):
        order_column_index = self.request.GET.get("order[0][column]")
        asc_or_desc = self.request.GET.get("order[0][dir]")
        if order_column_index == "1":
            if asc_or_desc == "asc":
                self.queryset = self.queryset.order_by(Lower("title"))
            else:
                self.queryset = self.queryset.annotate(lower_title=Lower('title')).order_by("-lower_title")
        return self.queryset

    def get_data(self, page):
        data = {"results": [], "records_total": self.queryset.count()}
        for query in page:
            if query.open is True:
                poll_is_open_or_not = f''
            else:
                poll_is_open_or_not = f''

            data["results"].append([
                f'Bearbeiten',
                # f'Bearbeiten',
                query.title, poll_is_open_or_not])
        return data


import os
import sys
import time
import wandb
import numpy as np
import torch
from torch.utils.data import DataLoader
from dataloader_from_numpy import STFTFeatureSet
from my_models import *
from torch.utils.tensorboard import SummaryWriter
from UniqueDraw import UniqueDraw
from utils import *

def compute_EER(sim_matrix):
    """
    Compute EER, FAR, FRR and the threshold at which EER occurs.

    Args:
    - sim_matrix (torch.Tensor): A similarity matrix of shape
      (num of speakers, num of utterances, num of speakers).

    Returns:
    - EER (float): Equal error rate.
    - threshold (float): The threshold at which EER occurs.
    - FAR (float): False acceptance rate at EER.
    - FRR (float): False rejection rate at EER.
    """
    num_of_speakers, num_of_utters, _ = sim_matrix.shape

    # Initialize values
    diff = float('inf')
    EER = 0.0
    threshold = 0.0
    EER_FAR = 0.0
    EER_FRR = 0.0

    # Iterate over potential thresholds
    for thres in torch.linspace(0.5, 1.0, 501):
        sim_matrix_thresh = sim_matrix > thres

        # Compute FAR and FRR
        FAR = sum([(sim_matrix_thresh[i].sum() - sim_matrix_thresh[i, :, i].sum()).float()
                    for i in range(num_of_speakers)]) / (num_of_speakers - 1.0) / (num_of_utters) / num_of_speakers

        FRR = sum([(num_of_utters - sim_matrix_thresh[i, :, i].sum()).float()
                   for i in range(num_of_speakers)]) / (num_of_utters) / num_of_speakers

        # Update if this is the closest FAR and FRR we've seen so far
        if diff > abs(FAR - FRR):
            diff = abs(FAR - FRR)
            EER = ((FAR + FRR) / 2).item()
            threshold = thres.item()
            EER_FAR = FAR.item()
            EER_FRR = FRR.item()

    return EER, threshold, EER_FAR, EER_FRR

def train_and_test_model(device, model, ge2e_loss, train_set, test_set, optimizer, a_res, p_res, num_epochs=2000, train_seperate=False):
    # writer = SummaryWriter()
    # number of user for train and test in each epoch. Train 4 users, test 2 users.
    batch_size = 5
    train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
    test_loader = DataLoader(test_set, batch_size=2, shuffle=True)

    if train_seperate:
        model_a, model_p = model

    for epoch in range(num_epochs):
        print(f'Epoch {epoch + 1}/{num_epochs}')
        print('-' * 10)

        # train and test model
        # for phase in ['train', 'test']:
        for phase in ['train', 'test']:
            if phase == 'train':
                # set model to training
                ge2e_loss.train()
                if train_seperate == False:
                    model.train()
                else:
                    model_a.train()
                    model_p.train()
                dataloader = train_loader
            else:
                # set model to test
                ge2e_loss.eval()
                if train_seperate == False:
                    model.eval()
                else:
                    model_a.eval()
                    model_p.eval()
                dataloader = test_loader

            # train each batch
            num_of_batches = 0
            loss_avg_batch_all = 0.0
            loss_avg_batch_aa = 0.0
            loss_avg_batch_pp = 0.0

            EERs = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]).astype(float)
            EER_FARs = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]).astype(float)
            EER_FRRs = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]).astype(float)
            EER_threshes = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]).astype(float)

            for batch_id, (utterance, ids) in enumerate(dataloader):
                # get shape of input
                # number of speakers, number of utterances
                # features * 2 (256 * 2), time frame (1000)
                num_of_speakers, num_of_utters, w, l = utterance.shape
                num_of_batches = num_of_batches + 1

                with torch.set_grad_enabled(phase == 'train') and torch.autograd.set_detect_anomaly(True):

                    if phase == 'train':
                        # training
                        # batch_size, time frame, length of feature
                        utterance = torch.reshape(utterance, (num_of_speakers * num_of_utters, w, l)).to(device)
                        piezos = utterance[: , : p_res, : ].to(device)
                        audios = utterance[: , w//2: w//2 + a_res, : ].to(device)
                        piezos = piezos.permute(0, 2, 1).to(device)
                        audios = audios.permute(0, 2, 1).to(device)

                        if train_seperate == False:
                            embeddings_piezo = model(piezos)
                            embeddings_audio = model(audios)
                        else:
                            embeddings_piezo = model_p(piezos)
                            embeddings_audio = model_a(audios)

                        # reshape the output to (number of speaker, number of utter, length of feature)
                        embeddings_piezo = embeddings_piezo.contiguous()
                        embeddings_piezo = embeddings_piezo.reshape([num_of_speakers, num_of_utters, -1])
                        embeddings_audio = embeddings_audio.contiguous()
                        embeddings_audio = embeddings_audio.reshape([num_of_speakers, num_of_utters, -1])
                        # embeddings_piezo = torch.reshape(embeddings_piezo, (num_of_speakers, num_of_utters, -1))
                        # embeddings_audio = torch.reshape(embeddings_audio, (num_of_speakers, num_of_utters, -1))

                        # loss calculation
                        embeddings_combined = torch.cat((embeddings_piezo, embeddings_audio), dim=1)
                        loss_combined, _ = ge2e_loss(embeddings_combined, embeddings_combined)
                        loss_aa, loss_pp = ge2e_loss(embeddings_audio, embeddings_piezo)
                        loss_extractor = loss_combined
                        # loss_extractor = loss_aa + loss_pp
                        # loss_extractor = loss_aa + loss_pp + loss_combined
                        loss_avg_batch_all += loss_extractor.item()
                        loss_avg_batch_aa += loss_aa.item()
                        loss_avg_batch_pp += loss_pp.item()
                        optimizer.zero_grad()
                        # loss_aa.backward()
                        # loss_pp.backward()
                        loss_extractor.backward()

                        if train_seperate == False:
                            torch.nn.utils.clip_grad_norm_(model.parameters(), 3.0)
                        else:
                            torch.nn.utils.clip_grad_norm_(model_a.parameters(), 3.0)
                            torch.nn.utils.clip_grad_norm_(model_p.parameters(), 3.0)
                        torch.nn.utils.clip_grad_norm_(ge2e_loss.parameters(), 1.0)
                        optimizer.step()

                    if phase == 'test':
                        num_of_speakers, num_of_utters, w, l = utterance.shape
                        enrollment_batch, verification_batch = torch.split(utterance, int(num_of_utters/2), dim=1)

                        enrollment_batch = torch.reshape(enrollment_batch, (num_of_speakers * num_of_utters // 2, w, l)).to(device)
                        enrollment_batch = enrollment_batch.permute(0, 2, 1)
                        enrollment_piezo = enrollment_batch[: , : , : p_res].to(device)
                        enrollment_audio = enrollment_batch[: , : , w//2: w//2+a_res].to(device)
                        verification_batch = torch.reshape(verification_batch, (num_of_speakers * num_of_utters // 2, w, l)).to(device)
                        verification_batch = verification_batch.permute(0, 2, 1)
                        verification_piezo = verification_batch[: , : , : p_res].to(device)
                        verification_audio = verification_batch[: , : , w//2: w//2+a_res].to(device)

                        if train_seperate == False:
                            embeddings_piezo = model(enrollment_piezo)
                            embeddings_audio = model(enrollment_audio)
                        else:
                            embeddings_piezo = model_p(enrollment_piezo)
                            embeddings_audio = model_a(enrollment_audio)

                        # embeddings_piezo = torch.reshape(embeddings_piezo, (num_of_speakers, num_of_utters//2, output_dim))
                        # embeddings_audio = torch.reshape(embeddings_audio, (num_of_speakers, num_of_utters//2, output_dim))
                        embeddings_piezo = embeddings_piezo.contiguous()
                        embeddings_piezo = embeddings_piezo.reshape([num_of_speakers, num_of_utters, -1])
                        embeddings_audio = embeddings_audio.contiguous()
                        embeddings_audio = embeddings_audio.reshape([num_of_speakers, num_of_utters, -1])

                        centroids_piezo = get_centroids(embeddings_piezo)
                        centroids_audio = get_centroids(embeddings_audio)

                        if train_seperate == False:
                            embeddings_piezo = model(verification_piezo)
                            embeddings_audio = model(verification_audio)
                        else:
                            embeddings_piezo = model_p(verification_piezo)
                            embeddings_audio = model_a(verification_audio)

                        # embeddings_piezo = torch.reshape(embeddings_piezo, (num_of_speakers, num_of_utters//2, output_dim))
                        # embeddings_audio = torch.reshape(embeddings_audio, (num_of_speakers, num_of_utters//2, output_dim))
                        embeddings_piezo = embeddings_piezo.contiguous()
                        embeddings_piezo = embeddings_piezo.reshape([num_of_speakers, num_of_utters, -1])
                        embeddings_audio = embeddings_audio.contiguous()
                        embeddings_audio = embeddings_audio.reshape([num_of_speakers, num_of_utters, -1])

                        loss_aa, loss_pp = ge2e_loss(embeddings_audio, embeddings_piezo)
                        loss_extractor = loss_aa + loss_pp
                        loss_avg_batch_all += loss_extractor.item()
                        loss_avg_batch_aa += loss_aa.item()
                        loss_avg_batch_pp += loss_pp.item()

                        # EER of Piezo
                        sim_matrix = get_modal_cossim(embeddings_piezo, centroids_piezo)
                        EER, EER_thresh, EER_FAR, EER_FRR = compute_EER(sim_matrix)
                        EERs[0] += EER
                        EER_FARs[0] += EER_FAR
                        EER_FRRs[0] += EER_FRR
                        EER_threshes[0] += EER_thresh

                        # EER of Audio
                        sim_matrix = get_modal_cossim(embeddings_audio, centroids_audio)
                        EER, EER_thresh, EER_FAR, EER_FRR = compute_EER(sim_matrix)
                        EERs[1] += EER
                        EER_FARs[1] += EER_FAR
                        EER_FRRs[1] += EER_FRR
                        EER_threshes[1] += EER_thresh

                        # EER of Audio + Piezo
                        sim_matrix = get_modal_cossim(torch.cat((embeddings_audio, embeddings_piezo), dim=-1),
                                        torch.cat((centroids_audio, centroids_piezo), dim=-1))
                        EER, EER_thresh, EER_FAR, EER_FRR = compute_EER(sim_matrix)
                        EERs[2] += EER
                        EER_FARs[2] += EER_FAR
                        EER_FRRs[2] += EER_FRR
                        EER_threshes[2] += EER_thresh

            epoch_loss_all = loss_avg_batch_all / num_of_batches
            epoch_loss_a = loss_avg_batch_aa / num_of_batches
            epoch_loss_p = loss_avg_batch_pp / num_of_batches

            print(f'{phase} Loss Extractor: {epoch_loss_all:.4f}')

            wandb.log({'epoch': epoch, f'Loss/{phase}_extractor': epoch_loss_all})
            wandb.log({'epoch': epoch, f'Loss/{phase}_loss_a': epoch_loss_a})
            wandb.log({'epoch': epoch, f'Loss/{phase}_loss_p': epoch_loss_p})

            if phase == 'test':
                EERs /= float(num_of_batches)
                EER_FARs /= float(num_of_batches)
                EER_FRRs /= float(num_of_batches)
                EER_threshes /= float(num_of_batches)

                print("\nCentroids: Piezo  Verification Input: Piezo "
                            "\nEER : %0.2f (thres:%0.2f, FAR:%0.2f, FRR:%0.2f)" % (EERs[0], EER_threshes[0], EER_FARs[0], EER_FRRs[0]))
                wandb.log({'epoch': epoch, 'EER/C_P_VI_P': EERs[0], 'FAR/C_P_VI_P': EER_FARs[0], 'FRR/C_P_VI_P': EER_FRRs[0]})
                wandb.log({'epoch': epoch, 'threshold/C_P_VI_P': EER_threshes[0]})

                print("\nCentroids: Audio  Verification Input: Audio "
                            "\nEER : %0.2f (thres:%0.2f, FAR:%0.2f, FRR:%0.2f)" % (EERs[1], EER_threshes[1], EER_FARs[1], EER_FRRs[1]))
                wandb.log({'epoch': epoch, 'EER/C_A_VI_A': EERs[1], 'FAR/C_A_VI_A': EER_FARs[1], 'FRR/C_A_VI_A': EER_FRRs[1]})
                wandb.log({'epoch': epoch, 'threshold/C_A_VI_A': EER_threshes[1]})

                print("\nCentroids: Audio + Piezo  Verification Input: Audio + Piezo "
                            "\nEER : %0.2f (thres:%0.2f, FAR:%0.2f, FRR:%0.2f)" % (EERs[2], EER_threshes[2], EER_FARs[2], EER_FRRs[2]))
                wandb.log({'epoch': epoch, 'EER/C_AP_VI_AP': EERs[2], 'FAR/C_AP_VI_AP': EER_FARs[2], 'FRR/C_AP_VI_AP': EER_FRRs[2]})
                wandb.log({'epoch': epoch, 'threshold/C_AP_VI_AP': EER_threshes[2]})

    return model if not train_seperate else (model_a, model_p)

if __name__ == "__main__":

    device = "cuda:0" if torch.cuda.is_available() else "cpu"

    # get the training and test data set
    ids = list(range(22))
    # valid_ids = [4, 7, 13, 21, 30, 36, 29, 41]
    # valid_ids = [1, 5, 15, 20, 31, 29, 35, 39]
    # valid_ids = [2, 8, 9, 14]
    valid_ids = [3, 6, 9, 14]
    # valid_ids = [3, 6, 8, 1]
    # valid_ids = [11, 12, 14, 16]
    train_ids = [item for item in ids if item not in valid_ids]
    print(train_ids)
    print(valid_ids)

    data_file_dir = './processed_data/stft/res_256_hop_128_t_160/' # folder where stores the data for training and test
    pth_store_dir = './pth_model/'
    os.makedirs(pth_store_dir, exist_ok=True)

    # set the params of each train
    # ----------------------------------------------------------------------------------------------------------------
    # Be sure to go through all the params before each run in case the models are saved in wrong folders!
    # ----------------------------------------------------------------------------------------------------------------

    lr = 0.001
    num_of_epoches = 1500
    train_seperate = False

    comment = 'old_model_out_128_b_2_u_20' # simple descriptions of specifications of this model, for example, 't_f' means we use the model which contains time and frequency nn layers

    ge2e_loss = GE2ELoss(device)

    if train_seperate == False:
        a_res = 256
        p_res = 256
        # extractor = Extractor_w_LSTM(device=device, layer_n=5, input_dim=a_res, output_dim=a_res).to(device)
        # extractor = Extractor_w_F_2D(device=device, f_len=a_res).to(device)
        # extractor = Extractor_w_F_1D(in_channel_n=128, device=device, f_len=a_res).to(device)
        extractor = extractor1DCNN(device=device, channel_number=160, in_feature=256, out_feature=128).to(device)
        optimizer = torch.optim.SGD([
            {'params': extractor.parameters()},
            {'params': ge2e_loss.parameters()}
        ], lr=lr)
        model_n = 1
    else:
        a_res = 512
        p_res = 256
        # extractor_a = Extractor_w_F_2D(device=device, f_len=a_res).to(device)
        # extractor_p = Extractor_w_F_2D(device=device, f_len=p_res).to(device)
        extractor_a = Extractor_w_LSTM(device=device, layer_n=5, input_dim=a_res, output_dim=a_res).to(device)
        extractor_p = Extractor_w_LSTM(device=device, layer_n=5, input_dim=p_res, output_dim=p_res).to(device)
        # extractor_a = Extractor_w_F_1D(in_channel_n=50, device=device, f_len=a_res).to(device)
        # extractor_p = Extractor_w_F_1D(in_channel_n=50, device=device, f_len=p_res).to(device)
        extractor = (extractor_a, extractor_p)
        optimizer = torch.optim.SGD([
            {'params': extractor_a.parameters()},
            {'params': extractor_p.parameters()},
            {'params': ge2e_loss.parameters()}
        ], lr=lr)
        model_n = 2 # number of models: wish to train one model jointly or 2 models seperately

    # create the folder to store the model
    model_struct = 'model_n_' + str(model_n) + '_res_a' + str(a_res) + '_p' + str(p_res) + '_' + comment
    # initialize the wandb configuration
    time_stamp = time.strftime("%Y_%m_%d_%H_%M", time.localtime())
    wandb.init(
        # team name
        entity="piezobuds",
        # set the project name
        project="PiezoBuds",
        # params of the task
        name=model_struct+'_'+time_stamp
    )
    model_store_pth = pth_store_dir + model_struct + '/'
    os.makedirs(model_store_pth, exist_ok=True)
    model_final_path = model_store_pth + time_stamp + '/'
    os.makedirs(model_final_path, exist_ok=True)

    # load the data
    train_set = STFTFeatureSet(False, train_ids, 20, data_file_dir)
    test_set = STFTFeatureSet(False, valid_ids, 20, data_file_dir)

    extractor = train_and_test_model(device=device, model=extractor, ge2e_loss=ge2e_loss,
                                 train_set=train_set, test_set=test_set,
                                 optimizer=optimizer, a_res=a_res, p_res=p_res,
                                 num_epochs=num_of_epoches, train_seperate=train_seperate)

    if train_seperate == False:
        torch.save(extractor.state_dict(), model_final_path+'m.pth')
    else:
        torch.save(extractor[0].state_dict(), model_final_path+'a.pth')
        torch.save(extractor[1].state_dict(), model_final_path+'p.pth')


from django.conf import settings

DEFAULT_SETTINGS = {
    'ALLOW_VERSION_OVERWRITE': False,
    'RELEASE_UPLOAD_TO': 'dists',
    'OS_NAMES': (
        ("aix", "AIX"),
        ("beos", "BeOS"),
        ("debian", "Debian Linux"),
        ("dos", "DOS"),
        ("freebsd", "FreeBSD"),
        ("hpux", "HP/UX"),
        ("mac", "Mac System x."),
        ("macos", "MacOS X"),
        ("mandrake", "Mandrake Linux"),
        ("netbsd", "NetBSD"),
        ("openbsd", "OpenBSD"),
        ("qnx", "QNX"),
        ("redhat", "RedHat Linux"),
        ("solaris", "SUN Solaris"),
        ("suse", "SuSE Linux"),
        ("yellowdog", "Yellow Dog Linux"),
    ),
    'ARCHITECTURES': (
        ("alpha", "Alpha"),
        ("hppa", "HPPA"),
        ("ix86", "Intel"),
        ("powerpc", "PowerPC"),
        ("sparc", "Sparc"),
        ("ultrasparc", "UltraSparc"),
    ),
    'DIST_FILE_TYPES': (
        ('sdist', 'Source'),
        ('bdist_dumb', '"dumb" binary'),
        ('bdist_rpm', 'RPM'),
        ('bdist_wininst', 'MS Windows installer'),
        ('bdist_egg', 'Python Egg'),
        ('bdist_dmg', 'OS X Disk Image'),
    ),
    'PYTHON_VERSIONS': (
        ('any', 'Any i.e. pure python'),
        ('2.1', '2.1'),
        ('2.2', '2.2'),
        ('2.3', '2.3'),
        ('2.4', '2.4'),
        ('2.5', '2.5'),
        ('2.6', '2.6'),
        ('2.7', '2.7'),
        ('3.0', '3.0'),
        ('3.1', '3.1'),
        ('3.2', '3.2'),
    ),
    'METADATA_FIELDS': {
        '1.0': ('platform', 'summary', 'description', 'keywords', 'home_page',
                'author', 'author_email', 'license', ),
        '1.1': ('platform', 'supported_platform', 'summary', 'description',
                'keywords', 'home_page', 'download_url', 'author', 'author_email',
                'license', 'classifier', 'requires', 'provides', 'obsoletes',),
        '1.2': ('platform', 'supported_platform', 'summary', 'description',
                'keywords', 'home_page', 'download_url', 'author', 'author_email',
                'maintainer', 'maintainer_email', 'license', 'classifier',
                'requires_dist', 'provides_dist', 'obsoletes_dist',
                'requires_python', 'requires_external', 'project_url')
    },
    'METADATA_FORMS': {
        '1.0': 'djangopypi.forms.Metadata10Form',
        '1.1': 'djangopypi.forms.Metadata11Form',
        '1.2': 'djangopypi.forms.Metadata12Form',
    },
    'FALLBACK_VIEW': 'djangopypi.views.releases.ReleaseListView',
    'ACTION_VIEWS': {
        "file_upload": 'djangopypi.views.distutils.register_or_upload',  # ``sdist`` command
        "submit": 'djangopypi.views.distutils.register_or_upload',  # ``register`` command
        "list_classifiers": 'djangopypi.views.distutils.list_classifiers',  # ``list_classifiers`` command
    },
    'XMLRPC_COMMANDS': {
        'list_packages': 'djangopypi.views.xmlrpc.list_packages',
        'package_releases': 'djangopypi.views.xmlrpc.package_releases',
        'release_urls': 'djangopypi.views.xmlrpc.release_urls',
        'release_data': 'djangopypi.views.xmlrpc.release_data',
        # 'search': xmlrpc.search, Not done yet
        # 'changelog': xmlrpc.changelog, Not done yet
        # 'ratings': xmlrpc.ratings, Not done yet
    },
    'MIRRORING': False,
    'PROXY_BASE_URL': 'http://pypi.python.org',
    'PROXY_MISSING': False,
}

USER_SETTINGS = DEFAULT_SETTINGS.copy()
USER_SETTINGS.update(getattr(settings, 'DJANGOPYPI_SETTINGS', {}))

# This is disabled on pypi.python.org, can be useful if you make mistakes
if hasattr(settings, 'DJANGOPYPI_ALLOW_VERSION_OVERWRITE'):
    USER_SETTINGS['ALLOW_VERSION_OVERWRITE'] = settings.DJANGOPYPI_ALLOW_VERSION_OVERWRITE

""" The upload_to argument for the file field in releases. This can either be
a string for a path relative to your media folder or a callable. For more
information, see http://docs.djangoproject.com/ """
if hasattr(settings, 'DJANGOPYPI_RELEASE_UPLOAD_TO'):
    USER_SETTINGS['RELEASE_UPLOAD_TO'] = settings.DJANGOPYPI_RELEASE_UPLOAD_TO

if hasattr(settings, 'DJANGOPYPI_OS_NAMES'):
    USER_SETTINGS['OS_NAMES'] = settings.DJANGOPYPI_OS_NAMES

if hasattr(settings, 'DJANGOPYPI_ARCHITECTURES'):
    USER_SETTINGS['ARCHITECTURES'] = settings.DJANGOPYPI_ARCHITECTURES

if hasattr(settings, 'DJANGOPYPI_DIST_FILE_TYPES'):
    USER_SETTINGS['DIST_FILE_TYPES'] = settings.DJANGOPYPI_DIST_FILE_TYPES

if hasattr(settings, 'DJANGOPYPI_PYTHON_VERSIONS'):
    USER_SETTINGS['PYTHON_VERSIONS'] = settings.DJANGOPYPI_PYTHON_VERSIONS

if hasattr(settings, 'DJANGOPYPI_METADATA_FIELDS'):
    USER_SETTINGS['METADATA_FIELDS'] = settings.DJANGOPYPI_METADATA_FIELDS

if hasattr(settings, 'DJANGOPYPI_METADATA_FORMS'):
    USER_SETTINGS['METADATA_FORMS'] = settings.DJANGOPYPI_METADATA_FORMS

if hasattr(settings, 'DJANGOPYPI_FALLBACK_VIEW'):
    USER_SETTINGS['FALLBACK_VIEW'] = settings.DJANGOPYPI_FALLBACK_VIEW

if hasattr(settings, 'DJANGOPYPI_ACTION_VIEWS'):
    USER_SETTINGS['ACTION_VIEWS'] = settings.DJANGOPYPI_ACTION_VIEWS

if hasattr(settings, 'DJANGOPYPI_XMLRPC_COMMANDS'):
    USER_SETTINGS['XMLRPC_COMMANDS'] = settings.DJANGOPYPI_XMLRPC_COMMANDS

""" These settings enable proxying of packages that are not in the local index
to another index, http://pypi.python.org/ by default. This feature is disabled
by default and can be enabled by setting DJANGOPYPI_PROXY_MISSING to True in
your settings file. """
if hasattr(settings, 'DJANGOPYPI_PROXY_BASE_URL'):
    USER_SETTINGS['PROXY_BASE_URL'] = settings.DJANGOPYPI_PROXY_BASE_URL

if not hasattr(settings, 'DJANGOPYPI_PROXY_MISSING'):
    USER_SETTINGS['PROXY_MISSING'] = settings.DJANGOPYPI_PROXY_MISSING = False

# USER_SETTINGS['FALLBACK_VIEW'] = import_item(USER_SETTINGS['FALLBACK_VIEW'])
# USER_SETTINGS['METADATA_FORMS'] = import_item(USER_SETTINGS['METADATA_FORMS'])
# action_views_dict = {}
# for key, val in USER_SETTINGS['ACTION_VIEWS'].items():
#     action_views_dict[key] = import_item(USER_SETTINGS['ACTION_VIEWS'][key])
# USER_SETTINGS['ACTION_VIEWS'] = action_views_dict
# xmlrpc_commands = {}
# for key, val in USER_SETTINGS['XMLRPC_COMMANDS'].items():
#     xmlrpc_commands[key] = import_item(USER_SETTINGS['XMLRPC_COMMANDS'][key])
# USER_SETTINGS['XMLRPC_COMMANDS'] = xmlrpc_commands

globals().update(USER_SETTINGS)


import codecs
import io
import sys
import xml.sax
import xml.sax.saxutils
import xml.sax.xmlreader

import lxml.etree

class GenericXSLTTransformer:
    def __init__(self, stylesheet):
        self.loc = stylesheet
        parsed = lxml.etree.parse(self.loc)
        self.transform = lxml.etree.XSLT(parsed)
        self.docbuf = io.StringIO()
        self.writer = xml.sax.saxutils.XMLGenerator(self.docbuf, "UTF-8")
    def _get_string(self):
        self.writer.endDocument()
        # lxml.etree refuses to read XML documents with an encoding declaration
        # from strings, so we have to convert it to bytes in order to get it to
        # parse correctly.
        sbuf = self.docbuf.getvalue()
        c = codecs.getencoder("UTF-8")
        bbuf = c(sbuf)[0]
        doc = lxml.etree.parse(io.BytesIO(bbuf))
        return str(self.transform(doc))
        print(str(result), file=sys.stderr)
        parsebuf = io.StringIO(str(result))
        self.reader.parse(parsebuf)
    def __getattr__(self, name):
        return getattr(self.writer, name)

class TextXSLTTransformer(GenericXSLTTransformer):
    def __init__(self, output, stylesheet):
        GenericXSLTTransformer.__init__(self, stylesheet)
        self.output = output
    def endDocument(self):
        self.s = self._get_string()
        if self.output is not None:
            self.output.write(self.s)
    def get_string(self):
        return self.s

class XSLTTransformer(GenericXSLTTransformer):
    def __init__(self, base, stylesheet):
        GenericXSLTTransformer.__init__(self, stylesheet)
        self.base = base
        self.reader = xml.sax.make_parser()
        self.reader.setContentHandler(self.base)
    def endDocument(self):
        s = self._get_string()
        parsebuf = io.StringIO(s)
        self.reader.parse(parsebuf)


import sys, os, argparse
import scad
from typing import List

def do_inline(src, dest, scad_paths):
    try:
        scad.inliner(src, dest, scad_paths)
    except FileNotFoundError as e:
        print(e)
        print('Try to use the -op argument or set up the OPENSCADPATH environment variable')

def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("src", help="the source")
    parser.add_argument("dest", help="the destination")
    parser.add_argument("-op", "--openscadpath", help="specify where to find user scad files")

    if(len(sys.argv) == 1):
        parser.print_help()
    else:
        args = parser.parse_args()

        scad_paths: List[str] = (
            args.openscadpath
                if args.openscadpath else os.environ['OPENSCADPATH']
        ).split(';')

        do_inline(args.src, args.dest, scad_paths)

main()


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

print(setuptools.find_packages())

setuptools.setup(
    name="TerraVide",                     # This is the name of the package
    version="0.1.7",                        #release version
    author="Sarang Pramode",                     # Full name of the author
    description="An open source python package to process and simulate large urban environments mapped with LiDAR data",
    long_description = long_description,
    long_description_content_type = "text/markdown",
    license='MIT',
    packages=setuptools.find_packages(),    # List of all python modules to be installed
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],                                      # Information to filter the project on PyPi website
    python_requires='>=3.6',                # Minimum version requirement of the package
    #py_modules=["info","dataset"],             # Name of the python package
    install_requires=[              # Install other dependencies if any
        "numpy",
        "pandas",
        "laspy",
    ]
)


import os
import json
import zipfile
from unittest import mock

@mock.patch.dict(os.environ, {"ROOT_FOLDER": os.path.dirname(os.path.realpath(__file__))})
def get_job_details():
    root = os.getenv('ROOT_FOLDER', '')
    """Reads in metadata information about assets used by the algo"""
    job = dict()
    job['dids'] = json.loads(os.getenv('DIDS'))
    job['metadata'] = dict()
    job['files'] = dict()
    job['algo'] = dict()
    job['secret'] = os.getenv('secret', None)
    if job['dids'] is not None:
        for did in job['dids']:
            # get the ddo from disk
            filename = root + '/data/ddos/' + did
            print(f'Reading json from {filename}')
            with open(filename) as json_file:
                ddo = json.load(json_file)
                # search for metadata service
                for service in ddo['service']:
                    if service['type'] == 'metadata':
                        job['files'][did] = list()
                        index = 0
                        for file in service['attributes']['main']['files']:
                            job['files'][did].append(
                                root + '/data/inputs/' + did + '/' + str(index))
                            index = index + 1
    return job

def descriptive_statistics(job_details):
    root = os.getenv('ROOT_FOLDER', '')
    print('Starting compute job with the following input information:')
    print(json.dumps(job_details, sort_keys=True, indent=4))
    """ Computes descriptive statistics for the first file in first did """
    first_did = job_details['dids'][0]
    for f in os.listdir('data/inputs/'+first_did+'/'):
        if os.path.splitext(f)[1] == '.zip':
            filename = f
    print(filename)
    with zipfile.ZipFile('data/inputs/'+ first_did + '/' +filename, 'r') as zip_ref:
        zip_ref.extractall('data/inputs/'+first_did)
    for f in os.listdir('data/inputs/'+first_did+'/'):
        if os.path.splitext(f)[1] !='.zip':
            folder = f
    images = list()
    for f in os.listdir('data/inputs/'+first_did+'/'+folder+'/'):
        images.append(f)
    print(images)

if __name__ == '__main__':
    descriptive_statistics(get_job_details())


from behave import *
import requests
import configparser

@given('the book details are given')
def stepimpl(context):

    context.config = configparser.ConfigParser()
    context.config.read('Utilities/properties.ini')
    # print(type(config))

@when('Add Book API is executed')
def step_impl(context):
    postresp = requests.post(context.config['API']['baseUrl']+'/Library/Addbook.php', json=datatransfer(6306096),
                             headers={'Content-Type': 'application/json'})
    assert postresp.status_code == 200
    context.postresp_json = postresp.json()

@then('book is successfully added')
def step_impl(context):
    assert context.postresp_json['Msg'] == "successfully added"


import os
import sys

from PIL import Image
from flask import Flask, render_template, request, url_for, redirect, flash, session, abort
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView
from sqlalchemy import func, or_
from sqlalchemy.exc import SQLAlchemyError
from werkzeug.utils import secure_filename

from config import MY_PROJECTS_DATABASE_NAME, CONTACTS_DATABASE_NAME, REVIEW_DATABASE_NAME, \
    MY_ACHIEVEMENTS_DATABASE_NAME, SITE_DATABASE_NAME
from db.models import db_init, db, Review, Project, Contact, Achievement
from utils import corvert_image

# <--------------------> #
#  InitializeBlockStart  #
# <--------------------> #

app = Flask(__name__, template_folder='templates', static_url_path='/static/')

# SQLAlchimy config
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + SITE_DATABASE_NAME
app.config['SQLALCHEMY_BINDS'] = {
    'contacts': 'sqlite:///' + CONTACTS_DATABASE_NAME,
    'reviews': 'sqlite:///' + REVIEW_DATABASE_NAME,
    'projects': 'sqlite:///' + MY_PROJECTS_DATABASE_NAME,
    'achievement': 'sqlite:///' + MY_ACHIEVEMENTS_DATABASE_NAME
}

app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
app.config['SECRET_KEY'] = '34567890oihgfcvbjktrtyuiop'
app.config['SESSION_COOKIE_NAME'] = 'EVM'
app.config['TIMEZONE'] = 'Europe/Paris'

# set optional bootswatch theme
app.config['FLASK_ADMIN_SWATCH'] = 'cerulean'

db.app = app
db.init_app(app)

# <------------------> #
#  InitializeBlockEnd  #
# <------------------> #

# <---------------> #
#  AdminBlockStart  #
# <---------------> #

class Controller(ModelView):
    def is_accessible(self):
        if "logged_in" in session:
            return True
        abort(403)

# Add administrative views here
admin = Admin(app, name='microblog', template_mode='bootstrap3')
admin.add_view(Controller(Review, db.session))
admin.add_view(Controller(Contact, db.session))
admin.add_view(Controller(Project, db.session))
admin.add_view(Controller(Achievement, db.session))

@app.route("/admin", methods=["POST"])
def photo():
    if request.args.get("f") == "add":
        file = request.files["photo"]
        path = os.getcwd() + url_for('static', filename=f'saved_images/{str(secure_filename(file.filename))}')
        img = Image.open(file.stream)
        img.save(path)
    if request.args.get("f") == "del":
        try:
            path = request.form.getlist('del')
            path_to_image = os.getcwd() + url_for('static', filename=f'saved_images/{path[0]}')
            print(path_to_image)
            os.remove(path_to_image)
        except:
            flash('something went wrong!', category='error')
    return redirect('/admin')

@app.route("/login", methods=['GET', 'POST'])
def login():
    if request.method == 'POST':
        if request.form.get("username") == "admin" and request.form.get("password") == "superqwerty":
            session['logged_in'] = True
            return redirect('/admin')
        else:
            return render_template('login.html', failed=True)
    return render_template('login.html')

@app.route("/logout")
def logout():
    session.clear()
    return redirect(url_for('index'))

# <--------------> #
#  AdminBlockEnd   #
# <--------------> #

# <---------------> #
#  RoutsBlockStart  #
# <---------------> #

@app.before_request
def make_permanent():
    session.permanent = True

@app.route("/")
@app.route("/index")
@app.route("/index.html")
def index():
    reviews = Review.query.filter(Review.visible).order_by(-Review.best).limit(10).all()
    return render_template('index.html',
                           title='Home',
                           reviews=reviews)

@app.route("/about")
@app.route("/about.html")
def about():
    return render_template('about_page.html', title="About")

@app.route("/projects")
@app.route("/projects.html")
def projects():
    projects = Project.query.all()
    reviews = Review.query.filter(Review.visible).order_by(Review.date).limit(5).all()
    return render_template('projects.html',
                           title="Projects",
                           projects=projects,
                           count=len(projects),
                           reviews=reviews)

@app.route("/projects/<int:id>")
@app.route("/projects_details.html/<int:id>")
def projects_details(id):
    project = Project.query.filter(Project.id == id).first()
    return render_template('projects_details.html', title="Details", project=project)

@app.route("/achievement", methods=['GET', 'POST'])
@app.route("/achievement.html", methods=['GET', 'POST'])
def achievement():
    if request.method == 'POST':
        try:
            achievements = Achievement.query.filter(or_(
                func.lower(Achievement.name).contains(request.form["res"]),
                func.lower(Achievement.categories).contains(request.form["res"]))).order_by(-Achievement.date_year,
                                                                                -Achievement.date_month,
                                                                                -Achievement.date_day).all()
            return render_template('achievement.html', title="Achievement", posts=achievements)
        except SQLAlchemyError as e:
            db.session.rollback()
            error = str(e.__dict__['orig'])
            print(error)
            print("Can't add new review to the Database")
            flash(error, category='error')
            return redirect(url_for('/achievement/all'))

    else:
        achievements = Achievement.query.order_by(-Achievement.date_year, -Achievement.date_month,
                                                  -Achievement.date_day).all()

    return render_template('achievement.html',
                           title="Achievement",
                           posts=achievements)

@app.route("/all_reviews", methods=['GET', 'POST'])
@app.route("/all_reviews.html", methods=['GET', 'POST'])
def all_reviews():
    best = None
    reviews = []
    projects = Project.query.limit(5).all()

    if request.method == 'POST':
        try:
            reviews = Review.query.filter(or_(
                func.lower(Review.name).contains(request.form["res"]),
                func.lower(Review.review).contains(request.form["res"]))).order_by(-Review.best, Review.date).all()
        except SQLAlchemyError as e:
            error = str(e.__dict__['orig'])
            print(error)
            print("Something wrong with the Database")
            flash(error, category='error')
            return redirect(url_for('index'))

    else:
        reviews = Review.query.filter(Review.visible).order_by(-Review.best, Review.date).all()

    count = len(reviews)
    if len(reviews) > 1:
        best, reviews = reviews[0], reviews[1:]
        count -= 1

    return render_template('all_reviews.html',
                           title="All reviews",
                           best=best,
                           reviews=reviews,
                           count=count,
                           projects=projects)

@app.route("/create_review", methods=['GET', 'POST'])
@app.route("/create_review.html", methods=['GET', 'POST'])
def create_review():
    if request.method == 'POST':
        try:
            file = request.files["photo"]
            name = f'src/default_image.png'

            if file.filename != '':
                filename = '.'.join(str(secure_filename(file.filename)).split('.')[:-1] + ['png'])
                name = f'saved_images/{filename}'
                path = url_for('static', filename=f'saved_images/{filename}')
                converted = corvert_image(file)
                converted.save(os.getcwd() + path)

            new_review = Review(name=request.form["name"] if request.form["name"] != '' else 'Anonim',
                                position=request.form["position"] if request.form["position"] != '' else 'Hidden',
                                review=request.form["review"] if request.form["review"] != '' else 'Good job!',
                                photo=name,
                                visible=True
                                )

            db.session.add(new_review)
            db.session.commit()
        except SQLAlchemyError as e:
            db.session.rollback()
            error = str(e.__dict__['orig'])
            print(error)
            print("Can't add new review to the Database")
            flash(error, category='error')
            return redirect(url_for('index'))

        flash("Thanks for your review. It will be added soon. Then you can check it in section: Review->All", category='success')
        return redirect(url_for('index'))
    else:
        return render_template('create_review.html', title="Create review")

@app.route("/contact", methods=['GET', 'POST'])
@app.route("/contact.html", methods=['GET', 'POST'])
def contact():
    if request.method == 'POST':
        try:
            new_contact = Contact(name=request.form["name"],
                                  email=request.form["email"],
                                  message=request.form["message"],
                                  subject=request.form["subject"]
                                  )
            db.session.add(new_contact)
            db.session.commit()

        except SQLAlchemyError as e:
            db.session.rollback()
            error = str(e.__dict__['orig'])
            print(error)
            print("Can't add contact to the Database")
            flash(error, category='error')
            return redirect(url_for('index'))

        flash("Thanks for contact with me. I will answer soon.", category='success')
        return redirect(url_for('index'))

    return render_template('contact.html', title="Contact")

# <-------------> #
#  RoutsBlockEnd  #
# <-------------> #

# Press the green button in the gutter to run the script.
if __name__ == '__main__':

    if len(sys.argv) > 1:
        if sys.argv[1] == "init":
            with app.app_context():
                db_init()

    app.run(host='0.0.0.0', port=5000, debug=True)


import sqlite3

cx = sqlite3.connect('banco.db')  # sera criado um arquivo banco.db

cursor = cx.cursor() # cursor

              # Criando tabela
                # Aqui criamos os comandos sql
cursor.execute('''CREATE TABLE cliente
               id INT PRIMARY KEY AUTOINCREMENT NOT NULL,
               nome TEXT NOT NULL,
               idade INT NOT NULL,
               ''')

cursor.execute(''' INSERT INTO banco (id)''')
cx.commit()


import csv

from django.core.management.base import BaseCommand

from ...models import *

# Usage example: python manage.py import_from_csv IA "<filename.csv>"

class Command(BaseCommand):
    help = "Load polling location data from a spreadsheet"

    def add_arguments(self, parser):
        parser.add_argument("state_code")
        parser.add_argument("file")

    def handle(self, *args, **options):
        state_code = options["state_code"]
        file_name = options["file"]
        lid = 1
        reader = csv.DictReader(open(file_name))
        for row in reader:
            print(row)
            precinct, created = Precinct.objects.get_or_create(
                van_precinct_id=row.get("Precinct ID"),
                state_code=state_code,
                county=row.get("County"),
                precinct_code=row.get("Precinct Code"),
            )
            precinct.save()

            polling_location = PollingLocation(
                location_id=lid,
                location_name=row.get("Location Name"),
                address=row.get("Street Address"),
                city=row.get("City"),
                state_code=state_code,
                zip=row.get("Zip Code"),
            )
            polling_location.save()

            ptpl = PrecinctToPollingLocation(
                precinct=precinct,
                state_code=state_code,
                distance=1.0,
                location=polling_location,
            )
            ptpl.save()
            lid += 1


# -*- coding: utf-8 -*-
"""
Energy Calibration Analysis v3.1

Lukas Kostal, 14.11.2023, ICL
"""

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import scipy.optimize as so
import scipy.signal as ss
import scipy.interpolate as si
import scipy.odr as sr
import itertools as it
import datetime

# gaussian function for fitting
def gauss(x, A, mu, sigma):
    y = A * np.exp(-((x - mu) / sigma)**2 / 2)
    return y

# gaussian function for fitting with ODR
def odr_gauss(B, x):
    A, mu, sigma = B
    y = A * np.exp(-((x - mu) / sigma)**2 / 2)
    return y

# polynomial function for fitting
def poly(x, *coef):
    if type(x) == np.float64:
        y = 0
    else:
        y = np.zeros(len(x))
    for i in range(0, len(coef)):
        y += coef[i] * x**i

    return y

# polynomial function for fitting with odr
def odr_poly(coef, x):
    y = np.zeros(len(x))
    for i in range(0, len(coef)):
        y += coef[i] * x**i
    return y

# function to load data
def load_mes(mes_num, src):
    # load the data as a dataset
    df = pd.read_csv(f'Data/Mes{mes_num}_{src}.csv')

    # slice into arrays
    t = df.iloc[:, 0].to_numpy()
    U = df.iloc[:, 4].to_numpy()
    N = df.iloc[:, 1].to_numpy()
    n = df.iloc[:, 2].to_numpy()
    g = df.iloc[0, -1]

    # remove nans from shorter columns
    t = t[~np.isnan(t)]
    U = U[~np.isnan(U)]
    n = n[~np.isnan(n)]
    N = N[~np.isnan(N)]

    return t, U, n, N, g

# function to combine bins
def cmb_bin(n_arr, N_arr, cmb):
    cmb = int(cmb)
    n_arr = n_arr[(len(n_arr) % cmb):]
    N_arr = N_arr[(len(N_arr) % cmb):]

    N_arr = np.sum(N_arr.reshape(-1, cmb), axis=1)
    n_arr = np.mean(n_arr.reshape(-1, cmb), axis=1)

    return n_arr, N_arr

# function to get reduced chi squared
def calc_chi(o, sig, e, dof):
    with np.errstate(all='ignore'):
        chi_arr = (o - e)**2 / (sig)**2

    chi_arr = chi_arr[~np.isinf(chi_arr)]
    chi = np.sum(chi_arr) / (len(chi_arr) - dof)

    return chi

# dataset number to use for calibration
mes_num = 3

# minimum no of events for combining bins
N_min = 1000

# order of calibration polynomial
cal_ord = 3

# factor of sigmas at which reduced chi sqaured is calculated
f_sig = np.sqrt(2 * np.log(2))

# load reference peaks as dataframe
df = pd.read_csv('Load/reference.csv', comment='#')

# slice to array of sources and energies
src = df.iloc[:, 0].to_numpy()
E = df.iloc[:, 1].to_numpy()
Eerr = df.iloc[:, 2].to_numpy()

# sort the sources with the energies
i_srt = np.argsort(src)
src = src[i_srt]
E = E[i_srt]
Eerr = Eerr[i_srt]

# get array of unique reference sources so remove duplicates
src_ref, i_rpt, n_rpt = np.unique(src, return_index=True, return_counts=True)

# get the number of reference sources and reference peaks
n_ref = len(src_ref)
nn_ref = np.sum(n_rpt)

# array to append arrays of energies for reference peaks
E_ref = np.zeros(n_ref, dtype=object)
Eerr_ref = np.zeros(n_ref, dtype=object)

# loop over all reference sources write the peak energies to array
for i in range(0, n_ref):
    E_ref[i] = E[i_rpt[i]:i_rpt[i] + n_rpt[i]]
    Eerr_ref[i] = Eerr[i_rpt[i]:i_rpt[i] + n_rpt[i]]

# load the background measurement
t_bg, U_bg, n_bg, N_bg, g_bg = load_mes(mes_num, 'Bg')

# calculate scaled channel no by accounting for gain
h_bg = n_bg / -g_bg

# calcualte total background measurement time and rate with error
T_bg = np.amax(t_bg)
R_bg = N_bg / T_bg
Rerr_bg = np.sqrt(N_bg) / T_bg

# array of reference peak energies and error in keV
E_pks = np.zeros(nn_ref)
Eerr_pks = np.zeros(nn_ref)

# array of scaled channel no from peak fit and error
h_pks = np.zeros(nn_ref)
herr_pks = np.zeros(nn_ref)

# array of reduced chi squared values for fitted peaks
chi_pks = np.zeros(nn_ref)

# loop over all reference sources with s as counting parameter
s=0
for i in range(0, n_ref):

    # load the data
    t, U, n, N, g = load_mes(mes_num, src_ref[i])

    # calculate scaled channel no by accounting for gain
    h = n / -g

    # calculate total measurement time and scale background and error
    T = np.amax(t)
    N_bg = R_bg * T
    Nerr_bg = Rerr_bg * T

    # prepare for combining bins
    cmb = 1
    N_cmb = N
    h_cmb = h

    # whole max no of counts in bin is less than specified combine bins
    while np.amax(N_cmb) < N_min:
        h_cmb, N_cmb = cmb_bin(h, N, cmb)
        cmb += 1
    h_cmb, N_cmb = cmb_bin(h, N, cmb)
    h_bg_cmb, N_bg_cmb = cmb_bin(h_bg, N_bg, cmb)

    # interpolate background with cubic spline for resampling
    bg_cs = si.CubicSpline(h_bg_cmb, N_bg_cmb)

    # resample background from cubic spline
    N_bg_rsmp = bg_cs(h_cmb)

    # subtract the background rate and propagate error
    h = h_cmb
    N = N_cmb - N_bg_rsmp
    Nerr = np.sqrt(N_cmb + np.abs(N_bg_rsmp))

    # minimum scaled channel below which peaks are not detected
    h_min = 0.08 * np.amax(h)
    i_min = np.argmin(np.abs(h - h_min))

    # find peaks and get their indices
    pks, _ = ss.find_peaks(N[i_min:], width=4)
    pks += i_min

    # no of reference peaks for the source
    n_pks = len(E_ref[i])

    # if no of found peaks is less than no of reference peaks raise exception
    if len(pks) < n_pks:
        raise Exception('Error: insufficient number of peaks found')

    # sort the peaks by height and take the required no of peaks starting from heighest
    pks = pks[np.argsort(N[pks])[-n_pks:]]

    # manual correction for 2nd Na22 peak
    if src_ref[i] == 'Na22':
        pks[0] = np.argmin(np.abs(h - 620))

    # array of optimised parameters, error on parameters, slice indices for each peak
    popt_arr = np.zeros(n_pks, dtype=object)
    perr_arr = np.zeros(n_pks, dtype=object)
    idx_arr = np.zeros(n_pks, dtype=object)

    # loop over each peak to fit
    for j in range(0, n_pks):

        # specify bounds on fitting parameters ±0.2 within the detected peak
        bnds_min = (N[pks[j]] * 0.8, h[pks[j]] * 0.8, 0)
        bnds_max = (N[pks[j]] * 1.2, h[pks[j]] * 1.2, 100)

        # initial parameteers from detecting peaks
        ig = np.array([N[pks[j]], h[pks[j]], 10])

        # perform preliiminary gaussian fit over entire spectrum
        popt, pcov = so.curve_fit(gauss, h, N, p0=ig, bounds=(bnds_min, bnds_max))

        # find indices at specified factor of sigma below and above peak
        idx_min = np.argmin(np.abs(h - (popt[1] - f_sig * popt[2])))
        idx_max = np.argmin(np.abs(h - (popt[1] + f_sig * popt[2]))) +1

        # slice spectrum to given factor of sigma
        h_fit = h[idx_min:idx_max]
        N_fit = N[idx_min:idx_max]
        Nerr_fit = Nerr[idx_min:idx_max]

        # perform final gaussian fit over part of spectrum
        popt, pcov = so.curve_fit(gauss, h_fit, N_fit, p0=popt, sigma=Nerr_fit, absolute_sigma=True)

        # estimate error on fitting parameters from covariance matrix
        perr = np.sqrt(np.diag(pcov))

        # write potinised parameters, error on parameters and slicing indices to arrays
        popt_arr[j] = popt
        perr_arr[j] = perr
        idx_arr[j] = np.array([idx_min, idx_max])

        # calculate expected no of counts from gaussian fit
        N_exp = gauss(h_fit, *popt)

        # calcualte reduced chi sqaured value
        chi = calc_chi(N_fit, Nerr_fit, N_exp, 3)

        # write reference peak energy, scaled channel no from fit, reduced chi sqaured
        E_pks[s] = E_ref[i][j]
        h_pks[s] = popt[1]
        Eerr_pks[s] = Eerr_ref[i][j]
        herr_pks[s] = perr[1]
        chi_pks[s] = chi

        # increment counting parameter
        s +=1

    # title for plotting
    tit = f'{src_ref[i]}, $g=${g:.2f}, $cmb=${cmb}, $\chi^2_\\nu=${chi_pks[s-1]:.1f}'

    # loop over each peak to append reduced chi squared value to title
    for j in range(1, n_pks):
        tit += f', {chi_pks[s-j-1]:.1f}'

    # plot each curve fit without saving
    plt.title(tit, pad=40)
    plt.xlabel(r'scaled MCA channel $h = \frac{n}{-g}$')
    plt.ylabel(r'counts $N$ (unitless)')
    plt.rc('grid', linestyle=':', color='black', alpha=0.8)
    plt.grid()

    plt.errorbar(h, N, yerr=Nerr, c='blue', label='data')
    plt.plot(h[pks], N[pks], '.', ms=8, c='red', zorder=8, label='peak')
    plt.axvline(x=h_min, ls='--', c='red', zorder=9, label='cutoff')

    # loop over each peak to plot gaussian fit
    for j in range(0, n_pks):
        h_plt = h[idx_arr[j][0]:idx_arr[j][1]]
        if j == 0:
            plt.plot(h_plt, gauss(h_plt, *popt_arr[j]), c='orangered', zorder=10, label='fit')
        else:
            plt.plot(h_plt, gauss(h_plt, *popt_arr[j]), c='orangered', zorder=10)

    plt.legend(loc=(0.1, 1.05), ncol=4)

    plt.show()

# array of 0 initial guess coefficients to specify polynomial order
ig = np.zeros(cal_ord+1)

# perform perliminary fit to get initial parameters for ODR
popt, pcov = so.curve_fit(poly, h_pks, E_pks, p0=ig)

# perform orthogonal distance regression with error in both variables
pmod = sr.Model(odr_poly)
pdat = sr.RealData(h_pks, E_pks, sx=herr_pks, sy=Eerr_pks)
podr = sr.ODR(pdat, pmod, beta0=popt)
pout = podr.run()

# get optimised parameters and error from orthogonal distance regression
popt = np.array(pout.beta)
perr = np.array(pout.sd_beta)

# array to hold expected scaled channel no from fit
h_exp = np.zeros(len(h_pks))

# loop over each reference peak
for i in range(0, len(h_pks)):

    # polynomial function with fit parameters offset by reference peak energy
    poly_inv = lambda x : poly(x, *popt) - E_pks[i]

    # use Newton-Raphson method to solve for expected scaled channel no
    h_exp[i] = so.newton(poly_inv, x0=h_pks[i])

# no of combinations of n object of 2 distinct types (2^n)
n_combs = 2**len(popt)

# create array of all possible combinations of +, -
combs = list(it.product([1, -1], repeat=len(popt)))
combs = np.array(combs)

# 2D array to hold expected scalled channel no for all possible combinations of fit parameters
h_exp_combs = np.zeros((len(h_pks), n_combs))

# loop over all possible combinations of errors on fit parameters
for i in range(0, n_combs):
    popt_combs = popt + combs[i] * perr

    # loop over each reference peak
    for j in range(0, len(h_pks)):

        # polynomial function with fit parameters offset by reference peak energy
        poly_inv = lambda x : poly(x, *popt_combs) - E_pks[j]

        # use Newton-Raphson method to solve for expected scaled channel no
        root, info = so.newton(poly_inv, x0=h_pks[j], disp=False, full_output=True)

        if info.converged and root > 0:
            h_exp_combs[j][i] = root
        else:
            h_exp_combs[j][i] = np.nan

# take halve of maximum difference in combinations to be error in expected value
herr_exp = (np.nanmax(h_exp_combs, axis=1) - np.nanmin(h_exp_combs, axis=1)) / 2

# combine error in measured and expected scaled channel no in quadrature for chi calcualtion
herr_chi = np.sqrt(herr_pks**2 + herr_exp**2)

# calculate reduced chi sqaured for polynomial calibration
chi = calc_chi(h_pks, herr_chi, h_exp, len(popt))

# array of scaled channel no for plotting
h_plt = np.linspace(0, np.amax(h_pks), 100)

# 2D array to hold energies for all possible combinations of fit parameters
E_fit_combs = np.zeros((len(h_plt), n_combs))
E_res_combs = np.zeros((len(h_pks), n_combs))

# loop over all possible combinations of errors on fit parameters
# for each calcualte expected energy
for i in range(0, n_combs):
    popt_combs = popt + combs[i] * perr
    E_fit_combs[:, i] = poly(h_plt, *popt_combs)
    E_res_combs[:, i] = poly(h_pks, *popt_combs)

# take highest and lowest energies for plotting uncertainty region
E_fit_min = np.amin(E_fit_combs, axis=1)
E_fit_max = np.amax(E_fit_combs, axis=1)

# get residuals from polynomial calibration
E_res = E_pks - poly(h_pks, *popt)

# calculate error in residuals from error in fitting parameters
Eerr_res = np.sqrt(Eerr_pks**2 + (np.ptp(E_res_combs, axis=1) / 2)**2)

# get timestamp for the calibration
ts = datetime.datetime.now()
ts = ts.replace(microsecond=0)

# write calibration coefficients to .csv file
with open('Load/calibration.csv', 'w') as cal:
    print('order, coefficient, uncertainty', file=cal)

    for i in range(0, cal_ord+1):
        print(f'{i:.0f}, {popt[i]}, {perr[i]}', file=cal)

    print(f'# dataset: Mes{mes_num}', file=cal)
    print(f'# rcs value: {chi:2f}', file=cal)
    print(f'# timestamp: {ts}', file=cal)

# parameters for plotting calibration curve
plt.figure(1)
plt.title(f'Calibration Curve dataset: Mes{mes_num}, order: {cal_ord}, $\chi^2_\\nu=$ {chi:.2f}', pad=40)
plt.xlabel(r'scaled ADC channel $h = \frac{n}{-g}$')
plt.ylabel(r'energy $E_\gamma$ (keV)')
plt.rc('grid', linestyle=':', color='black', alpha=0.8)
plt.grid()

# plot the datapoints and uncertainty
plt.errorbar(h_pks, E_pks, xerr=herr_pks, yerr=Eerr_pks, fmt='.', c='blue', capsize=4, label='data')
plt.plot(h_plt, poly(h_plt, *popt), c='orangered', label='polynomial')
plt.fill_between(h_plt, E_fit_min, E_fit_max, color='royalblue', alpha=0.3, label='uncertainty')

plt.legend(loc=(0.1, 1.05), ncol=3)

# save the plot
plt.savefig(f'Output/Calibration_mes{mes_num}.png', dpi=300, bbox_inches='tight')
plt.show()

# parameters for plotting residuals
plt.figure(2)
plt.title(f'Calibration Residuals dataset: Mes{mes_num}, order: {cal_ord}, $\chi^2_\\nu=$ {chi:.2f}')
plt.xlabel(r'scaled ADC channel $h = \frac{n}{-g}$')
plt.ylabel(r'energy $E_\gamma$ (keV)')
plt.rc('grid', linestyle=':', color='black', alpha=0.8)
plt.grid()

# plot the datapoints and uncertainty
#plt.plot(h_pks, E_res, 'x', c='blue')
plt.errorbar(h_pks, E_res, xerr=herr_pks, yerr=Eerr_res, fmt='.', c='blue', capsize=4)

# save the plot
plt.savefig(f'Output/Residuals_mes{mes_num}.png', dpi=300, bbox_inches='tight')
plt.show()