from rest_framework.viewsets import ModelViewSet, GenericViewSet
from rest_framework import permissions, generics, status
from rest_framework.decorators import action
from rest_framework.response import Response
from core.models import Department, CheckIn, Employee, Message
from core.api.serializer import DepartmentSerializer, EmployeeSerializer, MessageSerializer, CheckInSerializer
from datetime import datetime, timedelta, date
from django.db.models import Q
from core.api.generate_code import code
from users.models import User

class DepartmentViewSet(ModelViewSet):
    queryset = Department.objects.all()
    serializer_class = DepartmentSerializer
    permission_classes = [permissions.IsAuthenticated]

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        serializer.validated_data['user'] = request.user
        self.perform_create(serializer)
        headers = self.get_success_headers(serializer.data)
        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)

    def list(self, request, *args, **kwargs):
        name = request.query_params.get('name') or ""
        queryset = Department.objects.filter(user=request.user, name__icontains=name)

        page = self.paginate_queryset(queryset)
        if page is not None:
            serializer = self.get_serializer(page, many=True)
            return self.get_paginated_response(serializer.data)

        serializer = self.get_serializer(queryset, many=True)
        return Response(serializer.data)

    def update(self, request, *args, **kwargs):
        partial = kwargs.pop('partial', False)
        instance = self.get_object()
        serializer = self.get_serializer(instance, data=request.data, partial=partial)
        serializer.is_valid(raise_exception=True)
        self.perform_update(serializer)

        employee = Employee.objects.get(id=request.data['head'])
        employee.head= "SIM"

        return Response(serializer.data)

class EmployeeViewSet(ModelViewSet):
    queryset = Employee.objects.all()
    serializer_class = EmployeeSerializer
    permission_classes = [permissions.AllowAny]

    def list(self, request, *args, **kwargs):
        name = request.query_params.get('name') or ""
        try:
            department = Department.objects.get(id=request.query_params.get('department'))
            queryset = Employee.objects.filter(department=department, name__icontains=name)
        except Department.DoesNotExist:
            queryset = Employee.objects.filter(name__icontains=name)

        page = self.paginate_queryset(queryset)
        if page is not None:
            serializer = self.get_serializer(page, many=True)
            return self.get_paginated_response(serializer.data)

        serializer = EmployeeSerializer(queryset, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK)

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.initial_data['code'] = code()
        serializer.is_valid(raise_exception=True)
        serializer.validated_data['department'] = Department.objects.get(id=request.data['department'])
        self.perform_create(serializer)
        headers = self.get_success_headers(serializer.data)
        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)

    def update(self, request, *args, **kwargs):
        partial = kwargs.pop('partial', False)
        instance = self.get_object()
        serializer = self.get_serializer(instance, data=request.data, partial=partial)
        serializer.is_valid(raise_exception=True)
        self.perform_update(serializer)
        instance.department = Department.objects.get(id=request.data['department'])
        instance.save()

        return Response(serializer.data)

    @action(methods=['get'], detail=False, url_path='last_five')
    def last_five(self, request, *args, **kwargs):
        last_five = []

        for employee in Employee.objects.all():
            queryset = CheckIn.objects.filter(employee_id=employee.id)
            last_five.append({'nome': employee.name, 'qtd': queryset.count()})

        last_five = sorted(last_five, key=lambda d: d['qtd'])[:5]

        return Response(last_five)

    @action(methods=['get'], detail=False, url_path='un_checked')
    def un_checked(self, request, *args, **kwargs):

        unchecked_employees = []

        for employee in Employee.objects.all():

            employee = EmployeeSerializer(employee).data
            print(not employee)

            if(not bool(employee['status']['checkin'])):
                print(employee)
                unchecked_employees.append(employee)

        return Response(unchecked_employees)

    @action(methods=['get'], detail=False, url_path='percent')
    def percent(self, request, *args, **kwargs):
        queryset = Employee.objects.all()
        qtd_employee = queryset.count()
        data_atual = datetime.now().date()
        checkin = CheckIn.objects.filter(date__date=data_atual)
        percent = (checkin.count() / qtd_employee) * 100

        print('quanditdade de funcionario', qtd_employee, 'quanditdade de checkin', checkin.count())
        return Response(int(percent))

    @action(methods=['get'], detail=True, url_path='messages')
    def messages(self,request, *args, **kwargs):
        employee = Employee.objects.get(id=kwargs['pk'])

        messages = Message.objects.filter(Q(department=employee.department) | Q(employee=employee) | Q(Q(employee__isnull=True), Q(department__isnull=True)))
        queryset = MessageSerializer(messages, many=True)
        return Response(queryset.data, status=status.HTTP_200_OK)

    @action(methods=['get'], detail=True, url_path='today_yesterday')
    def today_yesterday(self,request,*args, **kwargs):
        employee = Employee.objects.get(id=kwargs['pk'])

        today = datetime.now()
        yesterday = datetime.now() - timedelta(days=1)

        checkins = CheckIn.objects.filter(employee=employee)
        checks = []
        for checkin in checkins:
            date = checkin.date - timedelta(hours=2)

            if date.date() == today.date() or date.date() == yesterday.date():
                checks.append(checkin)

        data = []
        for checkin in checks:
            data.append({"date" : checkin.date.date().strftime('%d/%m/%Y'), "time" : (checkin.date-timedelta(hours=3)).time().strftime('%H:%M')})

        return Response(data, status=status.HTTP_200_OK)

    @action(methods=['get'], detail=False, url_path='employee_code/(?P<code>[^/.]+)')
    def employee_by_code(self, request, *args, **kwargs):
        try:
            employee = Employee.objects.get(code=self.kwargs['code'])
            serializer = EmployeeSerializer(employee)
            return Response(serializer.data, status=status.HTTP_200_OK)
        except Employee.DoesNotExist:
            return Response({"Error": "Employee not found."}, status=status.HTTP_400_BAD_REQUEST)
class CheckInViewSet(generics.CreateAPIView, generics.ListAPIView, generics.RetrieveAPIView, GenericViewSet):

    queryset = CheckIn.objects.all()
    serializer_class = CheckInSerializer
    permission_classes = [permissions.AllowAny]

    @action(methods=['get'], detail=False, url_path='week_checkins')
    def week(self,request, *args, **kwargs):
        dt = datetime.today()
        week = [dt + timedelta(days=i) for i in range(0 - dt.weekday(), 7 - dt.weekday())]
        dias = ["Domingo", "Segunda-feira", "Terça-feira", "Quarta-feira", "Quinta-feira", "Sexta-Feira", "Sábado"]
        results = []
        for day in week:
            number = 0
            checkins = CheckIn.objects.all()
            for checkin in checkins:
                if checkin.date.date() == day.date():
                    number += 1
            results.append({'name': dias[day.weekday()], 'checkIns': number})

        return Response(results, status=status.HTTP_200_OK)

    # def __init__(self, *args, **kwargs):
    #     super().__init__(*args, **kwargs)
    #     self.add_tags(['checkin'])

class MessageViewSet(generics.CreateAPIView, generics.ListAPIView, generics.RetrieveAPIView, generics.DestroyAPIView ,GenericViewSet):
    queryset = Message.objects.all()
    serializer_class = MessageSerializer
    permission_classes = [permissions.AllowAny]

    def list(self, request, *args, **kwargs):
        type = request.query_params.get('type')
        name = request.query_params.get('name') or ""
        print(name)

        if type:
            queryset = Message.objects.filter(Q(employee__name__icontains=name) | Q(manager__name__icontains=name), message_type=type)
        else:
            queryset = Message.objects.filter(Q(employee__name__icontains=name) | Q(manager__name__icontains=name))

        page = self.paginate_queryset(queryset)
        if page is not None:
            serializer = self.get_serializer(page, many=True)
            return self.get_paginated_response(serializer.data)

        serializer = MessageSerializer(queryset, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK)

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        if 'department' in request.data:
            serializer.validated_data['department'] = Department.objects.get(id=request.data['department'])
        if not request.user.__class__.__name__ == "AnonymousUser":
            serializer.validated_data['manager'] = request.user
        if 'employee' in request.data:
            serializer.validated_data['employee'] = Employee.objects.get(id=request.data['employee'])
        self.perform_create(serializer)
        headers = self.get_success_headers(serializer.data)
        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)

    @action(methods=['get'], detail=False, url_path='message_unread')
    def count_message_read(self, request):
        queryset = self.queryset.filter(read=False)
        messages_unread = queryset.count()
        return Response({'total': messages_unread})

    def retrieve(self, request, *args, **kwargs):
        instance = self.get_object()
        instance.read = True
        instance.save()
        serializer = self.get_serializer(instance)

        return Response(serializer.data)

    @action(methods=['get'], detail=False, url_path='news')
    def news(self, request):
        return Response(Message.objects.filter(read=False).count(), status=status.HTTP_200_OK)


import numpy as np
import cv2
import boto3
import copy
import numpy as np

from os import listdir
from os.path import isfile, join

def getFaces(img):
    face_cascade = cv2.CascadeClassifier('classifiers/haarcascade_frontalface_alt2.xml')
    profile_cascade = cv2.CascadeClassifier('classifiers/haarcascade_profile.xml')
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces_profile = profile_cascade.detectMultiScale(gray, 1.3, 5)
    faces_front   = face_cascade.detectMultiScale(gray,1.3, 5)
    faces = [x for x in faces_profile]
    faces += [x for x in faces_front]
    return faces

def showFaces(img, faces):
    for (x,y,w,h) in faces:
        #print (x,y,w,h)
        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
        crop_img = img[y:y+h, x:x+w]
        cv2.imshow('crop',crop_img)

        #roi_gray = gray[y:y+h, x:x+w]
        roi_color = img[y:y+h, x:x+w]
    cv2.imshow('img',img)

def wanted(suspect_path,crop_image,info_path):
    wanted_path = "wantedTemplate/wanted.jpeg"
    wanted_image = cv2.imread(wanted_path)
    lenghtWanted = len(wanted_image[0])
    heightWanted = len(wanted_image)

    suspect_image = cv2.imread(suspect_path)
    suspect_image = cv2.resize(suspect_image,( ((lenghtWanted*80)//100) , ((heightWanted*78)//100) ))
    lenghtSuspect = len(suspect_image[0])
    heightSuspect = len(suspect_image)

    wanted_image[int(heightWanted*.2):int(heightWanted*.2) + heightSuspect,int(lenghtWanted*.1):int(lenghtWanted*.1) + lenghtSuspect] = suspect_image

    font = cv2.FONT_HERSHEY_SIMPLEX
    crop_image = cv2.resize(crop_image,(lenghtWanted,lenghtWanted))

    cv2.putText(crop_image,'Found',(10,len(crop_image)), font, 3,(255,255,255),2,cv2.LINE_AA)
    faces = np.concatenate((wanted_image, crop_image), axis=0)

    text = np.zeros((len(faces), (len(faces)//2), 3), np.uint8)
    text[:] = (255,255,255)

    with open(info_path) as f:
        content = f.readlines()
    content = [x.strip() for x in content]
    pos = 0
    for line in content:
        pos+=50
        cv2.putText(text,line,(10,pos), font, 1,(0,0,0),2,cv2.LINE_AA)

    full_image = np.concatenate((faces, text), axis=1)

    cv2.imshow("found",full_image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()


import numpy as np
import pytest
from sklearn.utils.estimator_checks import parametrize_with_checks

import random_tree_models.decisiontree as dtree
import random_tree_models.gradientboostedtrees as gbt
import random_tree_models.xgboost as xgboost

class TestXGBoostTemplate:
    model = xgboost.XGBoostTemplate()

    def test_tree_(self):
        assert not hasattr(self.model, "trees_")

    def test_fit(self):
        try:
            self.model.fit(None, None)
        except NotImplementedError as ex:
            pytest.xfail("XGBoostTemplate.fit expectedly refused call")

    def test_predict(self):
        try:
            self.model.predict(None)
        except NotImplementedError as ex:
            pytest.xfail("XGBoostTemplate.predict expectedly refused call")

class TestXGBoostRegressor:
    X = np.array(
        [
            [-1, -1],
            [1, -1],
            [1, 1],
            [-1, 1],
        ]
    )
    y = np.array([0.0, 0.0, 1.0, 1.0])

    def test_fit(self):
        model = xgboost.XGBoostRegressor()
        model.fit(self.X, self.y)
        assert all(
            [
                isinstance(model, dtree.DecisionTreeRegressor)
                for model in model.trees_
            ]
        )

    def test_predict(self):
        model = xgboost.XGBoostRegressor()
        model.fit(self.X, self.y)
        predictions = model.predict(self.X)
        assert np.allclose(predictions, self.y)

class TestXGBoostClassifier:
    model = xgboost.XGBoostClassifier()

    X = np.array(
        [
            [-1, -1],
            [1, -1],
            [1, 1],
            [-1, 1],
        ]
    )
    y = np.array([False, False, True, True])

    def test_classes_(self):
        assert not hasattr(self.model, "classes_")

    def test_fit(self):
        model = xgboost.XGBoostClassifier()
        model.fit(self.X, self.y)
        assert not hasattr(self.model, "classes_")
        assert all(
            [
                isinstance(model, dtree.DecisionTreeRegressor)
                for model in model.trees_
            ]
        )

    def test_predict(self):
        model = xgboost.XGBoostClassifier()
        model.fit(self.X, self.y)
        predictions = model.predict(self.X)
        assert (predictions == self.y).all()

@pytest.mark.slow
@parametrize_with_checks(
    [xgboost.XGBoostRegressor(), xgboost.XGBoostClassifier()]
)
def test_xgboost_estimators_with_sklearn_checks(estimator, check):
    """Test of estimators using scikit-learn test suite

    Reference: https://scikit-learn.org/stable/modules/generated/sklearn.utils.estimator_checks.parametrize_with_checks.html#sklearn.utils.estimator_checks.parametrize_with_checks
    """
    check(estimator)

@pytest.mark.parametrize(
    "y_float, start_estimate_exp",
    [
        (np.array([-1.0, 1.0]), 0),
        (np.array([-1.0, 1.0, 1.0, 1.0]), 0.5493061443340549),
        (np.array([-1.0, -1.0, -1.0, 1.0]), -0.5493061443340549),
        (np.array([True, True, False, False]), None),
        (np.array([-2.0, -2.0, 2.0, 2.0]), None),
    ],
)
def test_compute_start_estimate_binomial_loglikelihood(
    y_float: np.ndarray, start_estimate_exp: float
):
    try:
        # line to test
        start_estimate = xgboost.compute_start_estimate_binomial_loglikelihood(
            y_float
        )
    except ValueError as ex:
        if start_estimate_exp is None:
            pytest.xfail(f"expectedly failed for non -1 and 1 values")
        else:
            raise ex
    else:
        if start_estimate_exp is None:
            pytest.fail(f"unexpectedly passed for non -1 and 1 values")
        assert np.isclose(start_estimate, start_estimate_exp)

@pytest.mark.parametrize(
    "y,start_estimate,g_exp",
    [
        (np.array([1.0]), 0.5, np.array([0.5])),
        (np.array([1.0, 1.0]), 0.5, np.array([0.5, 0.5])),
    ],
)
def test_compute_derivatives_negative_least_squares(
    y: np.ndarray, start_estimate: float, g_exp: np.ndarray
):
    # line to test
    g, h = xgboost.compute_derivatives_negative_least_squares(y, start_estimate)

    assert g.shape == h.shape
    assert np.allclose(g, g_exp)
    assert np.allclose(h, -1)

@pytest.mark.parametrize(
    "y_float,start_estimate,g_exp,h_exp",
    [
        (
            np.array([-1.0, 1.0]),
            0.0,
            np.array([-1.0, 1.0]),
            np.array([-1.0, -1.0]),
        ),
        (
            np.array([-1.0, -1.0, 1.0, 1.0]),
            0.0,
            np.array([-1.0, -1.0, 1.0, 1.0]),
            np.array([-1.0, -1.0, -1.0, -1.0]),
        ),
        # failure cases
        (np.array([False, True]), 0.0, None, None),
        (np.array([-2.0, 2.0]), 0.0, None, None),
    ],
)
def test_compute_derivatives_binomial_loglikelihood(
    y_float: np.ndarray,
    start_estimate: float,
    g_exp: np.ndarray,
    h_exp: np.ndarray,
):
    yhat = np.ones_like(y_float) * start_estimate
    is_bad = g_exp is None and h_exp is None
    try:
        # line to test
        g, h = xgboost.compute_derivatives_binomial_loglikelihood(y_float, yhat)
    except ValueError as ex:
        if is_bad:
            pytest.xfail("Expectedly failed for incorrect y_float values")
        else:
            raise ex
    else:
        if is_bad:
            pytest.fail("Unexpectedly passed for incorrect y_float values")
        assert g.shape == h.shape
        assert np.allclose(g, g_exp)
        assert np.allclose(h, h_exp)


# python 3.6 and more
# FastAPI service for docker
#
# example:
# (build --build_arg WORKERS=5 --build-arg FASTAPI_PORT=8000 -t ASR2Format .)
# docker run -p 8000:8000 --rm ASR2Format
# or local start with uvicorn: uvicorn main:app

fastapi_description = '''
Сервис преобразования распознанных ответов на заданные вопросы в необходимый формат.

Состоит из следующих частей:
 * общая часть - отображает параметры и текущее состояние микросервиса,
   имеет подключаемый логгер со своими настройками
 * операции с числами и числительными - необходимые операции, которые необходимо выполнить после
   распознования речи человека содержащие числительные
 * операции с временем - необходимые операции, которые необходимо выполнить после
   распознования речи человека содержащие время, часть суток и пр.
 * операции с датами - необходимые операции, которые необходимо выполнить после
   распознования речи человека содержащие даты
 * операции с адресами - необходимые операции, которые необходимо выполнить после
   распознования речи человека содержащие описательные признаки адреса в зависимости от особенностей региона
 * операции с услугами - необходимые операции, которые необходимо выполнить после
   распознования речи человека содержащие описательные признаки услуги.

'''

from typing import Any
from fastapi import Depends, FastAPI, Body, Path, Query, Request, HTTPException, status
from functools import lru_cache
from api_settings import Settings
from sheme import *
import uvicorn
import os
import sys
import socket as sock

try:
    from app_logger import get_logger, LogLevel
    from app_logger import Settings as LogSettings
except Exception as e:
    print('Критическая ошибка!\nПроверте значения в конфигурационный файл log_config или переменные среды')
    print(e)
    sys.exit()

# Кэшируемые функции выполняющиеся один раз для заданных параметров
@lru_cache()
def get_settings():
    return Settings()

def get_log_conf():
    return LogSettings()

@lru_cache()
def get_status():
    try:
        host_name = os.uname().nodename
    except:
        host_name = sock.gethostname()
    host_ip = sock.gethostbyname(host_name)
    py_ver = f"{sys.version_info.major}.{sys.version_info.minor}"
    return InfoDynamic(docker_ip=host_ip, docker_name=host_name, py_version=py_ver)

# Вычисление строки сообщения для логгера

def get_log_str(request: Request, func_name: str, answer: Any):
    client_ip = request.client.host
    method = request.method
    path = request.url.path
    if method.upper() == 'POST':
        body = request.body()
        result = "POST POST POST {body}"
    else:
        result = f"From {client_ip} {method} {path} SUCCESS worked {func_name} and return json({answer})"
    return result

############################ FastAPI инициализация
tags_metadata = [
    {
        "name": "common",
        "description": "Состояние и параметры сервиса fastAPI",
    },
    {
        "name": "number_manipulation",
        "description": "Утилиты для работы с числителями",
        # "externalDocs": {
        #     "description": "Thrift IDL MiddleEWare схема",
        #     "url": "https://fastapi.tiangolo.com/",
        # }
    },
    {
        "name": "time_manipulation",
        "description": "Утилиты для работы с временем",
    },
    {
        "name": "date_manipulation",
        "description": "Утилиты для работы с датами",
    },
    {
        "name": "address_search",
        "description": """Утилиты для поиска необходимого офиса или места
        по произвольному сообщению человека, исходя из особенностей региона""",
    },
    {
        "name": "service_search",
        "description": """Утилиты для поиска необходимого сервиса или услуги
        по произвольному сообщению человека, исходя из особенностей региона""",
    },
]
try:
    settings = get_settings()
    if settings.log_driver:
        logger_drv = get_logger(__name__, LogLevel.debug.value)
    else:
        logger_drv = None
except Exception as e:
    print('Критическая ошибка!\nПроверте значения в конфигурационных файлах или переменные среды')
    print(e)
    sys.exit()

app = FastAPI(
    title = settings.name_app,
    description = fastapi_description,
    version = settings.version,
    contact = {
        "name": "Mineev Alexander",
        "email": "mineev.buryat@gmail.com"
    }
)

@app.get('/', tags=['common'], response_model=Settings)
def api_settings(
    request: Request,
    settings: Settings = Depends(get_settings)):
    '''Показать текущие параметры fastapi сервиса
    Параметры меняются через переменные среды. Имя параметра соответсвует переменной среды.'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'api_settings', settings))
    return settings

@app.get('/info', tags=['common'], response_model=InfoDynamic)
def api_info(
    request: Request,
    info: InfoDynamic = Depends(get_status)):
    '''Показать динамические параметры fastapi сервиса
    Отображается текущий ip адрес, dns имя хоста и пр.'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'api_info', info))
    return info

@app.get('/conf_log', tags=['common'], response_model=LogSettings)
def log_config(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Показать конфигурацию логгера'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

############################# УТИЛИТЫ ################################################

# числительные
# {
#   "utterance": "сто тридцат шеcть двадцать пять и восемьдесят девить",
#   "only_numbers": 0
# }
@app.get('/util/number', tags=['number_manipulation'], response_model=LogSettings)
def word_to_number(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Преобразовать строку содержащие  числительные в список обнаруженых чисел'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

# время
@app.get('/util/time/partday', tags=['time_manipulation'], response_model=LogSettings)
def word_to_partday(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Преобразовать строку содержащее названое человеком часть суток в временной промежуток'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

@app.get('/util/time/time', tags=['time_manipulation'], response_model=LogSettings)
def word_to_time(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Преобразовать строку содержащее названое человеком время в часы и минуты'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

# дата
@app.get('/util/date/date', tags=['date_manipulation'], response_model=LogSettings)
def word_to_date(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Преобразовать строку содержащее названое человеком дату в день, месяц, год с учетом перехода между месяцами и годами'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf
# адрес
@app.get('/util/address/namespace', tags=['address_search'], response_model=LogSettings)
def has_address_namespace(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Показать имеющиеся в базе пространства имен для различных регионов.'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

@app.post('/util/address/namespace', tags=['address_search'], response_model=LogSettings)
def add_address_namespace(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Добавить новый регион с особенностями описания адресов'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

@app.post('/util/address/namespace/{space}', tags=['address_search'], response_model=LogSettings)
def add_address_sign(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Добавить в регион новый описательный признак адреса'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

@app.get('/util/address/search', tags=['address_search'], response_model=LogSettings)
def search_address(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Поиск конкретного адреса по названному человеком описательным признакам'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

# услуга
@app.get('/util/service/namespace', tags=['service_search'], response_model=LogSettings)
def has_service_namespace(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Показать имеющиеся в базе пространства имен услуг для различных регионов.'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

@app.get('/util/service/search', tags=['service_search'], response_model=LogSettings)
def search_service(
    request: Request,
    log_conf: LogSettings = Depends(get_log_conf)):
    '''Поиск конкретной услуги по названному человеком описательным признакам'''
    if settings.log_driver and logger_drv:
        logger_drv.debug(get_log_str(request, 'log_config', log_conf))
    return log_conf

if __name__ == "__main__":
    uvicorn.run(
        "main:app",
        host=settings.fastapi_host,
        port=settings.fastapi_port,
        access_log=False)


import os
from logging import Logger
from typing import Union, Tuple, Sequence, Optional

import lightgbm as lgb
import matplotlib.pyplot as plt
import numpy as np
import optuna
import pandas as pd
from lightgbm import early_stopping, log_evaluation
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold

from utils.func_utils import create_empty_folder
from .predictor import Predictor

class LGBMPredictor(Predictor):
    def __init__(self, logger: Logger, log_folder: str, name: str = None, cat_feature_names: Optional['list[str]'] = None,
                 drop_feature_names: Optional['list[str]'] = None, override_logs: bool = True, use_random_search: bool = False,
                 task_type: str = 'CPU', perform_feature_analysis: bool = True):
        # avoids mutable arguments, set to an empty list by default if not provided
        if drop_feature_names is None:
            drop_feature_names = []
        if cat_feature_names is None:
            cat_feature_names = []

        # generate a relevant name if not set
        if name is None:
            name = f'LGBM_{task_type}_{"rs" if use_random_search else "default"}'

        super().__init__(logger, log_folder, name, override_logs)

        self.use_random_search = use_random_search
        self.perform_feature_analysis = perform_feature_analysis
        self.task_type = task_type

        self.feature_names = None  # type: list
        self.drop_feature_names = drop_feature_names
        self.label_name = None
        self.drop_column_indexes = None
        self.cat_feature_names = cat_feature_names
        self.cat_feature_indexes = []

    def __get_max_b(self, df: pd.DataFrame):
        return df['blocks'].max()

    def __setup_features_data(self, df: pd.DataFrame):
        # y to predict is always the first column in POPNAS case
        self.label_name = df.columns.values.tolist()[0]

        # drop label and specified columns
        df = df.drop(columns=self.drop_feature_names)
        self.feature_names = df.columns.values.tolist()

        self.cat_feature_indexes = [i for i, f_name in enumerate(self.feature_names) if f_name in self.cat_feature_names]

    def __objective(self, trial: optuna.Trial, X, y):
        param_grid = {
            #         "device_type": trial.suggest_categorical("device_type", ['gpu']),
            "objective": "regression",
            "metric": "l2",
            "n_estimators": trial.suggest_categorical("n_estimators", [2000, 4000]),
            "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.3),
            "num_leaves": trial.suggest_int("num_leaves", 20, 300, step=10),
            "max_depth": trial.suggest_int("max_depth", 3, 9),
            "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 10, 20),
            "max_bin": trial.suggest_int("max_bin", 200, 300),
            "lambda_l1": trial.suggest_float("lambda_l1", 0, 5, step=0.2),
            "min_gain_to_split": trial.suggest_float("min_gain_to_split", 0, 15),
            "bagging_fraction": trial.suggest_float(
                "bagging_fraction", 0.2, 0.9, step=0.1
            ),
            "bagging_freq": trial.suggest_categorical("bagging_freq", [1]),
            "feature_fraction": trial.suggest_float(
                "feature_fraction", 0.2, 0.9, step=0.1
            ),
        }

        cv = KFold(n_splits=5, shuffle=True, random_state=1121218)

        cv_scores = np.empty(5)
        for idx, (train_idx, test_idx) in enumerate(cv.split(X, y)):
            X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
            y_train, y_test = y[train_idx], y[test_idx]

            model = lgb.LGBMRegressor(**param_grid)
            model.fit(
                X_train,
                y_train,
                eval_set=[(X_test, y_test)],
                eval_metric="l2",
                callbacks=[
                    early_stopping(100),
                    log_evaluation(100)
                ],
            )
            preds = model.predict(X_test)
            cv_scores[idx] = mean_squared_error(y_test.values, preds)

        return np.mean(cv_scores)

    def train_ensemble(self, dataset: Union[str, 'list[tuple]'], splits: int = 5, use_data_augmentation=True):
        raise NotImplementedError("LGBM predictor doesn't support ensemble")

    def train(self, dataset: Union[str, 'list[Tuple]'], use_data_augmentation=True):
        # TODO
        if not isinstance(dataset, str):
            raise TypeError('LGBM supports only files, conversion to file is a TODO...')

        dataset_df = pd.read_csv(dataset)
        actual_b = self.__get_max_b(dataset_df)
        if self.feature_names is None:
            self.__setup_features_data(dataset_df)

        dataset_df = dataset_df.drop(columns=self.drop_feature_names)

        x = dataset_df.drop(columns=[self.label_name])
        y = dataset_df[self.label_name]

        train_ds = lgb.Dataset(x, y)
        train_log_folder = os.path.join(self.log_folder, f'B{actual_b}')
        create_empty_folder(train_log_folder)

        # train the model with random search
        if self.use_random_search:
            study = optuna.create_study(direction="minimize", study_name="LGBM Regressor")
            study.optimize(lambda trial: self.__objective(trial, x, y), n_trials=20)

            # LightGBMTunerCV(params, train_ds, folds=KFold(n_splits=5), callbacks=[early_stopping(100), log_evaluation(100)], return_cvbooster=True)

            self._logger.info("Best value (mse): %0.3f", study.best_value)
            self._logger.info("\tBest params:")

            for key, value in study.best_params.items():
                self._logger.info(f"\t\t{key}: {value}")

            self._logger.info("Training model with all samples, with best parameters found")
            self.model = lgb.train(study.best_params, train_ds)
        # else simply train the model with default parameters
        else:
            self.model = lgb.train({'objective': 'regression'}, train_ds)
            self._logger.info('LGBM training complete')

        if self.perform_feature_analysis:
            fig, ax = plt.subplots()
            lgb.plot_importance(self.model, ax)
            fig.savefig(os.path.join(train_log_folder, 'feature_importance.png'), bbox_inches='tight')

    def predict(self, x: list) -> float:
        # make sure categorical features are integers (and not float, pandas converts them to floats like 1.0)
        if self.cat_feature_indexes is not None:
            for cat_index in self.cat_feature_indexes:
                x[cat_index] = int(x[cat_index])

        # returned as a numpy array of single element
        return self.model.predict([x])[0]

    def predict_batch(self, x: 'Sequence[list]') -> np.ndarray:
        # make sure categorical features are integers (and not float, pandas converts them to floats like 1.0)
        if self.cat_feature_indexes is not None:
            for sample in x:
                for cat_index in self.cat_feature_indexes:
                    sample[cat_index] = int(sample[cat_index])

        # returned as a numpy array of single element
        preds = self.model.predict(x)  # type: np.ndarray
        return preds.reshape(-1)

    def prepare_prediction_test_data(self, file_path: str) -> 'tuple[list[Union[str, list[tuple]]], list[list], list[list[float]]]':
        dataset_df = pd.read_csv(file_path)
        max_b = self.__get_max_b(dataset_df)
        self.__setup_features_data(dataset_df)

        dataset_paths = []
        prediction_samples = []
        real_values = []

        for b in range(1, max_b):
            train_df = dataset_df[dataset_df['blocks'] <= b]
            save_path = os.path.join(self.log_folder, f'inputs_B{b}.csv')
            train_df.to_csv(save_path, index=False)
            dataset_paths.append(save_path)

            predictions_df = dataset_df[dataset_df['blocks'] == (b + 1)]
            # in datasets that use data augmentation, filter them in prediction phase
            if 'data_augmented' in predictions_df.columns:
                # use == False, otherwise it will not work properly
                predictions_df = predictions_df[dataset_df['data_augmented'] == False]

            real_values.append(predictions_df[self.label_name].values.tolist())
            predictions_df = predictions_df.drop(columns=[self.label_name] + self.drop_feature_names)
            prediction_samples.append(predictions_df.values.tolist())

        return dataset_paths, prediction_samples, real_values


""" Batch class for radial regime regression. """

import random
import numpy as np
import scipy as sc

from sklearn.ensemble import IsolationForest

from . import radial_batch_tools as bt
from .decorators import init_components
from ..batchflow import Batch, action, inbatch_parallel, any_action_failed, FilesIndex, R

class RadialBatch(Batch):
    """
    Batch class that stores multiple time series along with other parameters
    for radial flow regime regression.

    Parameters
    ----------
    index : DatasetIndex
        Unique identifiers of rig data in the batch.
    preloaded : tuple, optional
        Data to put in the batch if given. Defaults to ``None``.

    Attributes
    ----------
    index : DatasetIndex
        Unique identifiers of rig data in the batch.
    """
    def __init__(self, index, *args, preloaded=None, **kwargs):
        super().__init__(index, preloaded, *args, **kwargs)
        self.time = self.array_of_nones
        self.derivative = self.array_of_nones
        self.target = self.array_of_nones
        self.predictions = self.array_of_nones

    components = "time", "derivative", "target"

    @property
    def array_of_nones(self):
        """ 1-D ndarray: ``NumPy`` array with ``None`` values."""
        return np.array([None] * len(self.index))

    def _reraise_exceptions(self, results):
        """ Reraise all exceptions in the ``results`` list.

        Parameters
        ----------
        results : list
            Post function computation results.

        Raises
        ------
        RuntimeError
            If any paralleled action raised an ``Exception``.
        """
        if any_action_failed(results):
            all_errors = self.get_errors(results)
            raise RuntimeError("Cannot assemble the batch", all_errors)

    @action
    def load(self, src=None, fmt=None, components=None, *args, **kwargs):
        """ Load given batch components.

        This method supports loading of data from npz format.

        Parameters
        ----------
        src: str, optional
             an array with data
        fmt : str, optional
            Source format.
        components : str or array-like, optional
            Components to load.

        Returns
        -------
        batch : RadialBatch
            Batch with loaded components. Changes batch data inplace.
        """
        _ = args, kwargs
        if isinstance(components, str):
            components = list(components)
        if components is None:
            components = self.components

        if fmt is None:
            self.put_into_data(src, components)
            self._sort(components)
        else:
            self._load(fmt, components)

        return self

    @inbatch_parallel(init='indices')
    def _sort(self, index, components):
        i = self.get_pos(None, 'time', index)
        time_mask = np.argsort(getattr(self, 'time')[i])
        for component in components:
            data = getattr(self, component)[i]
            getattr(self, component)[i] = data[time_mask]

    @inbatch_parallel(init="indices", post="_assemble_load", target="threads")
    def _load(self, indice, fmt=None, components=None, *args, **kwargs):
        """ Load given components from file.

        Parameters
        ----------
        fmt : str, optional
            Source format.
        components : iterable, optional

        Returns
        -------
        batch : RadialBatch
            Batch with loaded components. Changes batch data inplace.

        Raises
        ------
        ValueError
            If source path is not specified and batch's ``index`` is not a
            ``FilesIndex``.
        """
        loaders = {'npz': bt.load_npz}

        if isinstance(self.index, FilesIndex):
            path = self.index.get_fullpath(indice)  # pylint: disable=no-member
        else:
            raise ValueError("Source path is not specified")

        return loaders[fmt](path, components, *args, **kwargs)

    def _assemble_load(self, results, *args, **kwargs):
        """ Concatenate results of different workers and update ``self``.

        Parameters
        ----------
        results : list
            Workers' results.

        Returns
        -------
        batch : RadialBatch
            Assembled batch. Changes components inplace.
        """
        _ = args, kwargs
        self._reraise_exceptions(results)
        components = kwargs.get("components", None)
        if components is None:
            components = self.components
        for comp, data in zip(components, zip(*results)):
            data = np.array(data + (None,))[:-1]
            if comp == 'target':
                data = np.array([d for d in data])
            setattr(self, comp, data)
        return self

    @action
    @init_components
    @inbatch_parallel(init="indices", target="threads")
    def drop_negative(self, index, src=None, dst=None, **kwargs):
        """ Leaves only positive values of derivative. src should be a list with
        two components name. First component must contain time values, second - derivative.

        Parameters
        ----------
        src : list of str len 2, optional
            list with components name where the first component name is a component with
            time, second with derivative.
        dst : list of str len 2, optional
            list with components name to put the result in the same order as src.

        Raises
        ------
        ValueError
            If all values in derivative component are negative.
        """
        _ = kwargs
        i = self.get_pos(None, src[0], index)
        time = getattr(self, src[0])[i]
        derivative = getattr(self, src[1])[i]
        mask = derivative > 0

        if sum(mask) == 0:
            raise ValueError("All values in derivative {} are negative!".format(index))

        getattr(self, dst[0])[i] = time[mask]
        getattr(self, dst[1])[i] = derivative[mask]
        return self

    @action
    @init_components
    @inbatch_parallel(init="indices", target="threads")
    def drop_outliers(self, index, contam=0.1, src=None, dst=None, **kwargs):
        """Drop outliers using Isolation Forest algorithm. src should be a list with
        two components name. First component must contain time values, second - derivative.

        Parameters
        ----------
        contam : float (from 0 to 0.5)
            The amount of contamination of the data set.
        src : list of str len 2, optional
            list with components name where the first component name is a component with
            time, second with derivative.
        dst : list of str len 2, optional
            list with components name to put the result in the same order as src.
        """
        _ = kwargs
        i = self.get_pos(None, src[0], index)

        derivative = getattr(self, src[1])[i]
        time = getattr(self, src[0])[i]

        x_data = np.array([derivative]).T
        isol = IsolationForest(contamination=contam).fit(x_data)
        pred = isol.predict(x_data)
        getattr(self, dst[0])[i] = time[pred == 1]
        getattr(self, dst[1])[i] = derivative[pred == 1]
        return self

    @action
    @init_components
    @inbatch_parallel(init="indices", target="threads")
    def get_samples(self, index, n_points, sampler=None, # pylint: disable=too-many-arguments
                    src=None, dst=None, interpolate='linear', seed=None, **kwargs):
        """ Draws samples from the interpolation of time and derivative components.

        Performs interpolation of the `derivative` on time using
        `scipy.interpolate.interp1d`, samples `n_points` within `time` component
        range, and calculates values of `derivative` in sampled points.
        Sampler should return values from [0, 1] interval, which later is
        stretched to the range on `time` component. Size of the sample equals
        `(1, n_points)`.

        Parameters
        ----------
        n_points : int
            Size of a sample.
        sampler : function or named expression
            Method to sample points within [0, 1] range.
            If callable, it should only take `size` as an argument.
        src : list of str len 2, optional
            list with components name where the first component name is a component with
            time, second with derivative.
        dst : list of str len 2, optional
            list with components name to put the result in the same order as src.
        interpolate : str, optional
            Specifies the kind of interpolation as a string
            ('linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic',
            'previous', 'next', where 'zero', 'slinear', 'quadratic' and 'cubic'
            refer to a spline interpolation of zeroth, first, second or third
            order; 'previous' and 'next' simply return the previous or next value
            of the point) or as an integer specifying the order of the spline
            interpolator to use.
        seed : int, optional
            Numpy seed to be fixed. Defaults to None.
        kwargs : misc
            Named arguments to the sampling
            or interpolation functions.

        Returns
        -------
        batch : RadialBatch
            Batch with changed `time` and `derivative`
            components. Changes components inplace.

        Note
        ----
        If sampler is name expression and it has a following parameters:
        `tmin`, `tmax`, 'size' and others. Than this params should be passed via kwargs.
        See ``beta_sampler`` method for exampleн.
        """
        _ = kwargs
        if seed:
            np.random.seed(seed)

        i = self.get_pos(None, src[0], index)
        time = getattr(self, src[0])[i]
        derivative = getattr(self, src[1])[i]

        interpolater = sc.interpolate.interp1d(time, derivative, kind=interpolate,
                                               bounds_error=True, assume_sorted=True)

        if  isinstance(sampler, R):
            sampler = R(sampler.name, **sampler.kwargs, size=(1, n_points))
            samples = sampler.get()
        elif callable(sampler):
            samples = sampler(size=(1, n_points))
        else:
            raise ValueError("You should specify sampler function!")

        tmin, tmax = np.min(time), np.max(time)
        sample_times = samples * (tmax - tmin) + tmin
        sample_times.sort(axis=-1)
        sample_derivatives = interpolater(sample_times)

        getattr(self, dst[0])[i] = np.array(sample_times)
        getattr(self, dst[1])[i] = np.array(sample_derivatives)
        return self

    @action
    @init_components
    def make_points(self, src=None, dst=None, **kwargs):
        """ Generated new component with name `dst` with value from `src`.

        Parameters
        ----------
        src : list of str len 2, optional
            list with component' name where the first component name is a component with
            time, second with derivative.
        dst : str, optional
            component' name to put data.
        """
        _ = kwargs
        points = []
        for comp in src:
            points.append(np.vstack(getattr(self, comp)))
        if not isinstance(dst, str):
            dst = dst[0]
        setattr(self, dst, np.array(list(zip(*points))).transpose(0, 2, 1))
        return self

    @action
    def make_target(self, src='target', **kwargs):
        """ Reshaped component co vector with shape = (-1, 1)."""
        _ = kwargs
        setattr(self, src, getattr(self, src).reshape(-1, 1))
        return self

    @action
    @init_components
    @inbatch_parallel(init='indices')
    def normalize(self, ix, src=None, dst=None, src_range=None, dst_range=None, **kwargs):
        """ Normalizes data element-wise to range [0, 1] by exctracting
        min(data) and dividing by (max(data)-min(data)).
        Parameters
        ----------
        src : list of either str or None
            Contains names of batch component(s) to be normalized. First component name is a
            component with time, second with derivative.
        dst : list of either str or None
            Contains names of batch component(s) where normalized components will be stored
            in the same order as src..
        dst_range : list of either str or None
            Contains names of batch component(s) where range of normalized componentes will
            be stored.
            Range is saved as a list of 2 elements:
            `[min(component_data), max(component_data) - min(component_data)].`
            if None then no range will be stored.
        src_range : list of either str or None
            Contains names of batch component(s) where range of normalized componentes have
            already been stored. If not None then component will be normalized with
            the given range.
        Returns
        -------
        self

        """
        _ = kwargs
        for i, component in enumerate(src):
            pos = self.get_pos(None, component, ix)
            comp_data = getattr(self, component)[pos]
            if src_range[i]:
                min_value, scale_value = getattr(self, src_range[i])[pos]
            else:
                min_value, scale_value = np.min(comp_data), (np.max(comp_data) - np.min(comp_data))
            new_data = (comp_data - min_value) / scale_value
            getattr(self, dst[i])[pos] = new_data

            if dst_range[i]:
                getattr(self, dst_range[i])[pos] = [min_value, scale_value]
        return self

    @action
    @init_components
    @inbatch_parallel(init='indices')
    def make_grid_data(self, ix, src=None, dst=None, grid_size=500, **kwargs):
        """ Makes grid
        Parameters
        ----------
        src : str or list of str
            Contains names of batch component(s) to make grid data from in format (x, y)
        dst : str or list of str
            Contains names of batch component(s) where (x_grid, y_grid) will be stored.
        grid_size : int
            Number of points in a grid

        Returns
        -------
        self
        """
        _ = kwargs
        if not (len(src) == 2 and len(dst) == 2):
            raise ValueError('Exactly two components must be passed to make grid data')

        pos = self.get_pos(None, src[0], ix)

        x_data = getattr(self, src[0])[pos]
        y_data = getattr(self, src[1])[pos]

        sorted_data = list(zip(*sorted(zip(x_data, y_data), key=lambda x: x[0])))

        x_grid = np.linspace(0, 1, num=grid_size)
        y_grid = np.interp(x_grid, sorted_data[0], sorted_data[1])

        getattr(self, dst[0])[pos] = x_grid
        getattr(self, dst[1])[pos] = y_grid.reshape((-1, 1))
        return self

    @action
    @init_components
    @inbatch_parallel(init='indices')
    def denormalize(self, ix, src=None, dst=None, src_range=None, **kwargs):
        """ Denormalizes component to initial range.

        Parameters
        ----------
        src : list of str
            Contains names of batch component(s) to be denormalized
        dst : list of either str or None
            Contains names of batch component(s) where denormalized components will be stored.
        src_range : list of str
            Contains names of batch component(s) where range data has been stored in format
            `[min(component_data), max(component_data) - min(component_data)].`

        Returns
        -------
        self
        """
        _ = kwargs

        for i, component in enumerate(src):
            pos = self.get_pos(None, component, ix)
            comp_data = getattr(self, component)[pos]
            if src_range[i]:
                min_value, scale_value = getattr(self, src_range[i])[pos]
                new_data = comp_data * scale_value + min_value
                getattr(self, dst[i])[pos] = new_data
            else:
                raise ValueError('Src_range must be provided to denormalize component')
        return self

    @action
    @init_components
    @inbatch_parallel(init='indices')
    def to_log10(self, ix, src=None, dst=None, **kwargs):
        """Takes a decimal logarithm from `src` and saves the resulting value to `dst`

        Parameters
        ----------
        src : src or list
            Name of the component with data
        dst : src or list
            Name of the component to save the result
        """
        _ = kwargs
        if isinstance(src, str):
            src = [src]
            dst = [dst]
        for i, component in enumerate(src):
            pos = self.get_pos(None, component, ix)
            comp_data = getattr(self, component)[pos]
            getattr(self, dst[i])[pos] = np.log10(comp_data)
        return self

    @action
    @init_components
    @inbatch_parallel(init='indices')
    def clip_values(self, ix, src, dst, **kwargs):
        """Clip values from `src` to 0, 1 and save it to `dst`
        """
        _ = kwargs
        if isinstance(src, str):
            src = [src]
            dst = [dst]

        for i, component in enumerate(src):
            pos = self.get_pos(None, component, ix)
            pred = getattr(self, component)[pos]
            getattr(self, dst[i])[pos] = np.clip(pred, 0, 1)
        return self

    @action
    @init_components
    def make_array(self, src=None, dst=None, **kwargs):
        """ Stacks a sequence of arrays in a given component.
        """
        _ = kwargs
        for i, component in enumerate(src):
            setattr(self, dst[i], np.stack(getattr(self, component)))
        return self

    @action
    @init_components
    def expand_dims(self, src=None, dst=None, **kwargs):
        """ Expands the shape of an array stored in a given component
        Inserts a new axis that will appear as the last axis.
        Acts like np.expand_dims with parameter axis=-1.
        Parameters
        ----------
        src : list of str
            Contains names of batch component(s) to be modiified
        dst : list of either str or None
            Contains names of batch component(s) where modiified components will be stored
        Returns
        -------
        self
        """
        _ = kwargs
        for i, component in enumerate(src):
            setattr(self, dst[i], getattr(self, component).reshape((-1, 1)))
        return self

    @action
    def hard_negative_sampling(self, statistics_name=None, fraction=0.5):
        """ Recreate batch with new indices corresponding to the largest loss
        value on the previous iterations.
        Parameters
        ----------
        statistics_name : str
            name of the pipeline variable where the dict with the index and
            corresponding loss values is stored.
        fraction : float in [0, 1]
            A fraction of the hard negative examples in the new batch.
        Returns
        -------
        self
        """
        btch_size = len(self.indices)
        if statistics_name and isinstance(self.pipeline.get_variable(statistics_name), dict):
            loss_history_dict = self.pipeline.get_variable(statistics_name)
            sorted_by_value = sorted(loss_history_dict.items(), key=lambda kv: kv[1])
            hard_count = int(btch_size * fraction)
            hard_indices = np.setdiff1d(np.array([x[0] for x in sorted_by_value]), self.indices)[:hard_count]
            new_index = np.hstack([np.random.choice(self.indices, size=btch_size - hard_count, replace=False), \
                                   hard_indices])
            random.shuffle(new_index)
            batch = RadialBatch(index=self.pipeline.dataset.index.create_subset(new_index))
            return batch
        return self

    @action
    def update_loss_history_dict(self, src='loss_history', dst='loss_history_dict'):
        """ Update pipeline variable dst that is dict with key - element's index and
        corresponding value, i.e. squarred error on that element from src
        Parameters
        ----------
        src : str
            name of the pipeline variable where element-wise statistics
            (e.g. unaggreagated loss) for the batch is stored.
            It can be a list or np.array of length batch_size.
        dst : str
            name of the pipeline variable where the dict with the index and
            corresponding values is stored.
        Returns
        -------
        self
        """
        new_loss_history_dict = dict(zip(self.index.indices, self.pipeline.get_variable(src)))
        if isinstance(self.pipeline.get_variable(dst), dict):
            new_loss_history_dict = {**new_loss_history_dict, **self.pipeline.get_variable(dst)}
        self.pipeline.update_variable(dst, new_loss_history_dict, mode='w')
        return self