import json

import src.data_loaders as module_data
import torch
from pytorch_lightning import seed_everything, Trainer
from torch.utils.data import DataLoader
from train import ToxicClassifier

def initialize_trainer(CONFIG):
    seed_everything(1234)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    def get_instance(module, name, config, *args, **kwargs):
        return getattr(module, config[name]["type"])(*args, **config[name]["args"], **kwargs)

    model = ToxicClassifier(CONFIG)
    model.to(device)

    dataset = get_instance(module_data, "dataset", CONFIG)
    val_dataset = get_instance(module_data, "dataset", CONFIG, train=False)

    data_loader = DataLoader(
        dataset,
        batch_size=CONFIG["batch_size"],
        num_workers=2,
        shuffle=True,
        drop_last=True,
        pin_memory=True,
    )

    valid_data_loader = DataLoader(
        val_dataset,
        batch_size=CONFIG["batch_size"],
        num_workers=2,
        shuffle=False,
    )

    trainer = Trainer(
        gpus=0 if torch.cuda.is_available() else None,
        limit_train_batches=2,
        limit_val_batches=2,
        max_epochs=1,
    )
    trainer.fit(model, data_loader, valid_data_loader)
    results = trainer.test(dataloaders=valid_data_loader)

    return results

def test_trainer():
    CONFIG = json.load(open("configs/Toxic_comment_classification_BERT.json"))
    CONFIG["dataset"]["args"][
        "train_csv_file"
    ] = "tests/dummy_data/jigsaw-toxic-comment-classification-challenge/train.csv"
    CONFIG["dataset"]["args"][
        "test_csv_file"
    ] = "tests/dummy_data/jigsaw-toxic-comment-classification-challenge/test.csv"
    CONFIG["batch_size"] = 2

    results = initialize_trainer(CONFIG)
    print(results)
    assert results[0]["test_acc"] > 0.3

if __name__ == "__main__":
    test_trainer()


import sys
from git import Repo, GitCommandError  # type: ignore
import requests
from requests import Response

from src.filepath import ROOT_PATH
from src.migration.migrator import Migrator, _Version
from src.logger_handler import LoggerHandler
from src import FUTURE_PYTHON_VERSION
from src.utils import restart_program

_logger = LoggerHandler("updater_module")
_GITHUB_RELEASE_URL = "https://api.github.com/repos/andrewohnsland/cocktailberry/releases"

class Updater:
    """Class to get update from GitHub"""

    def __init__(self):
        self.git_path = ROOT_PATH
        self.repo = Repo(self.git_path)

    def update(self):
        """Updates from the Git remote"""
        latest_tag = self._get_latest_tag()
        _logger.log_event("INFO", f"Updating to {latest_tag.name}")
        # Is there a better way to pull the state of a specific tag, without checking out to that tag?
        try:
            self.repo.remotes.origin.pull()
        except GitCommandError as err:
            _logger.log_event("ERROR", "Something went wrong while pulling the update")
            _logger.log_exception(err)
            return
        # restart the program, this will not work if executed over IDE
        print("Restarting the application!")
        _logger.log_event("INFO", "Restarting program to reload updated code")
        restart_program()

    def check_for_updates(self) -> tuple[bool, str]:
        """Check if there is a new version available"""
        # if not on master (e.g. dev) return false
        update_problem = ""
        if self.repo.active_branch.name != "master":
            update_problem = "Not on master branch, not checking for updates"
            _logger.log_event("WARNING", update_problem)
            return False, update_problem
        # Also do not make updates if current version does
        # not satisfy the future version requirement
        if sys.version_info < FUTURE_PYTHON_VERSION:
            update_problem = f"Python version is too old, not checking for updates. You need at least {FUTURE_PYTHON_VERSION}"  # noqa
            _logger.log_event("WARNING", update_problem)
            return False, update_problem
        # First fetch the origin latest data
        try:
            self.repo.remotes.origin.fetch()
        # if no internet connection, or other error, return False
        except GitCommandError as err:
            update_problem = "Something went wrong while fetching the repo data"
            _logger.log_event("ERROR", update_problem)
            _logger.log_exception(err)
            return False, update_problem + f"\n{err}"
        # Get the latest tag an compare the diff with the current branch
        # Usually this should work since the default is master branch and "normal" users shouldn't be changing files
        # Not using diff but local and remote tags to compare, since some problems exists comparing by diff
        latest_tag = self._get_latest_tag()
        # Either build diff or just simply check local version with latest
        # Currently using local version tag, this will prob work best,
        # if the programmer does not forget to update the version in the migrator
        migrator = Migrator()
        update_available = migrator.older_than_version(latest_tag.name.replace("v", ""))
        info = ""
        if update_available:
            _logger.log_event("INFO", f"Update {latest_tag.name} is available")
            release_data = requests.get(f"{_GITHUB_RELEASE_URL}?per_page=5", timeout=5000)
            info = self._parse_release_data(release_data)
        return update_available, info

    def _get_latest_tag(self):
        """Extracts the latest version number from the tags"""
        latest_tag = sorted(self.repo.tags, key=lambda t: _Version(t.name.replace("v", "")))[-1]
        return latest_tag

    def _parse_release_data(self, response: Response):
        """Converts the response into a string to display """
        migrator = Migrator()
        text = ""
        for release in response.json():
            name = release.get('name', '')
            body = release.get("body", '')
            tag_name = release.get("tag_name", '')
            # only add release information newer than local version
            if not migrator.older_than_version(tag_name.replace("v", "")):
                break
            sep = 70 * "_"
            text += f"Release: {name}\n{body}\n{sep}\n\n"
        return text


from collections import defaultdict
from functools import reduce
def solution(clothes):
    type = defaultdict(int)
    for cloth in clothes:
        type[cloth[1]]+=1
    return reduce(lambda x, y: x * y, [i+1 for i in type.values()])-1

print(solution([["yellow_hat", "headgear"], ["blue_sunglasses", "eyewear"], ["green_turban", "headgear"]]))

# from collections import Counter
# from functools import reduce
# def solution(clothes):
#     c = [a+1 for a in collections.Counter([x[1] for x in c]).values()]
#     answer = reduce(lambda x,y:x*y,c)-1
#     return answer
# reduce(lambda x,y:x*y,[a+1 for a in collections.Counter([x[1] for x in c]).values()])-1


import pygame
import little_ship_function as function
from little_ship_setting import Setting
from little_ship_ship import Little_ship
from pygame.sprite import Group
from rect import Rect

'''这是一个小飞船的游戏'''

def game_run():
    pygame.init()
    settings = Setting()
    screen = pygame.display.set_mode(
        (settings.screen_width, settings.screen_height))
    pygame.display.set_caption('little ship')

    little_ship = Little_ship(settings, screen)
    bullets = Group()
    rect = Rect(screen, settings)

    while True:
        function.check_event_key(settings, screen, little_ship, bullets)
        little_ship.little_ship_update()
        function.update_bullets(bullets, screen)
        function.update_rect(rect)
        function.update_screen(settings, screen, little_ship, bullets, rect)

game_run()


import os
import math
from urllib.parse import urlencode
import pandas as pd
import numpy as np
from ulttb import downsample
from bottle import abort

import config

class App():
    def __init__(self, *args, **kwargs):
        # set up output dir
        if not os.path.exists(config.APP_CONFIG["OUTPUT_DIR"]):
            os.makedirs(config.APP_CONFIG["OUTPUT_DIR"])

    def get_data(self, request):
        # check for params
        if (request.query.get("keys") is None or
            request.query.get("project") is None or
            request.query.get("source_id") is None):
            abort(400, "missing query parameter")

        # pull out parameters from the request
        bounds = [float(x) for x in request.query.get("bounds").split(",")] if request.query.get("bounds") else []
        in_keys = request.query.get("keys").split(",") if request.query.get("keys") else config.APP_CONFIG["DEFAULT_KEYS"]
        threshold = int(request.query.get("target")) if request.query.get("target") else config.APP_CONFIG["DEFAULT_TARGET"]
        project = request.query.get("project") if request.query.get("project") else "NONE"
        source_id = request.query.get("source_id") if request.query.get("source_id") else "NONE"
        output_format = request.query.get("format") if request.query.get("format") else config.APP_CONFIG["DEFAULT_FORMAT"]

        # deal with idioms
        keys = self._fix_keys(in_keys)

        # download the file if it's not cached
        filename = self._build_filename(project, source_id, keys)
        if not os.path.isfile(filename):
            url = self._build_url(project, source_id, keys)
            self._download_file(url, filename, keys)

        # decimate the data into the desired format
        data = None
        if "csv" in output_format:
            output_format = config.APP_CONFIG["AVAILABLE_FORMATS"]["csv"]
            data = self._decimate_data(filename, bounds, keys, in_keys, threshold, output_format).to_csv(index=False)
        elif "json" in output_format:
            output_format = config.APP_CONFIG["AVAILABLE_FORMATS"]["json"]
            data = self._decimate_data(filename, bounds, keys, in_keys, threshold, output_format)

        if data is None:
            abort(500, "Failed to decimate data")

        # clear cached data
        if not config.APP_CONFIG["CACHE_FILES"] and os.path.isfile(filename):
            os.remove(filename)

        return (output_format, data)

    def _fix_keys(self, in_keys):
        keys = ["_".join(key.split(" ")).lower() for key in in_keys]

        if len(keys) < 2:
            keys = [key for key in config.APP_CONFIG["DEFAULT_KEYS"]]

        time_found = False
        for i, key in enumerate(keys):
            if "time" in key:
                keys[i] = "measurement_date_time"
                time_found = True
            elif not key in config.APP_CONFIG["NO_MOD_KEYS"] and not key.endswith("_d"):
                keys[i] = key + "_d"

        # ensure there is always time in the data
        if not time_found:
            keys.append("measurement_date_time")

        return keys

    def _build_url(self, project, source_id, keys):
        query = {
            "facet": "on",
            "wt": "csv",
            "rows": "10000000",
            "sort": "%s asc" % (keys[0]),
            "fl": ",".join(keys),
            "q": "datatype:data AND project:%s AND source_id: %s" % (project, source_id)
        }

        return config.APP_CONFIG["SOLR_URL"] + "?" + urlencode(query)

    def _build_filename(self, project, source_id, keys):
        filename = "%s/%s__%s__%s.h5" % (config.APP_CONFIG["OUTPUT_DIR"], project, source_id, "_".join(keys))

        return filename

    def _download_file(self, url, filename, keys):
        # download file
        data = pd.read_csv(url)

        # remove empty lines
        data.dropna(inplace=True)

        # edit time entries
        for key in keys:
            if "time" in key.lower():
                to_sec = 10**9
                data[key] = (pd.to_datetime(data[key]).astype(np.int64)/to_sec).astype(np.int64)

        # mask out no data values
        masks = [data[key] != -9999 for key in keys]
        datamask = masks[0]
        for mask in masks:
            datamask = datamask & mask
        data = data[datamask]

        # save to hdf file
        data.to_hdf(filename, "table", append=True, data_columns=True)

        return filename

    def _decimate_data(self, filename, bounds=[], keys=[], header_keys=[], threshold=config.APP_CONFIG["DEFAULT_TARGET"], output_format=config.APP_CONFIG["DEFAULT_FORMAT"]):
        if len(keys) < 2:
            return None

        # edit time bounds
        if "time" in keys[0].lower():
            bounds = [ x / 1000 if self._magnitude(x) > 9 else x for x in bounds ]

        # read data from hdf file
        if(len(bounds) == 2):
            filter_str = "%s >= %f & %s <= %f" % (keys[0], bounds[0], keys[0], bounds[1])
            data = pd.read_hdf(filename, "table", where = filter_str)
        else:
            data = pd.read_hdf(filename, "table")

        # reset index bounds
        sub_data = data.reset_index(drop=True)

        # rearrange columns
        sub_data = sub_data[keys]

        # sort the data
        sub_data = sub_data.sort_values(by=[keys[0]])

        # decimate the data
        dec_data = sub_data.values.tolist()
        if(threshold != config.APP_CONFIG["NO_DECIMATION_TARGET"] and len(dec_data) > threshold):
            dec_data = downsample(dec_data, threshold)

        # format the data
        col_names = list(sub_data)
        if len(header_keys) > 0:
            col_names = [x for x in header_keys] # duplicate the list
            col_names.extend(list(sub_data)[len(header_keys):]) # add any additional columns (hidden time column)
        if output_format == config.APP_CONFIG["AVAILABLE_FORMATS"]["csv"]:
            dec_data = pd.DataFrame(dec_data, columns=col_names)
        elif output_format == config.APP_CONFIG["AVAILABLE_FORMATS"]["json"]:
            dec_data = {
                "data": dec_data,
                "meta": {
                    "sub_size": len(sub_data),
                    "dec_size": len(dec_data),
                    "columns": col_names
                }
            }

        return dec_data

    def _magnitude(self, x):
        return int(math.floor(math.log10(x)))


from django.shortcuts import render,redirect
from django.views.generic import ListView,DetailView,TemplateView,CreateView

from car_rent import settings
from main.models import Car,CarClass,Images,AboutUs,Contacts,CarMarka,SendList
from main.forms import OrderDataForm
from django.urls import reverse_lazy
from threading import Thread
from django.template.loader import render_to_string
import json
from django.core.mail import EmailMessage
from django.core.paginator import Paginator

# Create your views here.
class MainPageView(ListView):
    model = Car
    template_name = 'main.html'
    paginate_by = 3
    context_object_name = 'cars'

    def get_context_data(self, *args, **kwargs):
        context = super(MainPageView, self).get_context_data(**kwargs)
        context['classes'] = CarClass.objects.all()
        context['markas'] = CarMarka.objects.all()
        page = self.request.GET.get('page', 1) if self.request.GET.get('page', 1) != '' else 1
        data = self.get_queryset()
        if data:
            paginator = Paginator(data, self.paginate_by)
            results = paginator.page(page)
            index = results.number - 1
            max_index = len(paginator.page_range)
            start_index = index - 5 if index >= 5 else 0
            end_index = index + 5 if index <= max_index - 5 else max_index
            context['page_range'] = list(paginator.page_range)[start_index:end_index]
        return context

class CarDetailView(DetailView):
    model = Car
    template_name = 'detail.html'
    context_object_name = 'car'

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        images = Images.objects.filter(carr__id=self.get_object().pk)
        print(images)
        context['imagess'] = images
        return context

class CreateOrderView(CreateView):
    form_class = OrderDataForm
    template_name = 'form-fill.html'
    success_url='/success'

    # def get_success_url(self):
    #     return render(self.request,'success.html')

    #

    def form_valid(self,form):
        order = form.save(commit=False)
        order.order_data = self.request.POST.get('all-data')
        order.save()
        return super().form_valid(form)

# def send_email():
#     template_name = 'send-email.html'
#     context = {
#         'data': 'data'
#     }
#     msg = render_to_string(template_name, context)
#     subject = 'Новое заявление'
#     user_emails = SendList.objects.all().values_list('email', flat=True)
#     message = EmailMessage(subject=subject, body=msg, from_email=settings.EMAIL_HOST_USER, to=user_emails)
#     message.content_subtype = 'html'
#     message.send()

class OrderSuccessView(TemplateView):
    # Thread(target=send_email).start()
    template_name = 'success.html'

class AboutUsView(TemplateView):
    template_name = 'about-us.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        about = AboutUs.objects.first()
        context['about'] = about
        return context

class ContactPageView(TemplateView):
    template_name = 'contact.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        contacts = Contacts.objects.first()
        context['contacts'] = contacts
        return context


import streamlit as st
import datetime as DT
import pandas as pd

def gsc_data_work(df, site_url, days_selector, days_ago):
    try:
        #df = webproperty.query.search_type(search_type).range('today',days=days_ago).dimension('date', 'page').get().to_dataframe()
        df['date'] = pd.to_datetime(df['date']) # Convert date format
        df['clicks'] = pd.to_numeric(df['clicks'], downcast="integer")
        df['ctr'] = pd.to_numeric(df['ctr'], downcast="float")

        ########## INTRO MESSAGE BEGIN ############
        today = DT.date.today()

        start_date = today - DT.timedelta(days=days_selector)
        st.caption(f'You selected Google Discover Data for {site_url} from {start_date} to {today}.')

        ########## INTRO MESSAGE END ############

        return df

    except AttributeError:
        st.error("Check please your property URL, don't forget https://, www, the trailing slash if necessary.")

    if not site_url and refresh_button:
        st.error('Please enter the correct property URL exactly as in your Search Console.')


#! usr/bin/env python

from sys import argv
from os.path import exists
import simplejson as json

script, in_file, out_file = argv

data = json.load(open(in_file))

usable = data["data"]["list"]

geojson = {
    "type": "FeatureCollection",
    "features": [
    {
        "type": "Feature",
        "geometry" : {
            "type": "Point",
            "coordinates": [d["lng"], d["lat"]],
            },
            "properties" : {"price": float(d["price"].replace("M","").replace("K","")),"id_listing": d["id_listing"],"house_type": d["house_type"]}
    }    for d in usable]
}

output = open(out_file, 'w')
json.dump(geojson, output)

print(geojson)


import PySimpleGUI as sg

import planar_projection
import obj_reader
import workarounds

def RGB_2_HEX(x: tuple):
    return f"#{x[0]:02x}{x[1]:02x}{x[2]:02x}"

def refresh_view(RENDER_MODE):
    canvas: sg.Graph = window['-GRAPH-']
    canvas.erase()

    points = my_camera.project_object(my_object)

    if RENDER_MODE == 'LINES':
        if my_object.edges is None:
            canvas.draw_text("No lines data in file!", (0, 0), 'white')
            return

        for p1, p2 in my_camera.get_edges(my_object):
            canvas.draw_line(p1, p2, 'white', 3)

    if RENDER_MODE == 'POINTS':
        if my_object.verts is None:
            canvas.draw_text("No points in file!", (0, 0), 'white')
            return

        for p in points:
            canvas.draw_circle(p, 0.005, 'white', 'white')

    if RENDER_MODE == 'FACES':
        if my_object.faces is None:
            canvas.draw_text("No faces data in file!", (0, 0), 'white')
            return

        for f in my_camera.get_faces(my_object):
            verts = [points[p] for p in f]
            canvas.draw_polygon(verts, 'grey', 'orange', 0.02)

    if RENDER_MODE == 'SHADED':
        if my_object.faces is None:
            canvas.draw_text("No faces data in file!", (0, 0), 'white')
            return

        faces = my_camera.get_faces(my_object)
        for n, f in enumerate(faces):
            c = int(n / len(faces) * 150) + 50
            face_colour = RGB_2_HEX((c, c, c))
            verts = [points[p] for p in f]
            canvas.draw_polygon(verts, face_colour, face_colour, 0.02)

    # if RENDER_MODE == 'POINT_NUMBER':
    #     for n, p in enumerate(points):
    #         canvas.draw_text(str(n), p, 'white')

# Create Scene
my_object = obj_reader.import_obj('example_obj/utah_teapot.obj', rotation=('x', 90), translation=(0, 0, -1.5))
my_camera = planar_projection.Camera_3D(
    focal_distance=-12,
    projection_plane_distance=-10
)

GRAPH_SIZE = (350,350) if sg.running_trinket() else (500,500)
layout = [
    [sg.Graph(GRAPH_SIZE, (-1, -1), (1, 1), 'black', float_values=True, enable_events=True, key='-GRAPH-', drag_submits=True)],
    [sg.Text("R:", size=3), sg.Slider((0, 360), resolution=1, default_value=0, enable_events=True, orientation='horizontal', expand_x=True, key='-O-')],
    [sg.Text("X:", size=3), sg.Slider((-8, 8), resolution=0.1, default_value=0, enable_events=True, orientation='horizontal', expand_x=True, key='-X-')],
    [sg.Text("Y:", size=3), sg.Slider((-3, 3), resolution=0.1, default_value=0, enable_events=True, orientation='horizontal', expand_x=True, key='-Y-')],
    [sg.Text("Z:", size=3), sg.Slider((-3, 3), resolution=0.1, default_value=0, enable_events=True, orientation='horizontal', expand_x=True, key='-Z-')],
    [sg.Text("Render Mode:"), sg.Combo(['POINTS', "LINES", 'FACES', 'SHADED'], 'FACES', key="-REDNER_TYPE-", readonly=True, enable_events=True)]
]

window = sg.Window('3D Viewport', layout)

drag_loc = None
while True:
    event, values = window.read()

    if event == sg.WIN_CLOSED:
        break

    if event in ['-O-', '-X-', '-Y-', '-Z-']:
        my_object.orientation = values['-O-']
        my_object.position = [values['-X-'], values['-Y-'], values['-Z-']]

    if event == '-GRAPH-':
        new_drag_location = values['-GRAPH-']

        if not drag_loc:
            drag_loc = new_drag_location

        my_object.orientation += (drag_loc[0] - new_drag_location[0]) * 360
        my_object.position[0] += (drag_loc[1] - new_drag_location[1]) * -10

        drag_loc = new_drag_location

        window['-O-'].update(my_object.orientation)
        window['-X-'].update(my_object.position[0])

    if event == '-GRAPH-+UP':
        drag_loc = None

    refresh_view(values['-REDNER_TYPE-'])

window.close()


import configuracion
import vdb_generado
import utiles
import criticidades
import logging
import requests
import re
import time
from threading import Thread

class Iterador_Nessus_Vdb(Thread):

    conf = configuracion.Configuracion()

    componente = None
    columnas_fijas = None
    environmental_impact_subscore_modifiers = None
    ruta_nessus = None

    progressbar = None
    textbox = None

    vulnerabilidades_rejected = []

    def __init__(self, componente, columnas_fijas, environmental_impact_subscore_modifiers, ruta_nessus, progressbar, textbox):
        super().__init__()
        self.componente = componente
        self.columnas_fijas = columnas_fijas
        self.environmental_impact_subscore_modifiers = environmental_impact_subscore_modifiers
        self.ruta_nessus = ruta_nessus
        self.progressbar = progressbar
        self.textbox = textbox

    def run(self):
        logging.log(self.conf.vdbauto_nivel_log, "######## COMIENZA ITERACION DE "+utiles.get_nombre_fichero(self.conf.ruta_nessus)+" ########")
        self.insertar_texto_textbox("####### COMIENZA ITERACION DE "+utiles.get_nombre_fichero(self.conf.ruta_nessus)+" #######\n")

        self.iterar_nessus_csv(self.componente, self.columnas_fijas, self.environmental_impact_subscore_modifiers, self.ruta_nessus)

    def insertar_texto_textbox(self, texto):
        self.textbox.insert("end", texto)
        self.textbox.see("end")

    def iterar_nessus_csv(self, componente, columnas_fijas, environmental_impact_subscore_modifiers, ruta_nessus):

        vulnerabilidades_vdb = self.extraer_vulnerabilidades_vdb(self.conf.vdb_hoja, componente)
        vulnerabilidades_duplicadas = [] #almacena las vulnerabilidades que no se han insertado porque estaban en el vdb introducido
        vulnerabilidades_error = [] #almacena los cves que no se han añadido por un error en la peticion API

        lista_cabeceras_nessus_conf = list(self.conf.configuracion["vdb_generado"]["nessus"])
        cabeceras_nessus_csv = next(self.conf.nessus_reader) #sacamos las cabeceras del fichero csv Nessus para luego poder coger las columnas que se vayan a insertar en el metodo "anadir_columnas_nessus_fila"

        vdb_final = vdb_generado.Vdb_generado()

        plugin_id_anterior = "" #para evitar duplicidades ya que todos los cves asociados a un mismo plugin id tienen una entrada cada uno. (La aplicacion coge el plugin id y saca los CVEs que hay que meter, por tanto, las entradas que contienen esos CVEs en el excel no nos importan porque meten tambien las de Reference Information)

        numero_filas_total = sum(1 for row in self.conf.nessus_reader)

        self.conf.fichero_nessus.seek(0)
        next(self.conf.nessus_reader) #quitamos la cabecera

        indice_fila_plugin_id = utiles.get_indice_columna_nessus(cabeceras_nessus_csv, lista_cabeceras_nessus_conf[0])
        indice_fila_risk = utiles.get_indice_columna_nessus(cabeceras_nessus_csv, lista_cabeceras_nessus_conf[2])

        numero_fila_actual = 1
        for fila in self.conf.nessus_reader:
            vul_id = fila[indice_fila_plugin_id] #guardamos el plugin ID

            if fila[indice_fila_risk] != "None": #si tiene None en la columna de "Risk" es porque es INFO, y, por tanto, pasamos de largo
                if (vul_id != plugin_id_anterior): #para evitar la duplicidad
                    plugin_id_anterior = vul_id

                    fila_insertar = [""] * 26

                    multiples_cves = [] #por si tiene varios CVEs asociados un plugin ID

                    indice_columna_cve = utiles.get_indice_columna_nessus(cabeceras_nessus_csv, lista_cabeceras_nessus_conf[1])
                    if fila[indice_columna_cve] != "": #si tiene CVE, entonces tenemos que hacer una petición http para extraer los CVEs asociados a ese plugin ID, ya que si usamos los del csv nos metería también los CVEs de Reference Information, los cuales no metemos

                        conexion_tenable = True
                        contador_reintentos = 0
                        while(conexion_tenable):
                            try:
                                multiples_cves = self.extraer_cves_de_plugin_id(vul_id)
                                self.progressbar.configure(progress_color="#2CC985")
                                conexion_tenable = False
                                contador_reintentos = 0

                                for cve in multiples_cves:
                                    if cve in vulnerabilidades_vdb:
                                        self.insertar_texto_textbox("VULNERABILIDAD QUE CONTINÚA "+cve+"\n")
                                        logging.warning("vulnerabilidad que continúa: "+cve)
                                        vulnerabilidades_duplicadas.append(cve)
                                    else:
                                        self.anadir_columnas_fijas_fila(fila_insertar, columnas_fijas)
                                        self.anadir_columnas_nessus_fila(fila_insertar, cve, fila, cabeceras_nessus_csv)

                                        conexion_nist = True
                                        contador_reintentos = 0
                                        while (conexion_nist):
                                            try:
                                                time.sleep(2)

                                                respuesta = criticidades.peticion_cve_api_nist(cve, self.conf.configuracion["nist_api_key"])
                                                self.progressbar.configure(progress_color="#2CC985")
                                                cvss2_base_score, cvss3_base_score = criticidades.get_cvss2_cvss3_base_score_api_nist(respuesta) #devuelve una lista con cvss2.0 y cvss3.0 base scores
                                                version_cvss3 = criticidades.get_version_cvss3_api_nist(respuesta) #contiene 3.0, 3.1 o N/A
                                                vector_cvss3 = criticidades.get_vector_ataque_cvss3_api_nist(respuesta)
                                                fecha_publicacion_cve = criticidades.get_fecha_publicacion_cve_api_nist(respuesta)
                                                all_scores = self.get_criticidades_cve(cve, cvss2_base_score, cvss3_base_score, version_cvss3, vector_cvss3, environmental_impact_subscore_modifiers) #cvss2.0, cvss3.0 base, cvss3.1 base, cvss3.0 enviromental, cvss3.1 enviromental, fecha publicacion
                                                all_scores.append(utiles.formatear_fecha(fecha_publicacion_cve)) #añadimos la fecha de publicacion
                                                self.anadir_criticidades_fila(fila_insertar, all_scores)
                                                vdb_final.insertar_fila_vdb_generado(fila_insertar)

                                                conexion_nist = False
                                                contador_reintentos = 0
                                                logging.info("insertado el cve: "+cve+" que viene del plugin id: "+vul_id)
                                                self.insertar_texto_textbox("INSERTADO "+cve+"\n")
                                            except requests.exceptions.ConnectionError:
                                                if contador_reintentos == 0:
                                                    logging.critical("El ordenador ha sido desconectado de internet durante la ejecucion de: "+cve+" asociado al plugin id: "+vul_id)
                                                    self.progressbar.configure(progress_color="red")
                                                    self.insertar_texto_textbox("CONEXIÓN A INTERNET PÉRDIDA\n")
                                                contador_reintentos += 1
                                                self.insertar_texto_textbox("REINTENTO DE CONEXIÓN "+str(contador_reintentos)+"\n")
                                                time.sleep(5)
                                            except Exception:
                                                logging.error("en la peticion API para la vulnerabilidad: "+cve+" que viene del plugin id: "+vul_id)
                                                self.insertar_texto_textbox("ERROR PETICION API PARA "+cve+"\n")
                                                vulnerabilidades_error.append(cve)
                                                conexion_nist = False

                            except requests.exceptions.ConnectionError:
                                if contador_reintentos == 0:
                                    logging.critical("El ordenador ha sido desconectado de internet durante la ejecucion del plugin id: "+vul_id+" el cual contiene CVEs asociados")
                                    self.progressbar.configure(progress_color="red")
                                    self.insertar_texto_textbox("CONEXIÓN A INTERNET PÉRDIDA\n")
                                contador_reintentos += 1
                                self.insertar_texto_textbox("REINTENTO DE CONEXIÓN "+str(contador_reintentos)+"\n")
                                time.sleep(5)
                            except Exception:
                                logging.error("en la peticion HTTP para extraer los CVEs del plugin id: "+vul_id)
                                self.insertar_texto_textbox("ERROR PETICIÓN HTTP PARA PLUGIN ID "+vul_id+"\n")
                                vulnerabilidades_error.append(cve)
                                conexion_tenable = False

                    else: #Si no hay CVE, entonces se pone solo el plugin id
                        if vul_id in vulnerabilidades_vdb:
                                logging.info("vulnerabilidad que continúa: "+vul_id)
                                self.insertar_texto_textbox("VULNERABILIDAD QUE CONTINÚA PLUGIN ID"+vul_id+"\n")
                                vulnerabilidades_duplicadas.append(vul_id)
                        else:
                            self.anadir_columnas_fijas_fila(fila_insertar, columnas_fijas)
                            self.anadir_columnas_nessus_fila(fila_insertar, vul_id, fila, cabeceras_nessus_csv)

                            claves = list(self.conf.configuracion["vdb_generado"]["nessus"].keys())

                            conexion_tenable = True
                            contador_reintentos = 0
                            while conexion_tenable:
                                try:
                                    time.sleep(2)

                                    all_scores = self.get_criticidades_plugin_id(fila, cabeceras_nessus_csv, claves, vul_id, environmental_impact_subscore_modifiers) #cvss2.0, cvss3.0 base, cvss3.1 base, cvss3.0 enviromental, cvss3.1 enviromental
                                    self.progressbar.configure(progress_color="#2CC985")

                                    all_scores.append("") #el metodo anadir_criticalidades_fila añade tambien la fecha de publicacion. Añadimos esto para que no de error. Va a sobreescribir la fecha con "" pero da igual porque justo debajo vamos a añadir la fecha formateada sobreescribiendo este ""
                                    self.anadir_criticidades_fila(fila_insertar, all_scores)

                                    fila_insertar[utiles.get_indice_a_partir_columna(self.conf.configuracion["vdb_generado"]["nessus"][claves[5]])] = utiles.voltear_fecha(fila[utiles.get_indice_columna_nessus(cabeceras_nessus_csv, claves[5])]) #añadimos la fecha del plugin formateada

                                    vdb_final.insertar_fila_vdb_generado(fila_insertar)

                                    conexion_tenable = False
                                    contador_reintentos = 0
                                    logging.info("insertado el plugin id: "+vul_id)
                                    self.insertar_texto_textbox("INSERTADO PLUGIN ID "+vul_id+"\n")

                                except requests.exceptions.ConnectionError:
                                    if contador_reintentos == 0:
                                        logging.critical("El ordenador ha sido desconectado de internet durante la ejecucion del plugin id: "+vul_id)
                                        self.progressbar.configure(progress_color="red")
                                        self.insertar_texto_textbox("CONEXIÓN A INTERNET PÉRDIDA\n")
                                    contador_reintentos += 1
                                    self.insertar_texto_textbox("REINTENTO DE CONEXIÓN "+str(contador_reintentos)+"\n")
                                    time.sleep(5)
                                except Exception:
                                    logging.error("en la peticion HTTP para extraer el vector cvss3.X del plugin id: "+vul_id)
                                    self.insertar_texto_textbox("ERROR PETICION HTTP VECTOR CVSS3.X PLUGIN ID "+vul_id+"\n")
                                    vulnerabilidades_error.append(vul_id)
                                    conexion_tenable = False
            else:
                logging.debug("El plugin id: "+vul_id+" es una entrada de INFO, por tanto, no se inserta")
                self.insertar_texto_textbox("INFO: "+vul_id+"\n")
                self.textbox.see("end")

            self.progressbar.set(numero_fila_actual/(numero_filas_total+2))
            numero_fila_actual += 1

        self.insertar_duplicados_y_errores(vdb_final, vulnerabilidades_error, self.vulnerabilidades_rejected, vulnerabilidades_duplicadas, vulnerabilidades_vdb)

        self.progressbar.set(numero_fila_actual/(numero_filas_total))
        logging.log(self.conf.vdbauto_nivel_log, "######## FINALIZADA ITERACION DE "+utiles.get_nombre_fichero(ruta_nessus)+" ########")
        self.insertar_texto_textbox("###### FINALIZADA ITERACION DE "+utiles.get_nombre_fichero(ruta_nessus)+" ######")

    def insertar_duplicados_y_errores(self, vdb_final, vulnerabilidades_error, vulnerabilidades_rejected, vulnerabilidades_duplicadas, vulnerabilidades_antiguas):
        for i in range(2, len(vulnerabilidades_error)+2):
            vdb_final.insertar_celda(vdb_final.hoja2, i, 1, vulnerabilidades_error[i-2])

        for i in range(2, len(vulnerabilidades_rejected)+2):
            vdb_final.insertar_celda(vdb_final.hoja2, i, 3, vulnerabilidades_rejected[i-2])

        vulnerabilidades_pueden_cerrar = [item for item in vulnerabilidades_antiguas if item not in vulnerabilidades_duplicadas]
        for i in range(2, len(vulnerabilidades_pueden_cerrar)+2):
            vdb_final.insertar_celda(vdb_final.hoja2, i, 5, vulnerabilidades_pueden_cerrar[i-2])

        for i in range(2, len(vulnerabilidades_duplicadas)+2):
            vdb_final.insertar_celda(vdb_final.hoja2, i, 7, vulnerabilidades_duplicadas[i-2])

        vdb_final.workbook.save(vdb_final.ruta)

    def get_criticidades_cve(self, cve, cvss2_base_score, cvss3_base_score, version_cvss3, vector_cvss3, environmental_impact_subscore_modifiers):
        """Devuelve una lista con cvss2.0, cvss3.0 base, cvss3.1 base, cvss3.0 enviromental y cvss3.1 enviromental"""
        all_criticidades = []

        if version_cvss3 == "N/A": #solo hay cvss2.0
            if cvss2_base_score == "N/A": #ES UN CVE REJECTED (no tiene ni cvss3.x ni cvss2.0)
                all_criticidades.append("N/A")
                logging.warning(cve+" es un posible CVE rejected")
                self.insertar_texto_textbox("POSIBLE CVE REJECTED "+cve+"\n")
                self.vulnerabilidades_rejected.append(cve)
            else: all_criticidades.append(str(float(cvss2_base_score))) #lo pasamos a float para convertir lo 5 en 5.0 y luego a string porque sino si tienes el excel en español al insertarlo te convierte los . en , porque detecta que son numeros. Al ser un String no cambia nada
            for i in range(4):
                all_criticidades.append("N/A")

        elif version_cvss3 == "3.0":
            if cvss2_base_score == "N/A": all_criticidades.append("N/A") #Puede que tenga cvss3.x pero no tenga cvss2.0
            else: all_criticidades.append(str(float(cvss2_base_score))) #queremos que queden con 5.2. Reemplazamos las comas por puntos y convertimos a float para que pase los 5 a 5.0
            all_criticidades.append(str(float(cvss3_base_score))) #cvss3.0
            all_criticidades.append("N/A") #ponemos N/A al cvss3.1 base
            all_criticidades.append(str(float(criticidades.get_recalculo_cvssv3(vector_cvss3, environmental_impact_subscore_modifiers[0], environmental_impact_subscore_modifiers[1], environmental_impact_subscore_modifiers[2]))))
            all_criticidades.append("N/A") #ponemos N/A al cvss3.1 enviromental

        else: #es cvss3.1
            if cvss2_base_score == "N/A": all_criticidades.append("N/A") #Puede que tenga cvss3.x pero no tenga cvss2.0
            else: all_criticidades.append(str(float(cvss2_base_score)))
            all_criticidades.append("N/A") #ponemos N/A al cvss3.0 base
            all_criticidades.append(str(float(cvss3_base_score)))
            all_criticidades.append("N/A") #ponemos N/A al cvss3.0 enviromental
            all_criticidades.append(str(float(criticidades.get_recalculo_cvssv3(vector_cvss3, environmental_impact_subscore_modifiers[0], environmental_impact_subscore_modifiers[1], environmental_impact_subscore_modifiers[2]))))

        return all_criticidades

    def get_criticidades_plugin_id(self, fila, cabeceras_nessus_csv, claves, vul_id, environmental_impact_subscore_modifiers):
        """Devuelve una lista con cvss2.0, cvss3.0 base, cvss3.1 base, cvss3.0 enviromental y cvss3.1 enviromental"""
        all_scores = [] #cvss2.0, cvss3.0 base, cvss3.1 base, cvss3.0 enviromental, cvss3.1 enviromental, fecha_publicacion

        cvss2 = fila[utiles.get_indice_columna_nessus(cabeceras_nessus_csv, claves[3])]
        if (cvss2 == ""): all_scores.append("N/A") #Puede pasar que tenga un Risk Factor de High pero no tenga ni criticidad 2.0 ni 3.0.
        else: all_scores.append(str(float(cvss2))) #añadimos el CVSS2.0 basescore

        cvss3 = fila[utiles.get_indice_columna_nessus(cabeceras_nessus_csv, claves[4])] #CVSS3.0 basescore.

        if (cvss3 == ""): #no tiene cvss3.x
                all_scores.append("N/A") #3.0 base
                all_scores.append("N/A") #3.1 base
                all_scores.append("N/A") #3.0 env
                all_scores.append("N/A") #3.1 env
        else:
            vector_cvss3 = self.extraer_vector_cvss3_plugin_id(vul_id)
            cvss3_recalculo = cvss3_recalculo = criticidades.get_recalculo_cvssv3(vector_cvss3, environmental_impact_subscore_modifiers[0], environmental_impact_subscore_modifiers[1], environmental_impact_subscore_modifiers[2])

            if vector_cvss3[:8] == "CVSS:3.0":
                all_scores.append(str(float(cvss3))) #3.0 base
                all_scores.append("N/A") #3.1 base
                all_scores.append(str(float(cvss3_recalculo))) #3.0 env
                all_scores.append("N/A") #3.1 env
            elif vector_cvss3[:8] == "CVSS:3.1":
                all_scores.append("N/A") #3.0 base
                all_scores.append(str(float(cvss3))) #3.1 base
                all_scores.append("N/A") #3.0 env
                all_scores.append(str(float(cvss3_recalculo))) #3.1 env
            else:
                logging.error("Error en la iteracion del plugin: "+vul_id)

        return all_scores

    def extraer_vulnerabilidades_vdb(self, vdb_hoja, componente):
        """Devuelve las vulnerabilidades abiertas de un componente dado, iterando un VDB"""
        vulnerabilidades_vdb_componente = []

        indice_columna_componentes = utiles.get_indice_columna_vdb(vdb_hoja, "columna_componente")
        indice_columna_id_vulnerabilidad = utiles.get_indice_columna_vdb(vdb_hoja, "columna_vulnerabilidad")
        indice_columna_status = utiles.get_indice_columna_vdb(vdb_hoja, "estado")

        for fila in range(3, vdb_hoja.max_row+1):
            if vdb_hoja.cell(row=fila, column=indice_columna_componentes).value == componente and vdb_hoja.cell(row=fila, column=indice_columna_status).value == "Open":
                vulnerabilidades_vdb_componente.append(vdb_hoja.cell(row=fila, column=indice_columna_id_vulnerabilidad).value)

        return vulnerabilidades_vdb_componente

    def extraer_cves_de_plugin_id(self, plugin_id):
        """Extrae los CVEs asociados a un plugin id en la parte de CVSS2.0 y CVSS3.0"""
        url = 'https://www.tenable.com/plugins/nessus/'+plugin_id
        r = requests.get(url)

        htmlRespuesta = r.content.decode("utf-8")

        indiceInicial = htmlRespuesta.index("Risk Information")
        indiceFinal = htmlRespuesta.index("Vulnerability Information")

        htmlFormateado = htmlRespuesta[indiceInicial:indiceFinal]

        cveRegex = re.compile(r'CVE-\d*-\d*<')
        cvesBuenos = cveRegex.findall(htmlFormateado)

        for i in range(len(cvesBuenos)): #Elimina el < en los strings con cves extraidos del html
            cvesBuenos[i] = cvesBuenos[i][:len(cvesBuenos[i])-1]

        return cvesBuenos

    def extraer_vector_cvss3_plugin_id(self, plugin_id):
        """Extrae el vector de ataque cvss3 de un plugin id"""
        url = 'https://www.tenable.com/plugins/nessus/'+plugin_id
        r = requests.get(url)

        htmlRespuesta = r.content.decode("utf-8")

        indiceInicial = htmlRespuesta.index("CVSS v3")
        indiceFinal = htmlRespuesta.index("Vulnerability Information")

        htmlFormateado = htmlRespuesta[indiceInicial:indiceFinal]

        indice_inicio_vector = htmlFormateado.rfind("CVSS:3.")
        vector = htmlFormateado[indice_inicio_vector:]
        indice_fin_vector = vector.find("<")
        vector = vector[:indice_fin_vector]

        return vector

    def anadir_columnas_fijas_fila(self, fila_insertar, columnas_fijas):
        """Añade las columnas fijas a la fila que se va a insertar en el VBD generado"""
        for clave, valor in columnas_fijas.items():
            fila_insertar[utiles.get_indice_a_partir_columna(clave)] = valor

    def anadir_columnas_nessus_fila(self, fila_insertar, id_vulnerabilidad, fila_nessus, cabeceras_nessus_csv):
        """Añade las columnas del reporte de Nessus a la fila que se va a insertar en el VBD generado (excepto el CVSS2.0 basescore ya que para los CVE se hace con la petición a la API y para los plugin en "iterar_nessus_csv"). Recibe la fila y el Plugin ID/CVE"""
        i=0
        for clave, valor in self.conf.configuracion["vdb_generado"]["nessus"].items():
            if valor != "n/a":
                if i==1: #se mete el plugin/cve
                    fila_insertar[utiles.get_indice_a_partir_columna(valor)] = id_vulnerabilidad
                else:
                    fila_insertar[utiles.get_indice_a_partir_columna(valor)] = fila_nessus[utiles.get_indice_columna_nessus(cabeceras_nessus_csv, clave)]
            i+=1

    def anadir_criticidades_fila(self, fila_insertar, cvs):
        """Recibe la fila donde se insertan las criticidades y una lista con las mismas en el siguiente orden: cvss2.0, cvss3.0 base, cvss3.1 base, cvss3.0 enviromental, cvss3.1 enviromental y fecha de publicacion"""
        i = 0
        for clave, valor in self.conf.configuracion["vdb_generado"]["columnas_criticidades"].items():
            fila_insertar[utiles.get_indice_a_partir_columna(valor)] = cvs[i]
            i+=1


import torch
import torch.nn as nn
from torchvision import models
from torch.nn import functional as F

def get_backbone(name, pretrained=True):

    """ Loading backbone, defining names for skip-connections and encoder output. """

    # loading backbone model
    if name == 'resnet18':
        backbone = models.resnet18(pretrained=pretrained)
    elif name == 'resnet34':
        backbone = models.resnet34(pretrained=pretrained)
    elif name == 'resnet50':
        backbone = models.resnet50(pretrained=pretrained)
    elif name == 'resnet101':
        backbone = models.resnet101(pretrained=pretrained)
    elif name == 'resnet152':
        backbone = models.resnet152(pretrained=pretrained)
    elif name == 'vgg16':
        backbone = models.vgg16_bn(pretrained=pretrained).features
    elif name == 'vgg19':
        backbone = models.vgg19_bn(pretrained=pretrained).features
    elif name == 'densenet121':
        backbone = models.densenet121(pretrained=True).features
    elif name == 'densenet161':
        backbone = models.densenet161(pretrained=True).features
    elif name == 'densenet169':
        backbone = models.densenet169(pretrained=True).features
    elif name == 'densenet201':
        backbone = models.densenet201(pretrained=True).features
    elif name == 'unet_encoder':
        from unet_backbone import UnetEncoder
        backbone = UnetEncoder(3)
    else:
        raise NotImplemented('{} backbone model is not implemented so far.'.format(name))

    # specifying skip feature and output names
    if name.startswith('resnet'):
        feature_names = [None, 'relu', 'layer1', 'layer2', 'layer3']
        backbone_output = 'layer4'
    elif name == 'vgg16':
        feature_names = ['5', '12', '22', '32', '42']
        backbone_output = '43'
    elif name == 'vgg19':
        feature_names = ['5', '12', '25', '38', '51']
        backbone_output = '52'
    elif name.startswith('densenet'):
        feature_names = [None, 'relu0', 'denseblock1', 'denseblock2', 'denseblock3']
        backbone_output = 'denseblock4'
    elif name == 'unet_encoder':
        feature_names = ['module1', 'module2', 'module3', 'module4']
        backbone_output = 'module5'
    else:
        raise NotImplemented('{} backbone model is not implemented so far.'.format(name))

    print(feature_names)
    return backbone, feature_names, backbone_output

""" U-Net (https://arxiv.org/pdf/1505.04597.pdf) implementation with pre-trained torchvision backbones."""
class Encoder(nn.Module):

    """
    Convolutional batch norm block with relu activation (main block used in the encoding steps)
    """
    def __init__(self,
                 backbone_name='resnet50',
                 pretrained=True,
                 encoder_freeze=False,
                 shortcut_features='default'):
        super(Encoder,self).__init__()
        self.backbone_name = backbone_name
        self.backbone, self.shortcut_features, self.bb_out_name = get_backbone(backbone_name, pretrained=pretrained)
        if shortcut_features != 'default':
            self.shortcut_features = shortcut_features
            print('shortcut_features:')
            print(shortcut_features)

        if encoder_freeze:
            self.freeze_encoder()

    def freeze_encoder(self):

        """ Freezing encoder parameters, the newly initialized decoder parameters are remaining trainable. """

        for param in self.backbone.parameters():
            param.requires_grad = False

    def forward(self, x):

        """ Forward propagation in backbone encoder network.  """

        features = {None: None} if None in self.shortcut_features else dict()
        for name, child in self.backbone.named_children():
            x = child(x)
            if name in self.shortcut_features:
                features[name] = x
            if name == self.bb_out_name:
                break

        return x, features

class UpsampleBlock(nn.Module):

    def __init__(self, ch_in, ch_out=None, skip_in=0, use_bn=True, parametric=False):
        super(UpsampleBlock, self).__init__()

        self.parametric = parametric
        ch_out = ch_in/2 if ch_out is None else ch_out

        # first convolution: either transposed conv, or conv following the skip connection
        if parametric:
            # versions: kernel=4 padding=1, kernel=2 padding=0
            self.up = nn.ConvTranspose2d(in_channels=ch_in, out_channels=ch_out, kernel_size=(4, 4),
                                         stride=2, padding=1, output_padding=0, bias=(not use_bn))
            self.bn1 = nn.BatchNorm2d(ch_out) if use_bn else None
        else:
            self.up = None
            ch_in = ch_in + skip_in
            self.conv1 = nn.Conv2d(in_channels=ch_in, out_channels=ch_out, kernel_size=(3, 3),
                                   stride=1, padding=1, bias=(not use_bn))
            self.bn1 = nn.BatchNorm2d(ch_out) if use_bn else None

        self.relu = nn.ReLU(inplace=True)

        # second convolution
        conv2_in = ch_out if not parametric else ch_out + skip_in
        self.conv2 = nn.Conv2d(in_channels=conv2_in, out_channels=ch_out, kernel_size=(3, 3),
                               stride=1, padding=1, bias=(not use_bn))
        self.bn2 = nn.BatchNorm2d(ch_out) if use_bn else None

    def forward(self, x, skip_connection=None):

        x = self.up(x) if self.parametric else F.interpolate(x, size=None, scale_factor=2, mode='bilinear',
                                                             align_corners=None)
        if self.parametric:
            x = self.bn1(x) if self.bn1 is not None else x
            x = self.relu(x)

        if skip_connection is not None:
            x = torch.cat([x, skip_connection], dim=1)

        if not self.parametric:
            x = self.conv1(x)
            x = self.bn1(x) if self.bn1 is not None else x
            x = self.relu(x)
        x = self.conv2(x)
        x = self.bn2(x) if self.bn2 is not None else x
        x = self.relu(x)

        return x

class Decoder(nn.Module):

    def __init__(self,
                 backbone_name='resnet50',
                 pretrained=True,
                 classes=21,
                 decoder_filters=(256, 128, 64, 32, 16),
                 parametric_upsampling=True,
                 shortcut_features='default',
                 decoder_use_batchnorm=True):
        super(Decoder, self).__init__()
        self.backbone_name = backbone_name
        self.backbone, self.shortcut_features, self.bb_out_name = get_backbone(backbone_name, pretrained=pretrained)
        shortcut_chs, bb_out_chs = self.infer_skip_channels()
        # build decoder part
        self.upsample_blocks = nn.ModuleList()
        decoder_filters = decoder_filters[:len(self.shortcut_features)]  # avoiding having more blocks than skip connections
        decoder_filters_in = [bb_out_chs] + list(decoder_filters[:-1])
        num_blocks = len(self.shortcut_features)

        #print(shortcut_chs)
        #print(decoder_filters_in)
        #print(decoder_filters)
        for i, [filters_in, filters_out] in enumerate(zip(decoder_filters_in, decoder_filters)):
            print('upsample_blocks[{}] in: {}   out: {}'.format(i, filters_in, filters_out))

            self.upsample_blocks.append(UpsampleBlock(filters_in, filters_out,
                                                      skip_in=shortcut_chs[num_blocks-i-1],
                                                      parametric=parametric_upsampling,
                                                      use_bn=decoder_use_batchnorm))

        self.final_conv = nn.Conv2d(decoder_filters[-1], classes, kernel_size=(1, 1))

    def infer_skip_channels(self):

        """ Getting the number of channels at skip connections and at the output of the encoder. """

        x = torch.zeros(1, 3, 224, 224)
        has_fullres_features = self.backbone_name.startswith('vgg') or self.backbone_name == 'unet_encoder'
        channels = [] if has_fullres_features else [0]  # only VGG has features at full resolution

        # forward run in backbone to count channels (dirty solution but works for *any* Module)
        for name, child in self.backbone.named_children():
            x = child(x)
            if name in self.shortcut_features:
                channels.append(x.shape[1])
            if name == self.bb_out_name:
                out_channels = x.shape[1]
                break
        return channels, out_channels

    def forward(self, x, features):

        """ Forward propagation in U-Net. """

        for skip_name, upsample_block in zip(self.shortcut_features[::-1], self.upsample_blocks):
            skip_features = features[skip_name]
            x = upsample_block(x, skip_features)

        x = self.final_conv(x)
        return x

class UNaah(nn.Module):

    def __init__(self,
                 backbone_name='resnet50',
                 pretrained=True,
                 encoder_freeze=False,
                 shortcut_features='default',
                 classes=21,
                 decoder_filter_block=64,
                 parametric_upsampling=True,
                 decoder_use_batchnorm=True):
        super(UNaah,self).__init__()
        decoder_filters = (decoder_filter_block*16,decoder_filter_block*8,decoder_filter_block*4,
        decoder_filter_block*2,decoder_filter_block)
        self.encoder = Encoder(backbone_name=backbone_name,
                 pretrained=pretrained,
                 encoder_freeze=encoder_freeze,
                 shortcut_features=shortcut_features)
        self.decoder1 = Decoder(backbone_name=backbone_name,
                 pretrained=pretrained,
                 classes=classes,
                 decoder_filters=decoder_filters,
                 parametric_upsampling=parametric_upsampling,
                 shortcut_features=shortcut_features,
                 decoder_use_batchnorm=decoder_use_batchnorm)
        self.decoder2 = Decoder(backbone_name=backbone_name,
                 pretrained=pretrained,
                 classes=classes,
                 decoder_filters=decoder_filters,
                 parametric_upsampling=parametric_upsampling,
                 shortcut_features=shortcut_features,
                 decoder_use_batchnorm=decoder_use_batchnorm)

    def forward(self, x):
        x, features = self.encoder(x)
        out1 = self.decoder1(x, features)
        out2 = self.decoder2(x, features)
        return out1, out2

if __name__ == "__main__":

    # simple test run
    net = UNaah(backbone_name='resnet50')

    criterion = nn.MSELoss()
    optimizer = torch.optim.Adam(net.parameters())
    print('Network initialized. Running a test batch.')
    for _ in range(1):
        with torch.set_grad_enabled(True):
            batch = torch.empty(1, 3, 224, 224).normal_()
            targets = torch.empty(1, 21, 224, 224).normal_()

            o = net(batch)
            loss1 = criterion(o, targets)
            loss1.backward()
            optimizer.step()
        #print(out.shape)

    print('fasza.')


import pandas as pd
import numpy as np
from ACME.utils import clean_list
from ACME.ACME_function import computeACME, build_feature_exploration_table, predictACME
from ACME.ACME_plot import ACME_summary_plot, feature_exploration_plot, ACME_barplot_multicalss
from ACME.ACME_anomaly_detection import build_anomaly_detection_feature_exploration_table, computeAnomalyDetectionImportance
import plotly.express as px
import plotly.graph_objects as go

class ACME():

    def __init__(self, model, target, features=[], cat_features=[], K=50, task='regression', score_function=None):
        '''
        Initialization

        Params:
        ------
        - model: object
            the object of the model
        - target: str
            name of the target_feature
        - features: [str]
            list of the features name in SAME ORDER as the the model input
        - cat_features: [str]
            categorical features name
        - K: int
            number of quantiles to use
        - task: str
            type of model task {'regression','reg','r', 'c','class','classification', 'ad','anomaly detection'}
        - score_function: fun(model, x)
            a function that has in input the model and the input data to realize the prediction. It must return a numeric score

        '''

        self._model = model
        self._target = target
        self._features = features
        self._numeric_features = list(np.setdiff1d(features, cat_features))
        self._cat_features = cat_features
        self._task = task
        self._global_explain = {}
        self._local_explain = {}
        self._K = K
        self._score_function = score_function
        self._class_to_analyze = None
        self._label_class = None

    def explain(self, dataframe, robust = False, label_class = None):
        '''
        Fit the acme explainability.

        Params:
        -------
        - dataframe : pd.DataFrame
            input dataframe
        - robust : bool (default False)
            if True use only quantile from 0.05 and 0.95, if False use from 0 to 1
        - label_class : str,int (default None)
            if task is classification, specify the class of interest
        '''

        # if cat features and numeric features are not specified then all the columns of the dataset (not the target) are used as numeric feature
        if self._numeric_features == [] and self._cat_features == []:
            self._numeric_features = dataframe.drop(columns=self._target).columns.to_list()

        # create the dataframe of numeric feature
        if self._numeric_features == []:
            if self._cat_features == [] :
                self._numeric_features = dataframe.drop(columns= [self._target]).columns.to_list()
            else:
                self._numeric_features = dataframe.drop(columns= [self._target] + self._cat_features ).columns.to_list()

        self._numeric_df = dataframe[ self._numeric_features ].copy()

        # create the dataframe for cat feature
        self._cat_df = None
        if not self._cat_features == []:
           self._cat_df = dataframe[ self._cat_features ].copy()

        # we save the features used by the model in the original order (necessary to correctly compute the predictiom)
        self._features = clean_list( dataframe.columns.to_list(), self._numeric_features + self._cat_features)

        # if the label class is given, find the corrispective position in the model class map
        # else auto set as label class the first class
        # N.B. : label_class is the class name, class_to_analyze is the label_class corrispective number in the model class map
        if self._task  in ['c','class','classification']:
            class_map = np.array(self._model.classes_)
            if label_class is not None:
                try:
                    class_to_analyze = np.where(class_map == label_class)[0][0]
                except:
                    class_to_analyze = np.where(class_map == str(label_class))[0][0]
                self._class_to_analyze = class_to_analyze
            if label_class is None:
                label_class = list(class_map)

            self._label_class = label_class

        # compute local acme for regression or classifiction
        if self._task in ['r','reg','regression'] or self._task in ['ad','anomaly detection']:
            feature_table, importance_table, baseline_pred, baseline, perc_functions = computeACME( model = self._model, task = self._task,
                                                                                features = self._features,
                                                                                numeric_df = self._numeric_df, cat_df = self._cat_df,
                                                                                score_function = self._score_function,
                                                                                K = self._K, robust = robust )
        if self._task  in ['c','class','classification']:

            class_stack_importance = []
            class_stack_feature_table = []
            label_dict = {}

            if type(label_class) is list:
                label_list = range(0,len(label_class))
            else:
                label_list = [class_to_analyze]

            # explore every label

            for lab in label_list:
                label_dict[lab] = class_map[lab]
                feature_table, importance_table, baseline_pred, baseline, perc_functions = computeACME( model = self._model, task = self._task,
                                                                                features = self._features, class_to_analyze = lab,
                                                                                numeric_df = self._numeric_df, cat_df = self._cat_df,
                                                                                score_function = self._score_function,
                                                                                K=self._K, robust = robust )

                # rename the columns in case of multilabel
                if len(label_list) > 1:
                    importance_table.rename( columns = { 'importance' : 'importance_class_' + str(label_class[lab]) }, inplace=True )

                # save the results
                class_stack_importance.append(importance_table)
                class_stack_feature_table.append(feature_table)

            # concat the results
            feature_table = pd.concat(class_stack_feature_table)
            feature_table['class'] = feature_table['class'].map(label_dict)

            class_stack_importance = pd.concat(class_stack_importance,axis=1)
            #if multilabel sum the importance of each feature in each
            if len(label_list) > 1:
                class_stack_importance['importance'] = class_stack_importance.sum(axis=1).values

            class_stack_importance.sort_values('importance', ascending = False, inplace=True)
            importance_table = class_stack_importance

        # create the outputs
        self._global_explain = {'meta':feature_table.copy(),
                                'feature_importance':importance_table.sort_values('importance', ascending = False).copy(),
                                'baseline_pred':baseline_pred,
                                'baseline':baseline,
                                }
        #self._global_meta = feature_table.copy()
        #self._global_feature_importance = importance_table.sort_values('importance', ascending = False).copy()
        #self._global_baseline_pred = baseline_pred
        #self._global_baseline = baseline
        self._perc_functions = perc_functions

        return self

    def explain_local(self, series, label_class = None):
        '''
        Params:
        -------
        - series : pd.Series
            observation on which explain the prediction
        - label_class : str,int (default None)
            if task is classification, specify the class of interest

        Returns:
        --------
        - pd.DataFrame
        '''

        # if the label class is given, find the corrispective position in the model class map
        # else auto set as label class the first class
        # N.B. : label_class is the class name, class_to_analyze is the label_class corrispective number in the model class map

        if len(self._global_explain.keys())==0:
            raise InterruptedError("You must first fit the acme explaination model with the 'explain' command on the same data used to train the model")

        if self._task  in ['c','class','classification']:
            class_map = np.array(self._model.classes_)
            if label_class is not None:
                try:
                    class_to_analyze = np.where(class_map == label_class)[0][0]
                except:
                    class_to_analyze = np.where(class_map == str(label_class))[0][0]
            else:
                class_to_analyze = 0
                label_class = class_map[class_to_analyze]
                print('WARNING: in local interpretation, the label_class must be specified and not None. To default it\'s setted to class:' + str(class_map[0]))

            # save the class to analize and the label
            self._class_to_analyze = class_to_analyze
            self._label_class = label_class

        local_table, local_importance_table, local_baseline_pred, local_baseline = predictACME(model=self._model,
                    series=series,
                    features=self._features,
                    meta_table=self._global_explain['meta'],
                    percentile_functions=self._perc_functions,
                    task=self._task,
                    score_function=self._score_function,
                    class_to_analyze=self._class_to_analyze)

        if self._task in ['c','class','classification']:
            local_table['class'] = local_table['class'].map({class_to_analyze : class_map[class_to_analyze]})

        self._local_explain = {'meta':local_table.copy(),
                                'feature_importance':local_importance_table.sort_values('importance', ascending = False).copy(),
                                'baseline_pred':local_baseline_pred,
                                'baseline':local_baseline,
                                'local_name':series.name}  # save the index of the local observation, it's the name of the series (corrisponding to the dataframe index)

        #self._local_meta = local_table
        #self._local_baseline = local_baseline
        #self._local_importance_table = local_importance_table
        #self._local_baseline_pred = local_baseline_pred

        return self

    def feature_importance(self, local=False, weights = {}):
        '''
        Returns the feature importance calculated by AcME.
        In case of Anomaly Detection task, it provides ad hoc explanation for anomaly detection, studied for local interpretability.
        The score will show what features can altered the prediction from normal to anomalies and viceversa.

        Params:
        ------
        - local : bool
            if true and task is AD, it return the local AD version of feature importance
        - weights : dict
            Dictionary with the importance for each element. Sum must be 1
            * ratio : float
                importance of local score position
            * distance : float
                importance of inter-quantile distance necessary to change
            * change : float
                importance of the possibility to change prediction
            * delta : float
                importance of the score delta

        Return :
        -------
        - pd.DataFrame
        '''

        # if the task is anomaly detection and we have already fitted a local acme
        # calculate the importance for the selected row anomaly detection task
        if self._task in ['ad','anomaly detection'] and local:

            local_table = self._local_explain['meta'].drop(columns='size').copy()
            importance_df = computeAnomalyDetectionImportance(local_table, weights = weights)

            return importance_df

        # if local then we return the local importance
        elif local:
            return self._local_explain['feature_importance']

        # else simply return the importance calculated by acme for global explain
        else:
            return self._global_explain['feature_importance']

    def feature_exploration(self, feature, local=False, plot=False):
        '''
        Generate anomaly detection feature exploration table or a plot for local observation that,
        choosen a specific feature, shows how the anomaly score can change beacuse of the feature.

        Params:
        -------
        - feature : str
            selected feature's name
        - plot : bool
            if true returns the plot, else returs the table

        Returns:
        --------
        - feature_table : pd.DataFrame
        '''

        # extract the desired table
        if local:
            table = self._local_explain['meta']
        else:
            table = self._global_explain['meta']

        # build the correct feature exploration table
        if self._task in ['ad','anomaly detection']:
            feature_table = build_anomaly_detection_feature_exploration_table(table, feature)
        else:
            feature_table = build_feature_exploration_table(table, feature)

        # if plot return plot, else return the table
        if plot:
            fig = feature_exploration_plot(feature_table, feature, self._task)
            return fig
        else:
            return feature_table

    def summary_plot(self, local=False, weights = {}):
        '''
        Generate the recap plot

        Params:
        -------
        - local : bool
            if local or global plot
        - weights : dict
            Dictionary with the importance for each element used in AD. Sum must be 1
                * ratio : float
                    importance of local score position
                * distance : float
                    importance of inter-quantile distance necessary to change
                * change : float
                    importance of the possibility to change prediction
                * delta : float
                    importance of the score delta

        Returns:
        --------
        - plotly figure
        '''

        # if desired explainability is global, task is classification and there are multi label: produce the bar plot
        if self._task in ['c','class','classification'] and type(self._label_class) is list and not local:
            fig = ACME_barplot_multicalss(self._global_explain['feature_importance'], self._label_class)

        # generate the quantile/feature/effect plot
        else:
            meta = dict()
            meta['task'] = self._task
            meta['local'] = False

            if self._task  in ['c','class','classification']:
                meta['label_class'] = self._label_class
                meta['local'] = False

            if local:
                table = self._local_explain['meta']
                meta['local'] = True
                meta['index'] = self._local_explain['local_name']
                meta['baseline'] = self._local_explain['baseline_pred']

                if self._task in ['ad','anomaly detection']:
                    local_table = self._local_explain['meta'].drop(columns='size').copy()
                    importance_df = computeAnomalyDetectionImportance(local_table, weights = weights)
                    table = table.drop(columns=['importance'])
                    table = table.merge(importance_df[['importance']],left_index=True,right_index=True)

            else:
                table = self._global_explain['meta']
                meta['local'] = False

            # prepare for the plotting
            plot_df = table.sort_values(['importance','quantile']).reset_index().rename(columns={'index':'feature'}).copy()

            # if local set the refering x to the local values observation
            # for the global set to 0
            if local:
                meta['x'] = table['baseline_prediction'].values[0]
            else:
                meta['x'] = 0

            # set the top and the bottom of the y-axis (first and last feature)
            meta['y_bottom'] = plot_df['feature'].values[0]
            meta['y_top'] = plot_df['feature'].values[len(plot_df)-1]

            # generate the plot
            fig = ACME_summary_plot(plot_df, meta)

        return fig

    def bar_plot(self, local=False):
        '''
        Feature importance plot
        '''
        if self._task in ['r', 'reg', 'regression']:
            title = 'Barplot of feature importance: regression'
        elif self._task in ['ad','anomaly detection']:
            title = 'Barplot of feature importance: anomaly detection'
        else:
            title = 'Barplot of feature importance: classification'

        if local:
            table = self._local_explain['feature_importance']
            title = 'Local importance observation ID: ' + str(self._local_explain['local_name']) + '.
'+title
        else:
            table = self._global_explain['feature_importance']

        table = round(table.reset_index().sort_values('importance').rename(columns={'index':'feature'}),3)
        fig = px.bar(table,
                    x='importance',
                    y='feature',
                    orientation='h',
                    title = title)

        fig.update_traces(hovertemplate = 'Feature:%{y}
Importance:%{x}')
        return fig.update_layout( title={ 'y':0.9, 'x':0.5, 'xanchor': 'center', 'yanchor': 'top'} )

    def summary_table(self, local=False):
        '''
        Expose the global or local summary table
        '''
        if local:
            return self._local_explain['meta'].drop(columns='size')
        else:
            return self._global_explain['meta'].drop(columns='size')

    def baseline_values(self, local=False):
        '''
        Expose the baseline vector used for AcME
        '''
        if local:
            return self._local_explain['baseline']
        else:
            return self._global_explain['baseline']

    def metadata(self):

        return {'model':self._model,
        'target':self._target,
        'features':self._features,
        'numeric_features':self._numeric_features,
        'cat_features':self._cat_features,
        'task':self._task,
        'global_explain':self._global_explain,
        'local_explain':self._local_explain,
        'K':self._K,
        'score_function':self._score_function}


from tkinter import *
import webbrowser
import backend
from pygame import mixer
import pygame

class Frontend(object):
    def __init__(self,window):
        self.window = window
        self.bk = backend.Backend()
        myfont = ('New Times Roman', 18)

        #Create Labels
        self.lbl = Label(master=window, text="MUSIC DATABASE", font='Helvetica 18 bold')
        self.lbl.grid(row=0, column=2)

        self.lbl = Label(master=window, text="Title", font= myfont)
        self.lbl.grid(row=1, column=0)

        self.lbl2 = Label(master=window, text="Singer", font=myfont)
        self.lbl2.grid(row=1, column=2)

        self.lb3 = Label(master=window, text="Year", font=myfont)
        self.lb3.grid(row=2, column=0)

        #Create text label
        self.name_text =StringVar()
        self.et1 = Entry(master = window, textvariable= self.name_text, font =myfont)
        self.et1.grid(row=1, column =1)

        self.singer_text = StringVar()
        self.et2 = Entry(master=window, textvariable=self.singer_text, font=myfont)
        self.et2.grid(row=1, column=3)

        self.date_text = StringVar()
        self.et2 = Entry(master=window, textvariable=self.date_text, font=myfont)
        self.et2.grid(row=2, column=1)

        #Create Listbox
        self.listbox = Listbox(master=window, height=10, width=40, font=("Times New Roman", 15))
        self.listbox.grid(row=5, column=0, rowspan=8, columnspan=2)

        self.listbox.bind('<<ListboxSelect>>', self.get_row)

        self.scroll = Scrollbar(master=window)
        self.scroll.grid(row=5, column=2, rowspan=8, sticky="nsw")

        self.scroll.configure(command=self.listbox.yview)

        #Create buttons
        self.bt1 = Button(master=window, text="View all", width=20, font=myfont, command=self.view_command)
        self.bt1.grid(row=4, column=3)

        self.bt2 = Button(master=window, text="Add", width=20, font=myfont, command=self.add_command)
        self.bt2.grid(row=5, column=3)

        self.bt3 = Button(master=window, text="Delete", width=20, font=myfont, command= self.delete_command)
        self.bt3.grid(row=6, column=3)

        self.bt4 = Button(master=window, text="Search in Youtube", width=20, font=myfont, command = self.openURL)
        self.bt4.grid(row=7, column=3)

        self.bt5 = Button(master=window, text="Download Song", width=20, font=myfont, command=self.download)
        self.bt5.grid(row=8, column=3)

        self.bt1 = Button(master=window, text="Play from computer", width=20, font=myfont, command=self.playextension)
        self.bt1.grid(row=9, column=3)

        self.bt5 = Button(master=window, text="Close", width=20, font=myfont, command=self.close)
        self.bt5.grid(row=10, column=3)

        #Add functions
    def view_command(self):
        self.listbox.delete(0, END)
        rows = self.bk.view_all()
        for line in rows:
            self.listbox.insert(END, line)

    def add_command(self):
        self.listbox.delete(0, END)
        title = self.name_text.get()
        singer = self.singer_text.get()
        year = self.date_text.get()
        self.bk.add(title=title, singer=singer, year=year)
        self.listbox.insert(END, (title, singer, year))

    def get_row(self, event=None):
        index = self.listbox.curselection()[0]
        line = self.listbox.get(index)
        self.name_text.set(line[1])
        self.singer_text.set(line[2])
        self.date_text.set(line[3])
        return line

    def delete_command(self):
        line = self.get_row()
        self.bk.delete(line[0])
        self.view_command()

    def close(self):
        exit(0)

    def openURL(self):
        song = self.name_text.get()
        artist = self.singer_text.get()
        youtube_search= ('https://www.youtube.com/results?search_query=' + song + " " + artist)
        webbrowser.open(youtube_search)

    def download(self):
        song = self.name_text.get()
        artist = self.singer_text.get()
        download_search= ('https://freedsound.io/es/?q=' + song + " " + artist)
        webbrowser.open(download_search)

    def playextension(self):
        extension = self.name_text.get()
        file = "/Users/saraponton/Movies/Music/" + extension + ".mp3"
        pygame.init()
        pygame.mixer.init()
        mixer.init()
        mixer.music.load(file)
        mixer.music.play()
        pygame.mixer.music.get_busy()

window = Tk()

front = Frontend(window)
window.mainloop()


###Main module for OCR API application###
from __future__ import print_function
import os
import sys
import subprocess
import json
from PIL import Image
import logging

from flask import jsonify
from fuzzywuzzy import fuzz
from fuzzywuzzy import process

#configure pyhton logging module for log creation
logging.basicConfig(filename="/app/ocr.log",format='%(levelname)s:%(asctime)s:%(message)s', level=logging.DEBUG)

def notallowedfile():
    ocrreply = {
        "response":{
        "ocrtext":"Not allowed file type"
        },
    }
    with open('./ocr/casesdb.json', 'r') as f:
        cases = json.load(f)
    # return jsonify(ocr.file_not_found)
    return (cases["Not_Allowed_File_Ext"], ocrreply["response"])
#this function is used to run tesseract on the image file. Recieve the image file name
def ocr(image):
    print("ocr started", file=sys.stderr)
    logging.info(json.dumps({"action": "ocr started"}))
    command = ['tesseract', image, 'stdout']
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         )

    out, err = p.communicate()
    print("ocr ended", file=sys.stderr)
    logging.info(json.dumps({"action": "ocr finished"}))
    return out, err
#
# #Spcial cases for OCR
# #case a file was not found
file_not_found = {
    "query" : "default"
}
# #case of ocr failure
# ocr_dir_not_work  = {
#     "query": "temp_profile_ocrfail"
# }
# #case of no text was found in the image
# no_text_found = {
#     "query": "ocr_notextfound"
# }

#Function to resize the image file by x3. This increase OCR accuracy
def resizeimage(filename):
    if os.path.getsize(filename) < 999999:
        img = Image.open(filename)
        size = (((img.size[0] * 3), (img.size[1] * 3)))
        img_resized = img.resize(size, Image.ANTIALIAS)
        img_resized.save(filename, "PNG")
        print("image resized", file=sys.stderr)
        logging.info(json.dumps({"action": "image resized"}))

#Function to send the file for OCR function and ruturn the results
def process_file(filename):
    print("start process file", file=sys.stderr)
    logging.info(json.dumps({"action": "file processing started"}))
    #validate that the file exist
    if not os.path.exists(filename):
        print("file not found true", file=sys.stderr)
        logging.warn(json.dumps({"error": "file not found"}))
        return 1, file_not_found

    #send the file for resize if needed
    resizeimage(filename)
    print("start ocr", file=sys.stderr)
    logging.info(json.dumps({"action": "file sent for ocr"}))

#Run OCR on the file using tesseract
    out, err = ocr(filename)
    print("finish ocr", file=sys.stderr)
    logging.info(json.dumps({"action": "returned from ocr"}))
    print(err, file=sys.stderr)
    print(out, file=sys.stderr)

#OCR API Output cases - search for the text from the OCR in the errors DB. Return the OCR text and query
    #prepare the OCR text to be returned by OCR API
    ocrreply = out.decode('utf-8')
    ocrdata = {
        "response":{
        "ocrtext":ocrreply
        },
    }
    #open the cases DB for search
    with open('./ocr/casesdb.json', 'r') as f:
        data = json.load(f)

    #Case of image file withou text in it
    if out == b' \n\x0c':
         return 2, (data["No_Text_Found"], ocrdata["response"])

    #Compare the OCR text with error from the cases DB
    best_fuz =0
    for key in data.keys():
        # Perform fuzzynes comparision between OCR text and casesDB
        fuzzyness = fuzz.partial_ratio(str(key).lower(), str(out).lower())
        logging.info(json.dumps([str(key).lower(), fuzzyness]))
        if fuzzyness > 70:
            if fuzzyness > best_fuz:
                best_key = key
                best_fuz = fuzzyness
            logging.info(json.dumps({"ocr status": "known issue", "ocr text": ocrreply}))
            #return 0, (data[key], ocrdata["response"])
            logging.info(json.dumps([str(key).lower(),fuzzyness]))
        # if str(key).lower() in str(out).lower():
        #      logging.info(json.dumps({"ocr status": "known issue", "ocr text":ocrreply}))
        #      return 0,(data[key],ocrdata["response"])
    if best_fuz > 0:
        return 0, (data[best_key], ocrdata["response"])

    #case that the OCR data was not found in the errors DB
    logging.info(json.dumps({"ocr status": "issue not recognized", "ocr text": ocrreply}))
    return 1, (data["topic_not_recognized"], ocrdata["response"])


#!/usr/bin/env python
"""
Simple script to check availability of new docker images from hub.docker.com
"""
import docker

from helpers import mylogger

client = docker.from_env()

def get_name(image):
    """
    Get image name from "image" object
    """
    name = image.attrs["RepoDigests"][0]
    name = name.split("@")
    name = name[0]
    return name

def pull_images():
    """
    Pull images from hub.docker.com
    """
    image_list = client.images.list()
    for image in image_list:
        name = get_name(image)
        client.images.pull(name)

def check_updated():
    """
    Check if there are more than one image version pulled
    """
    image_list = client.images.list()
    message = ""
    myimages = []
    for image in image_list:
        name = get_name(image)
        if name in myimages:
            message += name
            message += "\n"
            mylogger.info("New image version pulled for %s", name)
        myimages.append(name)
    if not message:
        message = "No updates found."
        mylogger.info(message)
    else:
        message = "Following Docker images have updates:\n" + message
    return message


import json
import os

import scipy
from Evaluation import Evaluation
from OptimizationProblems import initialize_optimization_problem
from OptimizationTask import optimization_task
from billiard import Pool
from celery import Celery
from utils.Datatypes import OptimizationParameters
from utils.Logger import MyLogger

# Setting up logger
logger = MyLogger(__name__)

CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL'),
CELERY_RESULT_BACKEND = os.getenv('CELERY_RESULT_BACKEND')
NUMBER_OF_ATTEMPTS = 3

celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND)

def init_pool():
    scipy.random.seed()

@celery.task()
def optimization_evaluation_task(parameters: dict) -> str:
    """
    Optimize sequences using Optimization class, evaluate results using Evaluation class
    :param parameters: Dictionary created from OptimizationParameters class
    :return: JSON with the response dictionary containing both input and optimized sequences and their evaluation
    """

    parameters = OptimizationParameters.parse_obj(parameters)
    optimization_problems = [initialize_optimization_problem(parameters) for _ in range(parameters.number_of_sequences)]

    pool = Pool(initializer=init_pool)
    result = pool.map(optimization_task, optimization_problems)

    if len(set(result)) < parameters.number_of_sequences:
        logger.info('Less sequences than required!')
        attempt_count = 0
        while len(set(result)) < parameters.number_of_sequences and attempt_count < NUMBER_OF_ATTEMPTS:
            logger.info('Performing additional optimization attempt')
            result.append(optimization_task(initialize_optimization_problem(parameters)))
            attempt_count += 1

            if len(set(result)) == parameters.number_of_sequences:
                logger.info(f'Correct number achieved! Stopping...')
                break
            elif (attempt_count == parameters.number_of_sequences) and (
                    len(set(result)) < parameters.number_of_sequences):
                logger.info('Not able to provide required number of results. Stopping...')
                break
            else:
                pass

    evaluator = Evaluation(list(set(result)), parameters)
    final_response = evaluator.get_evaluation()
    final_response["input_parameters"] = parameters.dict()
    return json.dumps(final_response)


#
# @lc app=leetcode id=27 lang=python3
#
# [27] Remove Element
#

# @lc code=start
from typing import List

class Solution:
    def removeElement(self, nums: List[int], val: int) -> int:
        length = len(nums)
        index = 0
        for i in range(length):
            if nums[i] != val:
                nums[index] = nums[i]
                index += 1
        return index

# if __name__ == "__main__":
#     s = Solution()
#     print(s.removeElement([3, 2, 2, 3], 3))

# @lc code=end


from django.urls import path

from . import views

urlpatterns = [
    path(
        "transactions/",
        views.TransactionViewSet.as_view({"get": "get"}),
        name="transactions",
    ),
    path(
        "statements/",
        views.StatementViewSet.as_view({"get": "get", "post": "post"}),
        name="statements",
    ),
]