import math
import time

import os

import processing
from qgis.core import *
from qgis.utils import *
from qgis.gui import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5 import QtWidgets
import sys
import pandas as pd

from .Andalusian_Population_startup1 import *

class InterfazInstalacion(QDialog):
    def __init__(self, parent=None,lista_url_capas250=[]):
        super().__init__(parent)
        self.lista_url_capas250=lista_url_capas250
        self.salida=False
        self.fase="0/0"

        self.setGeometry(300, 300, 300, 220)
        self.setWindowTitle("Instalación de Capas")
        self.main_widget = QtWidgets.QWidget(self)
        #self.main_widget.setFocus()

        VerticalLayout = QVBoxLayout(self.main_widget)
        self.label=QLabel("Proceso de instalación de las capas del IECA (Instituto de Estadística y Cartografía de Andalucía)\n"
        "Este procedimiento puede llevar unos minutos")
        self.log = QPlainTextEdit(self.main_widget)
        self.log.setReadOnly(True)
        self.LabelEvolucion=QLabel()
        VerticalLayout.addWidget(self.label)
        VerticalLayout.addWidget(self.log)
        VerticalLayout.addWidget(self.LabelEvolucion)

        VerticalLayout.addStretch(1)
        btnLayout = QHBoxLayout()
        self.btn1 = QPushButton("Salir")
        self.btn1.clicked.connect(self.cerrar)
        btnLayout.addWidget(self.btn1)
        btnLayout.addStretch(1)
        self.btn2 = QPushButton("Finalizar")
        self.btn2.setEnabled(False)
        self.btn2.clicked.connect(self.cerrar)
        btnLayout.addWidget(self.btn2)
        self.btn3 = QPushButton("Comenzar")
        self.btn3.clicked.connect(self.comenzar)
        btnLayout.addWidget(self.btn3)
        VerticalLayout.addLayout(btnLayout)

        self.setLayout(VerticalLayout)

    def cerrar(self):
        self.close()

    def comenzar(self):
        self.btn1.setEnabled(False)
        self.btn2.setEnabled(False)
        self.btn3.setEnabled(False)

        self.thread = QThread()
        self.worker = Worker(None,self.lista_url_capas250)
        self.worker.moveToThread(self.thread)
        self.thread.started.connect(self.worker.run)
        self.worker.finished.connect(self.thread.quit)
        self.worker.progress.connect(self.PrintLogMain)
        self.thread.start()

        self.thread.finished.connect(lambda: self.btn1.setEnabled(True))
        self.thread.finished.connect(lambda: self.btn2.setEnabled(True))

        #self.PrintLog()
        #self.exportar_tablas()

        self.salida=True

    def PrintLogMain(self,msg="",fase="0/0"):
        self.log.appendPlainText(msg)
        self.LabelEvolucion.setText("Capas instaladas: " + fase)

class Worker(QObject):
    finished = pyqtSignal()
    progress = pyqtSignal(str,str)
    def __init__(self,parent=None,lista_url_capas250=[]):
        super().__init__(parent)
        self.lista_url_capas250=lista_url_capas250

    def PrintLog(self,msg):
        self.progress.emit(msg,self.fase)

    def run(self):
        basepath = os.path.dirname(os.path.realpath(__file__)) + "\\"
        tipoVersion=getVersion()

        self.fase="0/3"
        self.PrintLog("Obteniendo la capa de población (gridpob)")
        nombreAtributos=['grd_floaid','municipio','cmun','pob_tot','pob_m','edad0015','edad65_',
        'esp','ue15','mag','ams','tr_05','tr16_','afil_ss','afil_ss_m','pencinc','demp_pr','demp_pr_m']
        gridpobLayer=AnadirLayerQGIS(url=self.lista_url_capas250[2],nombre="Capa grid de poblacion", tipoCapa="WFS")
        OPTIONS=QgsVectorFileWriter.SaveVectorOptions()
        OPTIONS.driverName="CSV"
        idAtributos=[]
        for nombre in nombreAtributos:
            idAtributos.append(gridpobLayer.fields().indexOf(nombre))
            if idAtributos[-1]==-1:
                idAtributos.pop(-1)
                self.PrintLog("El atributo '"+ nombre + "' de la capa '" + gridpobLayer.name() + "' no se ha encontrado")
        OPTIONS.attributes=idAtributos
        OPTIONS.layerOptions =['SEPARATOR=SEMICOLON']
        OPTIONS.fileEncoding="UTF-8"
        if tipoVersion:
            QgsVectorFileWriter.writeAsVectorFormatV2(gridpobLayer, basepath + "tabla_gridpob.csv",QgsCoordinateTransformContext(),OPTIONS)
            pass
        else:
            QgsVectorFileWriter.writeAsVectorFormat(gridpobLayer, basepath + "tabla_gridpob.csv", 'UTF-8', gridpobLayer.crs(), 'CSV',attributes=idAtributos,layerOptions ='SEPARATOR=SEMICOLON')
        #self.PrintLog("Se ha finalizado la exportacion de la capa de población (gridpob)")

        self.PrintLog("Se están procesando los datos de la capa gridpob")
        df=pd.read_csv(basepath + "tabla_gridpob.csv",sep=";",header=0)
        df['grd_floaid']=df['grd_floaid'].str.replace('.','')
        k=0
        tam=df.shape[0]
        for i in range(tam):
            aux=df['cmun'][i].split(' / ')
            if len(aux)>1:
                aux2=df['municipio'][i].split(' / ')
                df['municipio'][i]=aux2[0]
                df['cmun'][i]=aux[0]
                j=1
                while j<len(aux):
                    df=df.append([df.loc[i]],ignore_index=True)
                    df['municipio'][tam-1+k+j]=aux2[j]
                    df['cmun'][tam-1+k+j]=aux[j]
                    j+=1
                k+=j-1

        df=df[(df[df.columns[3:]].T != -1).any()]
        #df.to_csv(basepath + "tabla_gridpob.csv",sep=";")

        self.fase="1/3"
        self.PrintLog("Obteniendo la capa tipologías constructivas (gridcattpLayer)")
        nombreAtributos=['grd_floaid','ctotal','porc_c011','porc_c012','porc_c02','porc_c03','porc_c04','porc_c05','porc_c08','porc_c09']

        gridcattpLayer=AnadirLayerQGIS(url=self.lista_url_capas250[3],nombre="Capa grid de tipologías constructivas", tipoCapa="WFS")
        OPTIONS=QgsVectorFileWriter.SaveVectorOptions()
        OPTIONS.driverName="CSV"
        idAtributos=[]
        for nombre in nombreAtributos:
            idAtributos.append(gridcattpLayer.fields().indexOf(nombre))
            if idAtributos[-1]==-1:
                idAtributos.pop(-1)
                self.PrintLog("El atributo '"+ nombre + "' de la capa '" + gridcattpLayer.name() + "' no se ha encontrado")
        OPTIONS.attributes=idAtributos
        OPTIONS.layerOptions =['SEPARATOR=SEMICOLON']
        OPTIONS.fileEncoding="UTF-8"
        if tipoVersion:
            QgsVectorFileWriter.writeAsVectorFormatV2(gridcattpLayer, basepath + "tabla_gridcattp.csv",QgsCoordinateTransformContext(),OPTIONS)
            #pass
        else:
            QgsVectorFileWriter.writeAsVectorFormat(gridcattpLayer, basepath + "tabla_gridcattp.csv", 'UTF-8', gridcattpLayer.crs(), 'CSV',attributes=idAtributos,layerOptions ='SEPARATOR=SEMICOLON')
        #self.PrintLog("Se ha finalizado la exportacion de la capa tipologías constructivas (gridcattpLayer)")

        df_aux=pd.read_csv(basepath + "tabla_gridcattp.csv",sep=";",header=0)
        df_aux=df_aux[(df_aux[df_aux.columns[1:]].T != 0).any()]
        df_aux['grd_floaid']=df_aux['grd_floaid'].str.replace('.','')
        df_res=pd.merge(df, df_aux, how='outer', on='grd_floaid')
        df=df_res

        self.fase="2/3"
        self.PrintLog("Obteniendo la capa de mortalidad RMES (gridmortalidad)")
        nombreAtributos=['GRD_FLOAID','RMES_A00','RMES_A45','RMES_A65']
        gridmortalidadLayer=AnadirLayerQGIS(url=self.lista_url_capas250[4],nombre="Capa grid de mortalidad", tipoCapa="WFS")
        OPTIONS=QgsVectorFileWriter.SaveVectorOptions()
        OPTIONS.driverName="CSV"
        idAtributos=[]
        for nombre in nombreAtributos:
            idAtributos.append(gridmortalidadLayer.fields().indexOf(nombre))
            if idAtributos[-1]==-1:
                idAtributos.pop(-1)
                self.PrintLog("El atributo '"+ nombre + "' de la capa '" + gridmortalidadLayer.name() + "' no se ha encontrado")
        OPTIONS.attributes=idAtributos
        OPTIONS.layerOptions =['SEPARATOR=SEMICOLON']
        OPTIONS.fileEncoding="UTF-8"
        if tipoVersion:
            QgsVectorFileWriter.writeAsVectorFormatV2(gridmortalidadLayer, basepath + "tabla_gridmortalidad.csv",QgsCoordinateTransformContext(),OPTIONS)
            #pass
        else:
            QgsVectorFileWriter.writeAsVectorFormat(gridmortalidadLayer, basepath + "tabla_gridmortalidad.csv", 'UTF-8', gridmortalidadLayer.crs(), 'CSV',attributes=idAtributos,layerOptions ='SEPARATOR=SEMICOLON')
        #self.PrintLog("Se ha finalizado la exportacion de la capa de mortalidad RMES (gridmortalidad)")
        df_aux=pd.read_csv(basepath + "tabla_gridmortalidad.csv",sep=";",header=0)
        df_aux=df_aux[(df_aux[df_aux.columns[1:]].T != -1).any()]

        df_aux=df_aux.rename(columns={"GRD_FLOAID":"grd_floaid"})

        df_aux['grd_floaid']=df_aux['grd_floaid'].str.replace('.','')
        df_res=pd.merge(df, df_aux, how='outer', on='grd_floaid')
        df=df_res

        self.fase="3/3"
        """
        self.PrintLog("Obteniendo la capa de tipología de viviendas (gridcatv)")
        nombreAtributos=['grd_floaid','nviv','nviv_i045m2','aant_mediana','nviv_i60a']
        gridcatvLayer=AnadirLayerQGIS(url=self.lista_url_capas250[5],nombre="Capa grid de mortalidad", tipoCapa="WFS")
        OPTIONS=QgsVectorFileWriter.SaveVectorOptions()
        OPTIONS.driverName="CSV"
        idAtributos=[]
        #for i in gridcatvLayer.fields():
        #    print(i.name())
        for nombre in nombreAtributos:
            idAtributos.append(gridcatvLayer.fields().indexOf(nombre))
            if idAtributos[-1]==-1:
                idAtributos.pop(-1)
                self.PrintLog("El atributo '"+ nombre + "' de la capa '" + gridcatvLayer.name() + "' no se ha encontrado")
        #print(idAtributos)
        OPTIONS.attributes=idAtributos
        OPTIONS.layerOptions =['SEPARATOR=SEMICOLON']
        OPTIONS.fileEncoding="UTF-8"
        if tipoVersion:
            QgsVectorFileWriter.writeAsVectorFormatV2(gridcatvLayer, basepath + "tabla_gridcatv.csv",QgsCoordinateTransformContext(),OPTIONS)
            #pass
        else:
            QgsVectorFileWriter.writeAsVectorFormat(gridcatvLayer, basepath + "tabla_gridcatv.csv", 'UTF-8', gridcatvLayer.crs(), 'CSV',attributes=idAtributos,layerOptions ='SEPARATOR=SEMICOLON')

        #self.PrintLog("Se ha finalizado la exportacion de la capa de tipología de viviendas (gridcatv)")
        df_aux=pd.read_csv(basepath + "tabla_gridcatv.csv",sep=";",header=0)
        df_aux=df_aux[(df_aux[df_aux.columns[1:]].T != 0).any()]
        df_aux['grd_floaid']=df_aux['grd_floaid'].str.replace('.','')
        #df_aux['grd_floaid'] = df_aux['grd_floaid'].astype(float)
        df_res=pd.merge(df, df_aux, how='outer', on='grd_floaid')
        df=df_res

        self.fase="4/4"
        """

        """self.PrintLog("Obteniendo la capa de tipología de viviendas (gridcatv)")
        nombreAtributos=['grd_floaid','nviv_i045m2','aant_mediana','nviv_i60a']
        gridcatvLayer=AnadirLayerQGIS(url=self.lista_url_capas250[5],nombre="Capa grid de mortalidad", tipoCapa="WFS")
        OPTIONS=QgsVectorFileWriter.SaveVectorOptions()
        OPTIONS.driverName="CSV"
        idAtributos=[]
        #for i in gridcatvLayer.fields():
        #    print(i.name())
        for nombre in nombreAtributos:
            idAtributos.append(gridcatvLayer.fields().indexOf(nombre))
            if idAtributos[-1]==-1:
                idAtributos.pop(-1)
                self.PrintLog("El atributo '"+ nombre + "' de la capa '" + gridcatvLayer.name() + "' no se ha encontrado")
        #print(idAtributos)
        OPTIONS.attributes=idAtributos
        OPTIONS.layerOptions =['SEPARATOR=SEMICOLON']
        OPTIONS.fileEncoding="UTF-8"
        if tipoVersion:
            QgsVectorFileWriter.writeAsVectorFormatV2(gridcatvLayer, basepath + "tabla_gridcatv.csv",QgsCoordinateTransformContext(),OPTIONS)
        else:
            QgsVectorFileWriter.writeAsVectorFormat(gridcatvLayer, basepath + "tabla_gridcatv.csv", 'UTF-8', gridcatvLayer.crs(), 'CSV',attributes=idAtributos,layerOptions ='SEPARATOR=SEMICOLON')

        #self.PrintLog("Se ha finalizado la exportacion de la capa de tipología de viviendas (gridcatv)")
        df_aux=pd.read_csv(basepath + "tabla_gridcatv.csv",sep=";",header=0)
        df_aux=df_aux[(df_aux[df_aux.columns[1:]].T != 0).any()]
        df_aux['grd_floaid']=df_aux['grd_floaid'].str.replace('.','')
        #df_aux['grd_floaid'] = df_aux['grd_floaid'].astype(float)
        df_res=pd.merge(df, df_aux, how='outer', on='grd_floaid')
        df=df_res"""

        df = df[df['cmun'].notna()]
        os.remove(basepath + 'tabla_gridpob.csv')
        os.remove(basepath + 'tabla_gridcattp.csv')
        os.remove(basepath + 'tabla_gridmortalidad.csv')
        #os.remove(basepath + 'tabla_gridcatv.csv')
        df.to_csv(basepath + "tabla.csv",sep=";",decimal=",")

        self.PrintLog("Se ha finalizado la exportacion de capas")
        #Señal de que ha finalizado
        self.finished.emit()


from gevent.event import AsyncResult
from raidex import messages
from raidex.signing import Signer
from raidex.raidex_node.offer_book import OfferDeprecated
from raidex.raidex_node.order.offer import OfferType
from raidex.utils import timestamp

class Swap(object):

    def __init__(self, maker_instance):
        self._maker = maker_instance
        self._taker = None
        self._maker_executed = False
        self._taker_executed = False

    def try_take(self, cs_instance):
        if self._taker is None:
            self._taker = cs_instance
            return True
        return False

    @property
    def is_completed(self):
        # no timeout comparison
        return self._maker_executed and self._taker_executed

    @property
    def is_taken(self):
        return (self._maker is not None) and (self._taker is not None)

    def report_executed(self, cs_instance):
        if self.is_completed:
            return False
        if not self._taker:
            return False
        if cs_instance not in (self._maker, self._taker):
            return False
        if cs_instance is self._maker:
            self._maker_executed = True
            return True
        if cs_instance is self._taker:
            self._taker_executed = True
            return True

class CommitmentServiceGlobal(object):
    """
    Serves as the brain of the CS-Server, that keeps track of all the reported swaps
    """
    def __init__(self, signer=None):
        if signer is None:
            signer = Signer()
        self.signer = signer
        self.swaps = {}

    @property
    def address(self):
        return self.signer.address

    def make_offer(self, commitment_service_client, offer_id):
        if offer_id in self.swaps:
            return False
        self.swaps[offer_id] = Swap(commitment_service_client)
        return True

    def try_take_offer(self, commitment_service_client, offer_id):
        swap = self.swaps.get(offer_id)
        if swap is None:
            return False
        success = swap.try_take(commitment_service_client)
        return success

    def report_swap_executed(self, commitment_service_client, offer_id):
        swap = self.swaps.get(offer_id)
        if swap is None:
            return False
        success = swap.report_executed(commitment_service_client)
        return success

    def swap_is_completed(self, offer_id):
        swap = self.swaps.get(offer_id)
        if swap is None:
            return False
        if swap.is_completed:
            return True
        else:
            return False

class NonFailingCommitmentServiceGlobal(object):
    """
    Replacement of CommitmentServiceGlobal
    This has no inherent logic and always returns True for all methods.
    This makes sense to use, when no Taker/Maker matchmaking logic should be present
    and the CommitmentService Clients are only being used to construct and broadcast the
    correct messages, signed by the CS.
    """
    def __init__(self, signer=None):
        if signer is None:
            signer = Signer.random()
        self.signer = signer

    @property
    def address(self):
        return self.signer.address

    def make_offer(self, commitment_service_client, offer_id):
        return True

    def try_take_offer(self, commitment_service_client, offer_id):
        return True

    def report_swap_executed(self, commitment_service_client, offer_id):
        return True

    def swap_is_completed(self, offer_id):
        return True

class CommitmentServiceClientMock(object):
    """Mock for the CommitmentService
    Has the same interface as the CommitmentService client, but also includes the mocked CS-Node behaviour

    At the moment just creates the messages that you would get for a proper commitment.
    Every raidex node has its own with the same key
    """
    _commitment_service_global = NonFailingCommitmentServiceGlobal()

    def __init__(self, node_signer, token_pair, message_broker, fee_rate=0.1, commitment_service_global=None):
        if commitment_service_global is not None:
            self._commitment_service_global = commitment_service_global
        self._global_sign = self._commitment_service_global.signer.sign
        self.commitment_service_address = self._commitment_service_global.address
        self.node_address = node_signer.address
        self.token_pair = token_pair
        self._sign = node_signer.sign
        self.fee_rate = fee_rate
        self.message_broker = message_broker

    def maker_commit_async(self, offer):
        # type: (OfferDeprecated) -> AsyncResult
        result = AsyncResult()
        success = self._commitment_service_global.make_offer(self, offer.offer_id)
        if success is False:
            result.set(None)
            return result
        offer_msg = self.create_offer_msg(offer)
        commitment_msg = messages.Commitment(offer.offer_id, offer_msg.hash, offer.timeout_date, 42)
        self._sign(commitment_msg)

        commitment_proof_msg = messages.CommitmentProof(commitment_msg.signature)
        self._global_sign(commitment_proof_msg)
        proven_offer_msg = messages.ProvenOffer(offer_msg, commitment_msg, commitment_proof_msg)
        self._sign(proven_offer_msg)
        result.set(proven_offer_msg)
        return result

    def taker_commit_async(self, offer):
        # type: (OfferDeprecated) -> AsyncResult
        result = AsyncResult()
        success = self._commitment_service_global.try_take_offer(self, offer.offer_id)
        if success is False:
            result.set(None)
            return result
        offer_msg = self.create_offer_msg(offer)
        commitment_msg = messages.TakerCommitment(offer.offer_id, offer_msg.hash, offer.timeout_date, 42)
        self._sign(commitment_msg)
        commitment_proof_msg = messages.CommitmentProof(commitment_msg.signature)
        self._global_sign(commitment_proof_msg)
        proven_commitment_msg = messages.ProvenCommitment(commitment_msg, commitment_proof_msg)
        self._sign(proven_commitment_msg)
        result.set(proven_commitment_msg)
        self.message_broker.broadcast(self.create_taken(offer.offer_id))
        return result

    def create_taken(self, offer_id):
        # type: (int) -> OfferTaken
        offer_taken_msg = messages.OfferTaken(offer_id)
        self._global_sign(offer_taken_msg)
        return offer_taken_msg

    def create_swap_completed(self, offer_id):
        # type: (int) -> SwapCompleted
        swap_completed_msg = messages.SwapCompleted(offer_id, timestamp.time())
        self._global_sign(swap_completed_msg)
        return swap_completed_msg

    def create_offer_msg(self, offer):
        # type: (OfferDeprecated) -> OfferMsg
        if offer.type == OfferType.SELL:
            return messages.SwapOffer(self.token_pair.quote_token, offer.quote_amount, self.token_pair.base_token,
                                      offer.base_amount, offer.offer_id, offer.timeout_date)
        else:
            return messages.SwapOffer(self.token_pair.base_token, offer.base_amount, self.token_pair.quote_token,
                                      offer.quote_amount, offer.offer_id, offer.timeout_date)

    def report_swap_executed(self, offer_id):
        # type: (int) -> None
        success = self._commitment_service_global.received_inbound_from_swap(self, offer_id)
        if success is True and self._commitment_service_global.swap_is_completed(offer_id):
            swap_completed_msg = self.create_swap_completed(offer_id)
            self.message_broker.broadcast(swap_completed_msg)

    def start(self):
        # in order to provide the same interface as the client.CommitmentService
        # the actual tasks are not running because this is a failsafe mock without CS-interaction
        pass


"""add uuid for visible keys

Revision ID: bb34f3d9a8f4
Revises: e356e6efd25d
Create Date: 2017-09-30 11:37:24.898992

"""
from alembic import op
import sqlalchemy as sa
import geoalchemy2
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = 'bb34f3d9a8f4'
down_revision = 'e356e6efd25d'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('carpools', sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=True))
    op.create_index(op.f('ix_carpools_uuid'), 'carpools', ['uuid'], unique=False)
    op.add_column('destinations', sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=True))
    op.create_index(op.f('ix_destinations_uuid'), 'destinations', ['uuid'], unique=False)
    op.add_column('people', sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=True))
    op.create_index(op.f('ix_people_uuid'), 'people', ['uuid'], unique=False)
    op.add_column('riders', sa.Column('uuid', postgresql.UUID(as_uuid=True), nullable=True))
    op.create_index(op.f('ix_riders_uuid'), 'riders', ['uuid'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_riders_uuid'), table_name='riders')
    op.drop_column('riders', 'uuid')
    op.drop_index(op.f('ix_people_uuid'), table_name='people')
    op.drop_column('people', 'uuid')
    op.drop_index(op.f('ix_destinations_uuid'), table_name='destinations')
    op.drop_column('destinations', 'uuid')
    op.drop_index(op.f('ix_carpools_uuid'), table_name='carpools')
    op.drop_column('carpools', 'uuid')
    # ### end Alembic commands ###


# -*- coding : utf-8 -*-
import os
import sys
from logger import Logger
import xlearn as xl
from config import Config

conf = Config()
sys.stdout = Logger(os.path.join(conf.log_path, conf.model+"_log.log"), sys.stdout)  # 将控制台输出同时输出到log文件

if __name__ == "__main__":
    # 设定sparse features和dense features
    sparse_features = ['uid', 'user_city', 'item_id', 'author_id', 'item_city', 'channel',
                       'music_id', 'did', 'gender']
    dense_features = ['video_duration', 'beauty']
    target = ['finish', 'like']
    epochs = 5
    split_num = 1

    for tar in target:
        #if tar == "like": continue
        print("==> Start to train, target : %s" % tar)
        valid_path = os.path.join(conf.data_path, "FFM_" + tar + "_valid.txt")
        model_path = os.path.join(conf.model_save_path, "FFM_"+tar+".model")

        print("==> Training model")
        for epoch in range(epochs):
            print("==> Epochs %d in all, training epoch : %d" % (epochs, epoch+1))
            # 设置训练属性和参数
            param = {'task': 'binary',
                     'k': 5,
                     'lr': 0.2 ,
                     'lambda': 0.003,
                     'metric': 'auc',
                     'epoch': 100,
                     'nthread': 6,
                     'block_size': 10000}
            for i in range(split_num):
                train_path = os.path.join(conf.data_path, "FFM_" + tar + "_train%d.txt"%(i+1))
                ffm_model = xl.create_ffm()
                ffm_model.setTrain(train_path)
                ffm_model.setValidate(valid_path)
                ffm_model.setOnDisk()
                if os.path.exists(model_path):
                    ffm_model.setPreModel(model_path)
                ffm_model.fit(param, model_path)
    print("==> Finish training")


import cv2
import numpy as np
import time
from math import ceil

"""
@hi: resmin yüksekliği
@ph: cismin resimdeki bulunduğu piksel
@Z: Kameranın yere uzaklığı
"""
def calcRealY(hi, ph, Z):
    # Kamera teknik sabitleri
    hs = 0.00274 # m (2.74mm)
    f = 0.00290 #m (3.29mm)

    # Cismin resimdeki orta noktaya göre konumu
    pv = hi/2 - ph # px

    # Cismin sensör alanındaki orta noktaya göre konumu
    ys = pv * hs / hi # m

    # Cismin gerçek konumu
    Y = ys * Z / f
    return Y

def calcSizePixel(width, height, altitude = 10, size = 2):
    s_width = 0.00376 # m
    s_height = 0.00274 # m
    focal = 0.00360 # m

    totalX = s_width * altitude / focal
    xsize = width * size / totalX

    totalY = s_height * altitude/ focal
    ysize = height * size / totalY

    return xsize*ysize

"""
@hi: resmin yüksekliği
@ph: cismin resimdeki bulunduğu piksel
@Z: Kameranın yere uzaklığı
"""
def calcRealX(hi, ph, Z):
    # Kamera teknik sabitleri
    hs = 0.00367 # m (2.74mm)
    f = 0.00360 #m (3.29mm)

    # Cismin resimdeki orta noktaya göre konumu
    pv = ph - hi/2  # px

    # Cismin sensör alanındaki orta noktaya göre konumu
    ys = pv * hs / hi # m

    # Cismin gerçek konumu
    Y = ys * Z / f
    return Y

def drawRealPos(img, pos, size):
    w = img.shape[1]
    h = img.shape[0]
    color = (0, 255,0)
    fsize = .7
    thick = 1

    # X ekseni çizilir
    cv2.line(img, (ceil(w/2), ceil(h/2)), (pos[0], ceil(h/2)), color)
    cv2.putText(img, ("%.2f" % (size[0]*100)) + "cm", (pos[0], ceil(h/2)), cv2.FONT_HERSHEY_SIMPLEX, fsize,
                 color, thick, cv2.LINE_AA, False)

    # Y ekseni çizilir
    cv2.line(img, (ceil(w/2), ceil(h/2)), (ceil(w/2), pos[1]), color)
    cv2.putText(img, ("%.2f" % (size[1]*100)) + "cm", (ceil(w/2),pos[1]), cv2.FONT_HERSHEY_SIMPLEX, fsize,
                 color, thick, cv2.LINE_AA, False)

# [xmin ymin xlen ylen hc wc]

def process(loc, altitude=1):
    (obj_start_x, obj_start_y,
     obj_width, obj_height,
     center_x, center_y
    ) = loc
    s_width = 0.00368 # m
    s_height = 0.00276 # m
    focal = 0.00304 # m

    width = center_x * 2
    height = center_y * 2

    img_center_x = obj_start_x + obj_width//2
    img_center_y = obj_start_y + obj_height//2

    # X için gerçek uzaklık hesabı
    # Resmin koordinatından orta noktanın koordinatına geçiş
    u = img_center_x - center_x # px = px - px
    # Pixelden sensör üzerindeki gerçek boyut dönüşümü
    x_sensor = s_width * u / width # m = m * px / px
    # Sensör üzerindeki boyuttan üçgen benzerliği ile gerçek boyuta dönüşüm
    x_real = x_sensor * altitude / focal  # m = m * m / m

    # Y için gerçek uzaklık hesabı
    # Resmin koordinatından orta noktanın koordinatına geçiş
    v = center_y - img_center_y # px = px - px
    # Pixelden sensör üzerindeki gerçek boyut dönüşümü
    y_sensor = s_height * v / height # m = m * px / px
    # Sensör üzerindeki boyuttan üçgen benzerliği ile gerçek boyuta dönüşüm
    y_real = y_sensor * altitude / focal # m = m * m / m

    return (x_real, y_real)

def waterSense(img):
    imgnew = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    #mat = imgnew[440:480,:,:]
    mat = imgnew[440:480,:,:]

    l = np.array([0,0,0])
    u = np.array([360,125,125])
    avg = mat.mean(axis=0).mean(axis=0)

    if(avg[0] > l[0] and avg[0] < u[1] and
    avg[1] > l[1] and avg[1] < u[1] and
    avg[2] > l[2] and avg[2] < u[2]):
        cv2.putText(img, "!!! SU BULUNDU !!!", (img.shape[0]//2-200, img.shape[1]//2-80), cv2.FONT_HERSHEY_SIMPLEX, 2,
                (0,255,0), 3, cv2.LINE_AA, False)
        return True
    else:
        cv2.putText(img, "SU ARANIYOR", (img.shape[0]//2-120, img.shape[1]//2-80), cv2.FONT_HERSHEY_SIMPLEX, 2,
                (0,250,220), 2, cv2.LINE_AA, False)
        return False

    #mat1 = img[440:470,:,:]
    #avg1 = mat1.mean(axis=0).mean(axis=0)
    #img = cv2.rectangle(img, (0,440), (640,480), (0,0,0), -1)
    #return img


from django.shortcuts import render, get_object_or_404
from news.models import MyNewsFb
# Create your views here.

def homeview(request):
    posts = MyNewsFb.objects.all()
    context = {
        'posts': posts
    }
    return render(request, 'index.html', context=context)

def postdetail(request, id):
    post = get_object_or_404(MyNewsFb, pk=id)
    context = {
        'post': post,
    }
    return render(request, 'detailpost.html', context=context)


from __future__ import annotations

from functools import cached_property
from typing import Iterable, Mapping, MutableMapping

from werkzeug.datastructures import ImmutableDict

from app.utilities.make_immutable import make_immutable
from app.utilities.types import (
    SupplementaryDataKeyType,
    SupplementaryDataListMapping,
    SupplementaryDataValueType,
)

class InvalidSupplementaryDataSelector(Exception):
    pass

class SupplementaryDataStore:
    """
    An object that stores supplementary data
    """

    def __init__(
        self,
        supplementary_data: MutableMapping | None = None,
        list_mappings: Mapping[str, list[SupplementaryDataListMapping]] | None = None,
    ):
        """
        Initialised with the "data" value from the sds api response
        and list mappings of the form
        {
            list_name: [
                {"identifier": identifier-1, "list_item_id": list_item_id-1 },
                {"identifier": identifier-2, "list_item_id": list_item_id-2 }
            ]
        }
        """
        self._raw_data = supplementary_data or {}
        self._list_mappings = list_mappings or {}
        # use shallow copy of the data, as items will be popped off
        self._data_map = self._build_map({**self._raw_data})

    @cached_property
    def raw_data(self) -> ImmutableDict:
        data: ImmutableDict = make_immutable(self._raw_data)
        return data

    @cached_property
    def list_mappings(self) -> ImmutableDict[str, list[ImmutableDict]]:
        mappings: ImmutableDict[str, list[ImmutableDict]] = make_immutable(
            self._list_mappings
        )
        return mappings

    @cached_property
    def list_lookup(self) -> dict[str, dict[str | int, str]]:
        """Create a lookup for easily finding the list_item_id for a given identifier"""
        return {
            list_name: {
                mapping["identifier"]: mapping["list_item_id"] for mapping in list_data
            }
            for list_name, list_data in self._list_mappings.items()
        }

    def _build_map(
        self, data: MutableMapping
    ) -> dict[SupplementaryDataKeyType, SupplementaryDataValueType]:
        """
        The raw data will be of the form
        {
          "some_key": "some_value"
          "items": {
            "some_list": [
                {"identifier": ... },
                {"identifier": ... }
            ]
          }
        }
        each list item has an identifier which will link to a list-item-id in self.list_lookup
        for example: {"some_list": {identifier-1: list_item_id-1, identifier-2: list_item_id-2 }}

        resulting map based off list mappings has the form
        {
            ("some_key", None): "some_value"
            ("some_list", list_item_id-1): {"identifier": identifier-1, ...}
            ("some_list", list_item_id-2): {"identifier": identifier-2, ...}
        }
        """
        list_items = data.pop("items", {})
        resulting_map: dict[SupplementaryDataKeyType, SupplementaryDataValueType] = {
            (key, None): value for key, value in data.items()
        }
        for list_name, list_data in list_items.items():
            for item in list_data:
                identifier = item["identifier"]
                list_item_id = self.list_lookup[list_name][identifier]
                resulting_map[(list_name, list_item_id)] = item
        return resulting_map

    def get_data(
        self,
        *,
        identifier: str,
        selectors: Iterable[str] | None = None,
        list_item_id: str | None = None,
    ) -> SupplementaryDataValueType:
        """
        Used to retrieve supplementary data in a similar style to AnswerStore
        the identifier is the top level key for static data, and the name of the list for list items
        selectors are used to reference nested data

        For example if you wanted the identifier for the first item in "some_list"
        it would be get_data(identifier="some_list", selectors=["identifier"], list_item_id=list_item_id-1)
        """
        if self.is_data_repeating(identifier) and not list_item_id:
            raise InvalidSupplementaryDataSelector(
                f"Cannot reference items from `{identifier}` outside a repeat"
            )

        value = self._data_map.get((identifier, list_item_id))
        # for nested data, index with each selector, or return None if there is no data to index
        for selector in selectors or []:
            if value is None:
                return None
            if not isinstance(value, Mapping):
                # if value is not None, and also not index able, raise an error
                raise InvalidSupplementaryDataSelector(
                    f"Cannot use the selector `{selector}` on non-nested data"
                )
            value = value.get(selector)

        return value

    def is_data_repeating(self, identifier: str) -> bool:
        """
        Returns true if the identifier is for one of the lists
        """
        return identifier in self._list_mappings

    def serialize(self) -> dict:
        return {
            "data": self._raw_data,
            "list_mappings": self._list_mappings,
        }

    @classmethod
    def deserialize(cls, serialized_data: Mapping) -> SupplementaryDataStore:
        if not serialized_data:
            return cls()

        return cls(
            supplementary_data=serialized_data["data"],
            list_mappings=serialized_data["list_mappings"],
        )


from django.shortcuts import render, redirect
from django.http import HttpResponse, JsonResponse
import datetime
import json
from django.template.loader import render_to_string
from django.core.mail import EmailMessage

from cart.models import CartItem
from .forms import OrderForm
from .models import Order, Payment, OrderProduct
from store.models import Product

# Create your views here.

def payments(request):
	body = json.loads(request.body)

	order = Order.objects.get(user=request.user, is_ordered=False, order_number=body.get('orderID'))

	payment = Payment(
			user = request.user,
			payment_id = body.get('transID'),
			payment_method = body.get('payment_method'),
			amount_paid = order.order_total,
			status = body.get('status'),
		)

	payment.save()

	order.payment = payment
	order.is_ordered = True
	order.save()

	# Move the cart items to Order Product Table
	cart_items = CartItem.objects.filter(user=request.user)

	for item in cart_items:
		orderproduct = OrderProduct()
		orderproduct.order_id = order.id
		orderproduct.payment = payment
		orderproduct.user_id = request.user.id
		orderproduct.product_id = item.product_id
		orderproduct.quantity = item.quantity
		orderproduct.product_price = item.product.price
		orderproduct.ordered = True
		orderproduct.save()

		cart_item = CartItem.objects.get(id=item.id)
		product_variation = cart_item.variations.all()
		orderproduct = OrderProduct.objects.get(id=orderproduct.id)
		orderproduct.variation.set(product_variation)
		orderproduct.save()

		# -----Reduce the stock of product after order-----

		product = Product.objects.get(id=item.product_id)
		product.stock -= item.quantity
		product.save()

	#--- Clear the cart after ordering product

	CartItem.objects.filter(user=request.user).delete()

	#--- Sending order confirmation email

	mail_subject = 'Thankyou for your order!'
	message = render_to_string('orders/order_recieved_email.html', {
		'user': request.user,
		'order': order,
		})

	to_email = request.user.email
	send_email = EmailMessage(mail_subject, message, to=[to_email])
	send_email.send()

	data = {
			'order_number': order.order_number,
			'transID': payment.payment_id
	}

	return JsonResponse(data)

def place_order(request, total=0, quantity=0):
	current_user = request.user

	cart_items = CartItem.objects.filter(user=current_user)
	cart_items_count = len(cart_items)

	if cart_items_count <= 0:
		return redirect('store')

	grand_total = 0
	tax = 0
	for cart_item in cart_items:
		total += (cart_item.product.price * cart_item.quantity)
		quantity +=cart_item.quantity

	tax = (2 * total)/100
	grand_total = total + tax

	if request.method == "POST":
		form = OrderForm(request.POST)

		if form.is_valid():
			data = Order()
			data.user = current_user
			data.first_name = form.cleaned_data['first_name']
			data.last_name = form.cleaned_data['last_name']
			data.phone = form.cleaned_data['phone']
			data.email = form.cleaned_data['email']
			data.address_line_1 = form.cleaned_data['address_line_1']
			data.address_line_2 = form.cleaned_data['address_line_2']
			data.country = form.cleaned_data['country']
			data.state = form.cleaned_data['state']
			data.city = form.cleaned_data['city']
			data.landmark = form.cleaned_data['landmark']
			data.zip_code = form.cleaned_data['zip_code']
			data.other_detail = form.cleaned_data['other_detail']
			data.order_total = grand_total
			data.tax = tax
			data.ip = request.META.get('REMOTE_ADDR')
			data.save()

			# GENERATE ORDER NUMBER
			yr = int(datetime.date.today().strftime('%Y'))
			mt = int(datetime.date.today().strftime('%m'))
			dt = int(datetime.date.today().strftime('%d'))
			d = datetime.date(yr,mt,dt)
			current_date = d.strftime("%Y%m%d")
			order_number = current_date + str(data.id)
			data.order_number = order_number
			data.save()

			order = Order.objects.get(user=current_user, is_ordered=False, order_number=order_number)

			context = {
					'order': order,
					'cart_items': cart_items,
					'total': total,
					'grand_total': grand_total,
					'tax': tax,

			}
			return render(request, 'orders/payments.html', context)

		else:
			return HttpResponse("form is invalid")

	else:
		return render(request, 'cart/cart.html')

def order_complete(request):
	order_number = request.GET.get('order_number')
	transID = request.GET.get('payment_id')

	try:
		order = Order.objects.get(order_number=order_number, is_ordered=True)
		ordered_products = OrderProduct.objects.filter(order_id=order.id)

		for item in ordered_products:
			print(item.variation.all)

		payment = Payment.objects.get(payment_id=transID)

		subtotal = 0
		for i in ordered_products:
			subtotal += i.product_price * i.quantity

		context = {
				'order': order,
				'ordered_products': ordered_products,
				'payment': payment,
				'subtotal': subtotal,
		}

		return render(request, 'orders/order_complete.html', context)

	except (Payment.DoesNotExist, Order.DoesNotExist):
		return redirect('home')


from os.path import dirname, abspath, join

from setuptools import setup, find_packages
from privex.exchange import VERSION
import warnings

with open("README.md", "r") as fh:
    long_description = fh.read()

extra_commands = {}

try:
    # noinspection PyUnresolvedReferences
    from privex.helpers import settings, BumpCommand, ExtrasCommand

    # This results in an absolute path to the folder where this setup.py file is contained
    BASE_DIR = dirname(abspath(__file__))

    # The file which contains "VERSION = '1.2.3'"
    settings.VERSION_FILE = join(BASE_DIR, 'privex', 'exchange', '__init__.py')

    extra_commands['extras'] = ExtrasCommand
    extra_commands['bump'] = BumpCommand
except (ImportError, AttributeError) as e:
    warnings.warn('Failed to import privex.helpers.setuppy.commands - the commands "extras" and "bump" may not work.')
    warnings.warn(f'Error Reason: {type(e)} - {str(e)}')

setup(
    name='privex_exchange',

    version=VERSION,

    description='A small library for querying Cryptocurrency exchanges and other price sources',
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/Privex/python-exchange",
    author='Chris (Someguy123) @ Privex',
    author_email='chris@privex.io',

    license='MIT',
    install_requires=[
        'httpx>=0.12.0',
        'attrs',
        'async-property',
        'privex-helpers[cache]>=2.18.0',
        'privex-steemengine>=2.1.0'
    ],
    packages=find_packages(exclude=['tests', 'tests.*', 'test.*']),
    cmdclass=extra_commands,
    extras_require={'dev': [
        'jupyter', 'wheel', 'setuptools', 'twine', 'privex-helpers[dev]', 'pytest', 'pytest-cov', 'coverage',
        'codecov', 'pytest-asyncio', 'nest-asyncio'
    ]},
    classifiers=[
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
)


from twisted.internet import defer

from scrapy.http import Request
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import create_spider_for_request
from scrapy.utils.python import stringify_dict
from scrapy import log

class ExecutionQueue(object):

    def __init__(self, spiders, queue, poll_interval=5, keep_alive=False):
        self.spider_requests = []
        self.poll_interval = poll_interval
        self._spiders = spiders
        self._queue = queue
        self._keepalive = keep_alive

    @defer.inlineCallbacks
    def _append_next(self):
        """Called when there are no more items left in self.spider_requests.
        This method is meant to be overriden in subclasses to add new (spider,
        requests) tuples to self.spider_requests. It can return a Deferred.
        """
        msg = yield self._queue.pop()
        if msg:
            name = msg.pop('name')
            msg = stringify_dict(msg) # see #250
            self.append_spider_name(name, **msg)

    def get_next(self):
        """Return a tuple (spider, requests) containing a list of Requests and
        the Spider which will be used to crawl those Requests. If there aren't
        any more spiders to crawl it must return (None, []).

        This method can return a deferred.
        """
        if self.spider_requests:
            return self._get_next_now()
        d = defer.maybeDeferred(self._append_next)
        d.addCallback(lambda _: self._get_next_now())
        return d

    def _get_next_now(self):
        try:
            return self.spider_requests.pop(0)
        except IndexError:
            return (None, [])

    def is_finished(self):
        """Return True if the queue is empty and there won't be any more
        spiders to crawl (this is for one-shot runs). If it returns ``False``
        Scrapy will keep polling this queue for new requests to scrape
        """
        return not self._keepalive and not bool(self.spider_requests)

    def append_spider(self, spider):
        """Append a Spider to crawl"""
        requests = spider.start_requests()
        self.spider_requests.append((spider, requests))

    def append_request(self, request, spider=None, **kwargs):
        if spider is None:
            spider = create_spider_for_request(self._spiders, request, **kwargs)
        if spider:
            self.spider_requests.append((spider, [request]))

    def append_url(self, url=None, spider=None, **kwargs):
        """Append a URL to crawl with the given spider. If the spider is not
        given, a spider will be looked up based on the URL
        """
        if url is None:
            raise ValueError("A url is required")
        if spider is None:
            spider = create_spider_for_request(self._spiders, Request(url), \
                **kwargs)
        if spider:
            requests = arg_to_iter(spider.make_requests_from_url(url))
            self.spider_requests.append((spider, requests))

    def append_spider_name(self, name=None, **spider_kwargs):
        """Append a spider to crawl given its name and optional arguments,
        which are used to instantiate it. The SpiderManager is used to lookup
        the spider
        """
        if name is None:
            raise ValueError("A spider name is required")
        try:
            spider = self._spiders.create(name, **spider_kwargs)
        except KeyError:
            log.msg('Unable to find spider: %s' % name, log.ERROR)
        except:
            log.err(None, 'Error creating spider %r with arguments: %r' % \
                (name, spider_kwargs))
        else:
            self.append_spider(spider)


import math

import config
import tags
from tags import ResultTag, WorldTag, MoveTag
from matplotlib import pyplot as plt
import numpy as np
from matplotlib import cm
from move import cell_exponent
from config import Params
from disk import load_all, quicksave, quickload, load_histories
from genetic_keras.data import EpochData

def display_results(results, params):
    result_params = results[ResultTag.PARAM]
    if params.is_plotting_trajectories:
        if ResultTag.POSITION in results:
            fig, ax = plt.subplots(1, 1)
            if params.is_plotting_area_units:
                if ResultTag.AREA_INDICES in results:
                    plot_area_units(ax, results[ResultTag.AREA_INDICES][0], result_params.area_unit_size)
                else:
                    print("Can't plot area units as there are no area indices saved in results.")

            plot_world(ax, result_params, results[ResultTag.POSITION])

            plot_trajectories(ax, results[ResultTag.POSITION])
        else:
            print("Can't plot position over time as there is no position data saved in results.")

    #plt.show()

def plot_world(ax, result_params, result_positions):
    min_x = np.amin(result_positions[:, :, :, 0])
    min_y = np.amin(result_positions[:, :, :, 1])
    max_x = np.amax(result_positions[:, :, :, 0])
    max_y = np.amax(result_positions[:, :, :, 1])

    if result_params.selected_world == WorldTag.CONVEX_CELLS:
        plot_world_convex_cells(ax, max_x, max_y, min_x, min_y, result_params)
    elif result_params.selected_world == WorldTag.CONCAVE_CELLS:
        plot_world_concave_cells(ax, max_x, max_y, min_x, min_y, result_params)

    if result_params.selected_mover == MoveTag.LEVY_OPTIMAL_ALPHA \
            or result_params.selected_mover == MoveTag.LEVY_VARYING_DELTA:
        plot_varying_delta(ax, result_params, min_x, max_x)

def plot_world_convex_cells(ax, max_x, max_y, min_x, min_y, result_params):
    cell_size = result_params.cell_size
    radius = result_params.obstacle_size * cell_size / 2

    min_x_cell = np.int64(min_x // cell_size)
    min_y_cell = np.int64(min_y // cell_size)
    max_x_cell = np.int64(max_x // cell_size)
    max_y_cell = np.int64(max_y // cell_size)

    for x_cell in range(int(min_x_cell), max_x_cell + 1):
        for y_cell in range(min_y_cell, max_y_cell + 1):
            center_x = x_cell * cell_size + cell_size / 2
            center_y = y_cell * cell_size + cell_size / 2
            poly_x = np.cos(np.linspace(0, 2 * np.pi)) * radius + center_x
            poly_y = np.sin(np.linspace(0, 2 * np.pi)) * radius + center_y

            ax.fill(poly_x, poly_y, 'gray')

def plot_world_concave_cells(ax, max_x, max_y, min_x, min_y, result_params):
    cell_size = result_params.cell_size
    x = result_params.obstacle_size
    h = np.sqrt((np.power(1 - x, 2) + 1) / 4)

    min_x_cell = np.int64(min_x // cell_size)
    min_y_cell = np.int64(min_y // cell_size)
    max_x_cell = np.int64(max_x // cell_size)
    max_y_cell = np.int64(max_y // cell_size)

    for x_cell in range(int(min_x_cell), max_x_cell + 1):
        for y_cell in range(min_y_cell, max_y_cell + 1):
            center_x = x_cell * cell_size + cell_size / 2
            center_y = y_cell * cell_size + cell_size / 2

            x_corners = [0.5, -0.5, -0.5, 0.5]
            y_corners = [0.5, 0.5, -0.5, -0.5]
            for i in range(4):
                unit_poly_x = np.cos(np.linspace(i * np.pi / 2, (i + 1) * np.pi / 2, 200)) * h
                bad_x = np.nonzero(abs(unit_poly_x) > 0.5)
                unit_poly_y = np.sin(np.linspace(i * np.pi / 2, (i + 1) * np.pi / 2, 200)) * h
                bad_y = np.nonzero(abs(unit_poly_y) > 0.5)
                bad = np.append(bad_x, bad_y)

                unit_poly_x = np.delete(unit_poly_x, bad)
                unit_poly_y = np.delete(unit_poly_y, bad)

                unit_poly_x = np.append(unit_poly_x, x_corners[i])
                unit_poly_y = np.append(unit_poly_y, y_corners[i])

                poly_x = unit_poly_x * cell_size + center_x
                poly_y = unit_poly_y * cell_size + center_y

                ax.fill(poly_x, poly_y, 'gray')

def plot_varying_delta(ax, result_params, min_x, max_x):
    x = np.linspace(min_x, max_x, math.ceil(max_x-min_x)*100)
    y = cell_exponent(x, result_params.world_width)

    ax.plot(x, y)

def plot_trajectories(ax, positions_over_time):
    ax.plot(positions_over_time[0, :, :, 0], positions_over_time[0, :, :, 1])
    ax.set_aspect('equal')
    ax.legend(["alpha = 1", "alpha = 1.5", "alpha = 2"])

def plot_area_units(ax, indices, area_unit_size):
    for pair in indices:
        base_x = pair[0] * area_unit_size
        base_y = pair[1] * area_unit_size
        x = [base_x, base_x + area_unit_size, base_x + area_unit_size, base_x]
        y = [base_y, base_y, base_y + area_unit_size, base_y + area_unit_size]

        ax.fill(x, y, 'green')

def plot_area_over_alpha(results):
    area = [np.mean(result[ResultTag.AREA]) for result in results]
    max_area = np.max(area)
    area = [x / max_area for x in area]
    alpha = [result[ResultTag.PARAM].alpha for result in results]

    num_repeats = results[0][ResultTag.PARAM].num_repeats
    num_agents = results[0][ResultTag.PARAM].num_agents

    fig, ax = plt.subplots()
    ax.scatter(alpha, area)
    ax.set_title(f"Explored area, mean over {num_repeats * num_agents} agents.")
    ax.set_xlabel("alpha")
    ax.set_ylabel("Normalized explored area.")
    plt.show()

def prepare_units_over_alpha_data(params: Params):
    result_list, _ = load_all(params)
    area_times_list = [x[ResultTag.AREA_TIME] for x in result_list]
    mean_areas = []
    for area_times in area_times_list:
        sum = 0
        for at in area_times:
            area = len(at)
            sum += area

        mean = sum / len(area_times)

        mean_areas.append(mean)
    params_list = [x[ResultTag.PARAM] for x in result_list]
    zipped = list(zip(params_list, mean_areas))
    single_alpha_data = list(filter(lambda it: it[0].selected_mover == MoveTag.LEVY_VARYING_DELTA_CONTRAST, zipped))

    x = [it[0].alpha for it in single_alpha_data]
    y = [it[1] for it in single_alpha_data]

    data = x, y
    quicksave(data, params, "units_v_alpha")
    return data

def plot_units_over_alpha(params, force_recalculation=False):
    data, success = quickload(params, "units_v_alpha")

    x, y = data if success and not force_recalculation else prepare_units_over_alpha_data(params)

    fig, ax = plt.subplots()
    ax.scatter(x, y)

    if params.selected_world == tags.WorldTag.CONVEX_CELLS:
        env_str = "convex"
    elif params.selected_world == tags.WorldTag.CONCAVE_CELLS:
        env_str = "concave"
    else:
        env_str = "homogenous"

    ax.set_title(f"{env_str} env, $r_0 = {params.tic_rate_0}$, $r_1 = {params.tic_rate_1}$")
    ax.set_xlabel("$\\alpha$")
    ax.set_ylabel("Area units discovered")
    return fig

def plot_top_contenders(params, force_recalculation=False):
    data, success = quickload(params, "top_contenders")

    best_alpha, best_alpha_data, genetic_data, instant_opti_data, single_alpha_data, slow_opti_data, worst_alpha, \
        worst_alpha_data = data if success and not force_recalculation else prepare_top_contender_data(params)

    fig, ax = plt.subplots()

    if len(single_alpha_data) > 0:
        plot_bar_category(ax, worst_alpha_data, 0)
        plot_bar_category(ax, best_alpha_data, 1)

    if len(slow_opti_data) > 0:
        plot_bar_category(ax, slow_opti_data, 2)

    if len(instant_opti_data) > 0:
        plot_bar_category(ax, instant_opti_data, 3)

    if len(genetic_data) > 0:
        plot_bar_category(ax, genetic_data, 4)

    labels = [f"$\\alpha_{{worst}}: {worst_alpha}$", f"$\\alpha_{{best}}: {best_alpha}$",
              "$L$", "$L_{s}$", "genetic"]
    ax.bar(range(5), np.zeros(5), tick_label=labels)

    ax.set_title(f"Performance of different strategies when: $r_0 = {params.tic_rate_0}, r_1 = {params.tic_rate_1}$")
    ax.set_ylabel("Area units discovered")

    #print(f"({params.tic_rate_0}, {params.tic_rate_1}) - alpha_worst: {np.mean([it[1] for it in worst_alpha_data]):.0f}, "
    #      f"alpha_best: {np.mean([it[1] for it in best_alpha_data]):.0f}, local: "
    #      f"{np.mean([it[1] for it in slow_opti_data]):.0f}, local_s:  {np.mean([it[1] for it in instant_opti_data]):.0f}, "
    #      f"genetic_mean: {np.mean([it[1] for it in genetic_data]):.0f}")

    m_aw = np.mean([it[1] for it in worst_alpha_data])
    m_ab = np.mean([it[1] for it in best_alpha_data])
    m_op = np.mean([it[1] for it in slow_opti_data])
    m_ops = np.mean([it[1] for it in instant_opti_data])
    m_gn = np.mean([it[1] for it in genetic_data])

    means = [m_aw, m_ab, m_op, m_ops, m_gn]
    colors = ["\\ca", "\\cb", "\\cc", "\\cd", "\\ce"]
    sort = np.argsort(np.argsort(-np.array(means)))
    str = ""
    for i in range(len(means)):
        str += f" & {colors[sort[i]]} {means[i]:.0f}"

    str = f"({params.tic_rate_0}, {params.tic_rate_1})" + str + "\\\\ \\hline"

    print(str)

def prepare_top_contender_data(params):
    result_list, _ = load_all(params)
    area_times_list = [x[ResultTag.AREA_TIME] for x in result_list]
    mean_areas = []
    for area_times in area_times_list:
        sum = 0
        for at in area_times:
            area = len(at)
            sum += area

        mean = sum / len(area_times)

        mean_areas.append(mean)
    params_list = [x[ResultTag.PARAM] for x in result_list]
    zipped = list(zip(params_list, mean_areas))
    genetic_data = list(filter(lambda it: it[0].selected_mover == MoveTag.DIRECT_TIMER, zipped))
    instant_opti_data = list(
        filter(lambda it: it[0].selected_mover == MoveTag.LEVY_OPTIMAL_ALPHA_CONTRAST_INSTANT_SWITCH, zipped))
    slow_opti_data = list(filter(lambda it: it[0].selected_mover == MoveTag.LEVY_OPTIMAL_ALPHA_CONTRAST, zipped))
    single_alpha_data = list(filter(lambda it: it[0].selected_mover == MoveTag.LEVY_VARYING_DELTA_CONTRAST, zipped))
    worst_alpha = "X"
    best_alpha = "X"
    if len(single_alpha_data) > 0:
        best_alpha = single_alpha_data[np.argmax([it[1] for it in single_alpha_data])][0].alpha
        worst_alpha = single_alpha_data[np.argmin([it[1] for it in single_alpha_data])][0].alpha
        best_alpha_data = list(filter(lambda it: it[0].alpha == best_alpha, single_alpha_data))
        worst_alpha_data = list(filter(lambda it: it[0].alpha == worst_alpha, single_alpha_data))

    data = best_alpha, best_alpha_data, genetic_data, instant_opti_data, single_alpha_data, slow_opti_data, worst_alpha, worst_alpha_data
    quicksave(data, params, "top_contenders")
    return data

def plot_bar_category(ax, data, offset):
    cmap = plt.get_cmap("tab10")
    bar_width = 0.7 / len(data) if len(data) != 0 else 1
    ax.bar(np.arange(len(data)) * bar_width + offset - (len(data)-1)/2*bar_width, [it[1] for it in data],
           bar_width, color=cmap(range(len(data))))

def plot_area_in_range(result_list, start_step, end_step, file_names=None, title=""):
    results = [(x[ResultTag.AREA_TIME], x[ResultTag.PARAM]) for x in result_list]

    mean_areas = []
    labels = []
    for area_times, params in results:
        sum = 0
        for at in area_times:
            area = np.sum(np.logical_and(start_step <= np.array(at), np.array(at) <= end_step))
            sum += area

        mean = sum / len(area_times)

        mean_areas.append(mean)
        labels.append(params.save_id)

    if file_names is not None:
        labels = [name[:-4] for name in file_names]

    fig, ax = plt.subplots()

    ax.bar(range(len(mean_areas)), mean_areas, tick_label=labels)

    ax.set_title(title)

def plot_many_area_at_time(result_list, time):
    fig, ax = plt.subplots()
    for result in result_list:
        params = result[ResultTag.PARAM]
        label = "alpha: " + str(params.alpha) if len(params.alpha_times) == 0 else "varying alpha"
        max_area = 20 ** 2 * np.pi

        plot_area_at_time(ax, result, time, max_area=max_area, label=label)

    ax.set_title(f"Explored area at time = {time}, mean over {params.num_repeats * params.num_agents} agents.")
    ax.set_ylabel("Normalized explored area.")
    plt.draw()

def plot_area_at_time(ax, results, target_time, max_area, label):
    area, time = get_area_time(max_area, results)

    index = np.argmax(np.array(time) >= target_time)

    ax.bar(label, area[index])

def plot_many_area_over_time(result_list, time=None):
    fig, ax = plt.subplots()

    if time is not None:
        ax.plot([time, time], [0, 1], 'k')

    i = 0
    for result in result_list:
        i += 1
        params = result[ResultTag.PARAM]
        label = "alpha: " + str(params.alpha) if len(params.alpha_times) == 0 else "varying alpha"
        line = "--" if i > 10 else "-"
        max_area = 20 ** 2 * np.pi

        plot_area_over_time(ax, result, max_area=max_area, label=label, line=line)

    ax.set_title(f"Explored area, mean over {params.num_repeats * params.num_agents} agents.")
    ax.set_xlabel("time")
    ax.set_ylabel("Normalized explored area.")
    ax.legend()
    plt.draw()

def plot_area_over_time(ax, results, max_area, label, line):
    area, time = get_area_time(max_area, results)

    ax.plot(time, area, line, label=label)

def get_area_time(max_area, results):
    params = results[ResultTag.PARAM]
    area = get_area_over_steps(max_area, results)
    if len(params.viscosity_times) > 0:
        time = get_varied_time(params)
    else:
        time = [step * params.delta_time for step in range(params.num_steps)]
    return area, time

def get_area_over_steps(max_area, results):
    params = results[ResultTag.PARAM]
    area_times = results[ResultTag.AREA_TIME]
    area = np.zeros(params.num_steps)
    for at in area_times:
        for step in at:
            area[step:] += 1
    area = area / max_area / len(area_times)
    return area

def get_varied_time(params):
    time = []
    clock = 0
    timer = 0
    i = 0
    base_speed = params.speed
    base_delta_t = 0.5  # TODO revert params.delta_time
    for step in range(params.num_steps):
        if timer == 0:
            speed_factor, timer = params.viscosity_times[i]
            delta_time = base_delta_t / speed_factor

            i = (i + 1) % len(params.viscosity_times)

        clock += delta_time
        timer -= 1
        time.append(clock)
    return time

def plot_alpha_delta_surface(results, highlighted=None, old_data=False, last_line=False):
    if old_data:
        X, Y, Z, counts = generate_surface_plot_data_old(results, last_line)
    else:
        X, Y, Z, counts = generate_surface_plot_data_new(results)

    x_opti, y_opti = plot_surface_helper(X, Y, Z, counts, highlighted)

    return x_opti, y_opti

def generate_surface_plot_data_new(results):
    params = [res[ResultTag.PARAM] for res in results]
    alphas = set()
    rs = set()
    for p in params:
        p: Params
        alphas.add(p.alpha)
        rs.add(p.tic_rate_0)
    alphas = list(alphas)
    alphas.sort()
    rs = list(rs)
    rs.sort()
    X = -np.ones((len(rs), len(alphas)))
    Y = -np.ones((len(rs), len(alphas)))
    Z = -np.ones((len(rs), len(alphas)))
    counts = np.zeros((len(rs), len(alphas)))
    for i in range(len(rs)):
        for j in range(len(alphas)):
            X[i, j] = rs[i]
            Y[i, j] = alphas[j]
            matching_results = list(filter(lambda it: it[ResultTag.PARAM].alpha == alphas[j]
                                                      and it[ResultTag.PARAM].tic_rate_0 == rs[i], results))

            count = 0
            mean = 0
            for r in matching_results:
                areas = r[ResultTag.AREA]
                num_a_units = np.array(areas) * 20 ** 2
                mean += np.mean(num_a_units)
                count += 1

            if count == 0:
                mean = -1
            else:
                mean = mean / count

            Z[i, j] = mean
            counts[i, j] = count

    return X, Y, Z, counts

def generate_surface_plot_data_old(results, last_line):
    params = [r[ResultTag.PARAM] for r in results]
    alphas = set()
    deltas = set()
    for p in params:
        p: Params
        alphas.add(p.alpha)
        deltas.add(p.delta_time)
    alphas = list(alphas)
    alphas.sort()
    deltas = list(deltas)
    deltas.sort()
    X = -np.ones((len(deltas), len(alphas)))
    Y = -np.ones((len(deltas), len(alphas)))
    Z = -np.ones((len(deltas), len(alphas)))
    counts = np.zeros((len(deltas), len(alphas)))
    for i in range(len(deltas)):
        for j in range(len(alphas)):
            X[i, j] = deltas[i] / 0.5
            Y[i, j] = alphas[j]
            matching_results = list(filter(lambda it: it[ResultTag.PARAM].alpha == alphas[j]
                                                      and it[ResultTag.PARAM].delta_time == deltas[i], results))

            count = 0
            mean = 0
            for r in matching_results:
                areas = r[ResultTag.AREA]
                num_a_units = np.array(areas) * 20 ** 2
                if last_line:
                    num_a_units = num_a_units[-1, :]
                mean += np.mean(num_a_units)
                count += 1

            if count == 0:
                mean = -1
            else:
                mean = mean / count

            Z[i, j] = mean
            counts[i, j] = count

    X = np.log2(X)
    return X, Y, Z, counts

def plot_surface_helper(X, Y, Z, counts, highlighted):
    max_indices = np.argmax(Z, axis=1)
    scatter_x = X[np.array(range(25)), max_indices]
    scatter_y = Y[np.array(range(25)), max_indices]
    if highlighted is None:
        string = ""
        for i in range(len(scatter_x)):
            string += str(scatter_x[i]) + ": " + "{:.2f}".format(scatter_y[i]) + ", "
        print(string)
    fig, ax = plt.subplots()
    c = ax.pcolor(X, Y, Z, shading="nearest")
    fig.colorbar(c, ax=ax, label="Area units discovered")
    if highlighted is not None:
        idx = np.array([np.where(it == scatter_x)[0] for it in highlighted])
        ax.scatter(scatter_x[idx], scatter_y[idx], s=12 ** 2, c='white', label='_nolegend_')
        ax.scatter(scatter_x[idx], scatter_y[idx], s=10 ** 2, c='black', label="Current environment")
    ax.scatter(scatter_x, scatter_y, c='red', label="Best alpha for each resistance")
    ax.set_xlabel("Resistance: $r$")
    ax.set_ylabel("$\\alpha$")
    #ax.set_title(f"Mean of {np.min(counts)*10000} simulated agents.")
    ax.legend()

    return scatter_x, scatter_y

def scatter_alpha_speed_surface(results):
    params = [r[ResultTag.PARAM] for r in results]
    areas = [r[ResultTag.AREA] for r in results]

    x = [np.log2(it.speed) for it in params]
    y = [it.alpha for it in params]
    z = [np.mean(it) for it in areas]

    fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
    ax.scatter(x, y, z)
    ax.set_xlabel("log2(speed)")
    ax.set_ylabel("alpha")
    ax.set_zlabel("area")

def plot_loss_over_time(results, rolling_mean=1):
    loss_collections = [r[ResultTag.LOSS] for r in results]
    agent_losses = [item for sublist in loss_collections for item in sublist]

    if rolling_mean > 1:
        agent_losses = [fast_moving_average(al, rolling_mean) for al in agent_losses]

    fig, ax = plt.subplots()
    for al in agent_losses:
        ax.plot(al)

def moving_average(list, n):
    averaged = np.zeros(len(list)+n)
    for i in range(len(list)):
        if list[i] is not None:

            averaged[i:i+n] += list[i]
    averaged /= n
    return averaged[n-1:-n]

def fast_moving_average(list, n):
    ret = np.cumsum([i for i in list if i], dtype=float)
    ret[n:] = ret[n:] - ret[:-n]
    return ret[n - 1:] / n

alpha_list = np.linspace(1, 2, 11)

def __create_memory(random_index, at, params: config.Params):
    memory = np.zeros(params.memory_length)
    for i in range(len(memory)):
        start = random_index - (params.memory_length-i)*params.memory_compression_factor
        end = start + params.memory_compression_factor
        memory[i] = sum(np.logical_and(at >= start, at < end))
    return memory

def generate_memory_examples(result):
    params: config.Params = result[ResultTag.PARAM]
    num_examples = 10
    memory_red_big_delta = np.zeros((len(alpha_list), num_examples, params.memory_length))
    memory_blue_small_delta = np.zeros((len(alpha_list), num_examples, params.memory_length))

    area_times = result[ResultTag.AREA_TIME]
    positions: np.ndarray = result[ResultTag.POSITION][0]

    for i in range(len(alpha_list)):
        at = np.array(area_times[i])
        x_pos = positions[:, i, 0]

        num_reds_saved = 0
        num_blues_saved = 0
        while num_reds_saved < num_examples or num_blues_saved < num_examples:
            random_index = math.floor((np.random.random_sample()*params.num_steps)-params.memory_length*params.memory_compression_factor) + params.memory_length*params.memory_compression_factor

            if np.floor(x_pos[random_index]) % 2 == 0:
                if num_reds_saved >= num_examples:
                    continue
                memory_red_big_delta[i, num_reds_saved] = __create_memory(random_index, at, params)
                num_reds_saved += 1

            else:
                if num_blues_saved >= num_examples:
                    continue
                memory_blue_small_delta[i, num_blues_saved, :] = __create_memory(random_index, at, params)
                num_blues_saved += 1

    return memory_red_big_delta, memory_blue_small_delta

def plot_example_analysis(result):
    area_times = result[ResultTag.AREA_TIME]
    positions: np.ndarray = result[ResultTag.POSITION][0]

    for i in range(len(area_times)):
        at = area_times[i]
        x_pos = positions[:, i, 0]
        at_x_pos = x_pos[at]
        at_color = ["red" if np.floor(x) % 2 == 0 else "blue" for x in at_x_pos]
        pos_color = ["red" if np.floor(x) % 2 == 0 else "blue" for x in x_pos]

        fig, ax = plt.subplots(1, 2)

        plot_world(ax[0], result[ResultTag.PARAM], result[ResultTag.POSITION])
        ax[0].scatter(positions[:, i, 0], positions[:, i, 1], c=pos_color, s=0.5)
        ax[0].set_aspect('equal')

        ax[1].scatter(at, range(len(at)), c=at_color, s=1)

        fig.suptitle("alpha = " + str(alpha_list[i]) + ", Red == large tic, Blue == low tic rate")
        print("wow")
    print("hej")

def plot_single_dist(ax, bin_size, bins, bin_count, title, color):
    x = bins + bin_size/2
    bin_count[1] += bin_count[0]
    y = bin_count[1:]/sum(bin_count)
    ax.bar(x, y, bin_size, color=color)

    ax.set_title(title)
    #ax.set_xlabel("time")
    #ax.set_ylabel("frequency")

def plot_single_dist_log(ax, times, title):
    bins = np.logspace(-3, np.log10(max(times)), 100)
    bin_indices = np.digitize(times, bins)
    bin_count = np.bincount(bin_indices)

    bin_sizes = np.insert(arr=bins[1:] - bins[:-1], obj=0, values=bins[0])
    bin_count = bin_count[:len(bin_sizes)]
    bin_sizes = bin_sizes[:len(bin_count)]

    relative_frequency = bin_count/bin_sizes
    relative_frequency = relative_frequency/sum(relative_frequency)
    x = np.repeat(np.insert(bins, 0, 0), 2)[1:-1]
    y = np.repeat(relative_frequency, 2)
    ax.loglog(x, y)

    ax.set_title(title)
    ax.set_xlabel("time")
    ax.set_ylabel("frequency")

def plot_distribution(params):
    qs_tag = "distribution"

    data, success = quickload(params, qs_tag)

    if success:
        plot_data_list = data
    else:
        plot_data_list = prepare_dist_data(params, qs_tag)

    fig, ax = plt.subplots(5, 3)

    cmap = plt.get_cmap("tab10")
    for i in range(len(plot_data_list)):
        big_plot_data = plot_data_list[i][0]
        small_plot_data = plot_data_list[i][1]
        combined_plot_data = (big_plot_data[0], big_plot_data[1], big_plot_data[2]+small_plot_data[2])

        color = cmap(i)
        plot_single_dist(ax[i, 0], *big_plot_data, "$r_0$" if i == 0 else "", color)
        plot_single_dist(ax[i, 1], *small_plot_data, "$r_1$" if i == 0 else "", color)
        plot_single_dist(ax[i, 2], *combined_plot_data, "$combined$" if i == 0 else "", color)

    fig.suptitle(f"$r_0 = {params.tic_rate_0}, r_1 = {params.tic_rate_1}$")
    fig.supxlabel("Effort")
    fig.supylabel("Frequency")
    fig.set_tight_layout(True)
    fig.set_size_inches(8, 10)

def prepare_dist_data(params, qs_tag):
    results, filenames = load_all(params)
    plot_data_list = []
    for r in results:
        if tags.ResultTag.DIST in r:
            big, small = r[tags.ResultTag.DIST]

            plot_data = []
            for times in [big, small]:
                num_bins = 30
                bin_size = max(times) / num_bins
                bins = np.linspace(0, max(times), num_bins + 1)
                bin_indices = np.digitize(times, bins)
                bin_count = np.bincount(bin_indices)
                plot_data.append((bin_size, bins, bin_count))

            plot_data_list.append(plot_data)
    quicksave(plot_data_list, params, qs_tag)
    return plot_data_list

def plot_single_cumdist(ax, times, title):
    bins = np.linspace(0, max(times), 100)
    bin_indices = np.digitize(times, bins)
    bin_count = np.bincount(bin_indices)
    bin_count[1] += bin_count[0]
    bin_count = bin_count[1:len(bins)]

    relative_frequency = bin_count / sum(bin_count)
    cumsum = np.cumsum(relative_frequency)
    x = np.repeat(bins, 2)[1:-1]
    y = 1 - np.repeat(np.insert(cumsum, 0, 0), 2)[:-2]
    ax.plot(x, y)

    ax.set_title(title)
    ax.set_xlabel("time")
    ax.set_ylabel("inverse cumulative frequency")

def plot_single_cumdist_log(ax, times, title):
    bins = np.logspace(-3, np.log10(max(times)), 100)
    bin_indices = np.digitize(times, bins)
    bin_count = np.bincount(bin_indices)
    bin_count = bin_count[:len(bins)]

    relative_frequency = bin_count / sum(bin_count)
    cumsum = np.cumsum(relative_frequency)
    x = np.repeat(np.insert(bins, 0, 0), 2)[1:-1]
    y = 1 - np.repeat(np.insert(cumsum, 0, 0), 2)[:-2]
    ax.loglog(x, y)

    ax.set_title(title)
    ax.set_xlabel("time")
    ax.set_ylabel("inverse cumulative frequency")

def plot_inverse_cumulative_distribution(results, title=""):
    big_tic_list = []
    small_tic_list = []

    for r in results:
        if tags.ResultTag.DIST in r:
            big, small = r[tags.ResultTag.DIST]
            big_tic_list.extend(big)
            small_tic_list.extend(small)

    fig, ax = plt.subplots(2, 3)

    plot_single_cumdist(ax[0, 0], big_tic_list, "Inverse cumulative distribution in high tic areas")
    plot_single_cumdist(ax[0, 1], small_tic_list, "Inverse cumulative distribution in low tic areas")
    plot_single_cumdist(ax[0, 2], big_tic_list + small_tic_list, "Combined inverse cumulative distribution")

    plot_single_cumdist_log(ax[1, 0], big_tic_list, "Inverse cumulative distribution in high tic areas")
    plot_single_cumdist_log(ax[1, 1], small_tic_list, "Inverse cumulative distribution in low tic areas")
    plot_single_cumdist_log(ax[1, 2], big_tic_list + small_tic_list, "Combined inverse cumulative distribution")

    fig.suptitle(title)

def plot_genetic_training_history(params):
    histories, dir_names = load_histories(params)
    score_histories = []
    for hist in histories:
        score_histories.append([it.median_loss_training for it in hist[0]])

    fig, ax = plt.subplots()
    for i in range(len(score_histories)):
        ax.plot(-np.array(score_histories[i])/0.05**2, label=dir_names[i])

    ax.legend()
    ax.set_xlabel("Generations")
    ax.set_ylabel("Area units discovered")

def plot_2d_comparison(x_bug, y_bug, x_many, y_many, x_single, y_single):

    fig, ax = plt.subplots()
    #ax.scatter(scatter_x, scatter_y, s=10 ** 2, c='black', label="Current environment")
    #ax[0].scatter(x_bug, y_bug, s=10 ** 2, label="Bugged")
    ax.scatter(x_many, y_many, s=7 ** 2, label="100 agents")
    ax.scatter(x_single, y_single, s=5 ** 2, label="Single agent")
    ax.set_xlabel("Resistance: $r$")
    ax.set_ylabel("$\\alpha$")
    #ax.set_title(f"Mean of {np.min(counts)} simulations.")
    ax.legend()


"""Summary statistics table.

Functions
---------
print_summary_statistics(data_dictionary, sigfigs=2, plusfigs=1, file=sys.stdout)

"""
import sys

import numpy as np
import tabulate

__author__ = "Randal J Barnes and Elizabeth Barnes"
__version__ = "30 October 2021"

def print_summary_statistics(data_dictionary, sigfigs=2, plusfigs=1, file=sys.stdout):
    """Print a summary statistics table for the data in the dictionary.

    The summary statistics include:
        "cnt" : count
        "min" : minimum value
        "25%" : 25 percentile
        "50%" : median
        "75%" : 75 percentile
        "max" : maximum value
        "avg" : arithmetic average
        "std" : standard deviation (sample)
        "iqr" : inter-quartile range

    Arguments
    ---------
    data_dictionary : dict
        The key:values pairs in the dictionary are name and data vector.
        The name is a str, the data vector is a ndarray of shape (n,).

    sigfigs : int, default=2
        The number of places to the right of the decimal point used to
        print the data order statistics.

    plusfigs : int, default=1
        The number of additional places to the right of the decimal point
        used to print the mean and standard deviation.

    file : file object, default=sys.stdout (i.e. the screen)
        Where to print the table.

    Returns
    -------
    None

    Usage
    -----
    print_summary_statistics({"y_train" : y_train, "y_val" : y_val})

    Notes
    -----
    * The computations do not filter out nan values.  So, if there are
        nan or inf values in the data vector, the resulting statistics
        will be impacted.

    * If the computation of any statistic fails and raises an exception,
        then the entire row in the table is set to nan.

    """
    # Make the table.
    table = []
    for key in data_dictionary:
        data_vector = data_dictionary[key]
        try:
            row = [
                key,
                np.shape(data_vector)[0],
                np.min(data_vector),
                np.percentile(data_vector, 25),
                np.median(data_vector),
                np.percentile(data_vector, 75),
                np.max(data_vector),
                np.mean(data_vector),
                np.std(data_vector),
                np.percentile(data_vector, 75) - np.percentile(data_vector, 25),
            ]
        except:
            row = [
                key,
                np.nan,
                np.nan,
                np.nan,
                np.nan,
                np.nan,
                np.nan,
                np.nan,
                np.nan,
                np.nan,
            ]
        table.append(row)

    # Print a table.
    print("\n", file=file)
    print(
        tabulate.tabulate(
            table,
            [" ", "cnt", "min", "25%", "50%", "75%", "max", "avg", "std", "iqr"],
            tablefmt="presto",
            floatfmt=(
                "s",
                ".0f",
                f".{sigfigs}f",
                f".{sigfigs}f",
                f".{sigfigs}f",
                f".{sigfigs}f",
                f".{sigfigs}f",
                f".{sigfigs+plusfigs}f",
                f".{sigfigs+plusfigs}f",
                f".{sigfigs}f",
            ),
        ),
        file=file,
    )
    print("\n", file=file)


import json
from unittest.mock import Mock

import numpy as np
import pytest

from atldld.sync import (
    DatasetDownloader,
    DatasetNotFoundError,
    RMAParameters,
    get_parallel_transform,
    pir_to_xy,
    xy_to_pir,
)

class TestGetParallelTransform:
    @pytest.mark.parametrize(
        "downsample_ref", [25, 50]
    )  # p, i, r are divisble by these
    def test_local_equals_API(self, pir_to_xy_response, downsample_ref):
        p = pir_to_xy_response["p"]
        i = pir_to_xy_response["i"]
        r = pir_to_xy_response["r"]

        axis = pir_to_xy_response["axis"]

        x = pir_to_xy_response["x"]
        y = pir_to_xy_response["y"]

        affine_2d = np.array(pir_to_xy_response["affine_2d"])
        affine_3d = np.array(pir_to_xy_response["affine_3d"])

        if axis == "coronal":
            assert i % downsample_ref == 0
            assert r % downsample_ref == 0

            slice_coordinate = p
            grid_shape = (8000 / downsample_ref, 11400 / downsample_ref)

        elif axis == "sagittal":
            assert p % downsample_ref == 0
            assert i % downsample_ref == 0

            slice_coordinate = r
            grid_shape = (13200 / downsample_ref, 8000 / downsample_ref)

        else:
            raise ValueError

        df = get_parallel_transform(
            slice_coordinate,
            affine_2d,
            affine_3d,
            axis=axis,
            downsample_ref=downsample_ref,  # the goal is to reduce computation
        )

        tx, ty = df.transformation

        assert tx.shape == grid_shape
        assert ty.shape == grid_shape

        # Asserts single pixel

        if axis == "coronal":
            i_ = int(i // downsample_ref)
            r_ = int(r // downsample_ref)

            x_pred, y_pred = tx[i_, r_], ty[i_, r_]
        elif axis == "sagittal":
            p_ = int(p // downsample_ref)
            i_ = int(i // downsample_ref)

            x_pred, y_pred = tx[p_, i_], ty[p_, i_]

        assert x_pred == pytest.approx(x, abs=1e-2)
        assert y_pred == pytest.approx(y, abs=1e-2)

class TestDatasetDownloader:
    def test_invalid_dataset(self, monkeypatch):
        # If `rma_all` returns an empty list, the API does not have any entries
        # satisfying the parameters

        mock_rma_all = Mock(return_value=[])
        monkeypatch.setattr("atldld.sync.rma_all", mock_rma_all)

        downloader = DatasetDownloader(
            dataset_id=434324132413241,
        )
        with pytest.raises(DatasetNotFoundError, match="does not seem to exist"):
            downloader.fetch_metadata()

    @pytest.mark.parametrize("include_expression", [True, False])
    def test_patched(self, include_expression, data_folder, monkeypatch):
        """Does not require internet, everything is patched."""

        # Parameters
        dataset_id = 123  # Sagittal dataset
        downsample_ref = 25
        grid_shape = (13200 // downsample_ref, 8000 // downsample_ref)

        # Mocking
        def mock_rma_all(value):
            parameters_dataset = RMAParameters(
                model="SectionDataSet",
                criteria={
                    "id": dataset_id,
                },
                include=["alignment3d"],
            )
            parameters_images = RMAParameters(
                model="SectionImage",
                criteria={
                    "data_set_id": dataset_id,
                },
                include=["alignment2d"],
            )
            if value == parameters_dataset:
                with open(data_folder / "rma_all" / "SectionDataSet.json") as f:
                    data = json.load(f)
                return [
                    data,
                    None,
                ]  # We extract only the first element of the list here.
            elif value == parameters_images:
                with open(data_folder / "rma_all" / "SectionImage.json") as f:
                    data = json.load(f)
                return [
                    data,
                ]  # In a list because can have several images
            else:
                raise ValueError("Not expected")

        fake_rma_all = Mock(side_effect=mock_rma_all)
        fake_get_image = Mock(return_value=np.zeros(grid_shape))
        monkeypatch.setattr("atldld.sync.rma_all", fake_rma_all)
        monkeypatch.setattr("atldld.sync.get_image", fake_get_image)

        # Call the function
        downloader = DatasetDownloader(
            dataset_id=dataset_id,
            include_expression=include_expression,
            downsample_ref=downsample_ref,
        )
        downloader.fetch_metadata()
        gen = downloader.run()

        # Asserts - first iteration
        x = next(gen)
        assert len(x) == 5

        img_id, slice_coordinate, img, img_expression, df = x

        assert img_id == 101945191
        if include_expression:
            assert isinstance(img_expression, np.ndarray)
        else:
            assert img_expression is None
        assert df.delta_x.shape == grid_shape
        assert df.delta_y.shape == grid_shape

        # Asserts remaining
        with pytest.raises(StopIteration):
            next(gen)

    def test_metadata(self):
        """Test metadata"""
        # Parameters
        dataset_id = 123
        include_expression = True
        downsample_ref = 25

        # Call the function
        downloader = DatasetDownloader(
            dataset_id=dataset_id,
            include_expression=include_expression,
            downsample_ref=downsample_ref,
        )

        with pytest.raises(RuntimeError):
            len(downloader)

        with pytest.raises(RuntimeError):
            gen = downloader.run()
            _ = next(gen)

        metadata = {"test": True}
        downloader.metadata = metadata
        downloader.fetch_metadata()
        # As metadata exists, check that did not fetch metadata
        assert downloader.metadata == metadata

        # Create very small metadata
        downloader.metadata = {}
        downloader.metadata["images"] = list(np.arange(10))
        downloader.metadata["dataset"] = {}
        downloader.metadata["dataset"]["plane_of_section_id"] = 3

        # Check that len is working correctly
        assert len(downloader) == 10

        # Raise ValueError because plane_of_section_id in {1, 2}
        with pytest.raises(ValueError, match="Unrecognized plane"):
            gen = downloader.run()
            next(gen)

        # Check that everything working fine if plane_of_section_id = 1
        downloader.metadata["dataset"]["plane_of_section_id"] = 1

def test_pir_to_xy(pir_to_xy_response):
    coords_ref = np.array(
        [
            pir_to_xy_response["p"],
            pir_to_xy_response["i"],
            pir_to_xy_response["r"],
        ]
    )[
        :, None
    ]  # (3, 1)

    coords_img_API = np.array([pir_to_xy_response["x"], pir_to_xy_response["y"]])[
        :, None
    ]  # We do not care about the section coordinate

    coords_img_local = pir_to_xy(
        coords_ref,
        np.array(pir_to_xy_response["affine_2d"]),
        np.array(pir_to_xy_response["affine_3d"]),
    )[:2]

    assert np.allclose(coords_img_API, coords_img_local, rtol=0, atol=1e-3)

def test_xy_to_pir(xy_to_pir_response):
    coords_img = np.array(
        [
            xy_to_pir_response["x"],
            xy_to_pir_response["y"],
            xy_to_pir_response["section_number"]
            * xy_to_pir_response["section_thickness"],
        ]
    )[
        :, None
    ]  # (3, 1)

    coords_ref_API = np.array(
        [
            xy_to_pir_response["p"],
            xy_to_pir_response["i"],
            xy_to_pir_response["r"],
        ]
    )[:, None]

    coords_ref_local = xy_to_pir(
        coords_img,
        np.array(xy_to_pir_response["affine_2d"]),
        np.array(xy_to_pir_response["affine_3d"]),
    )

    assert np.allclose(coords_ref_API, coords_ref_local, rtol=0, atol=1e-3)


#! /usr/bin/env python3

"""
Tests functionality of :class: `paper_sorts.DatabaseConnector`.
"""

import unittest
import logging

from paper_sorts.database_connector import DatabaseConnector
from paper_sorts.config_reader import ConfigReader

class DataBaseTest(unittest.TestCase):
    """
    Tests adding, searching, updating and deleting functionality, but doesn't cover a significant amount of code.
    """
    def test_search_by_author(self):
        """ Test if an entry know to be in the database can be found if searched for by author name."""
        config_reader = ConfigReader("../../database.crypt", "postgresql", "../../key")
        database = DatabaseConnector(
            config_reader.db_config,
            logging.DEBUG,
            "database_tester_logger",
            log_file="db_connector_test.log",
        )
        author_search = database.search_by_author("Pino, J.")
        self.assertEqual(
            author_search[0][3],
            "Large-scale Self- an Semi-Supervised learning for speech translation",
        )
        self.assertEqual(author_search[0][4], "Wang2021LargeScaleSA")
        self.assertRaises(KeyError, database.search_by_author, "no author")

    def test_search_by_title(self):
        """ Test if an entry know to be in the database can be found if searched for by publication title."""
        config_reader = ConfigReader("../../database.crypt", "postgresql", "../../key")
        database = DatabaseConnector(
            config_reader.db_config,
            logging.DEBUG,
            "database_tester_logger",
            log_file="db_connector_test.log",
        )
        self.assertEqual(
            database.search_by_title(
                "Direct speech-to-speech translation with discrete units"
            )[0][0],
            "Lee, Ann and Chen, Peng-Jen and Wang, Changhan and Gu, Jiatao and Ma, Xutai and Polyak, A. and Adi, Yossi "
            "and He, Qing and Tang, Yun and Pino, J. and Hsu, Wei-Ning",
        )
        self.assertRaises(KeyError, database.search_by_title, "no title")

    def test_adding_and_removing(self):
        """Test whether an entry can be added and removed from the database safely. """
        config_reader = ConfigReader("../../database.crypt", "postgresql", "../../key")
        database = DatabaseConnector(
            config_reader.db_config,
            logging.DEBUG,
            "database_tester_logger",
            log_file="db_connector_test.log",
        )

        self.assertRaises(
            ValueError,
            database.delete_paper_entry_from_database,
            "test",
            ["list_add_and_remove"],
            "x",
            "This is a remove test",
            "This is a test",
        )
        self.assertTrue(
            database.add_entry_to_db(
                "test",
                ["list_add_and_remove"],
                "x",
                "This is an add test",
                "This is a test",
            )
        )
        self.assertRaises(
            ValueError,
            database.add_entry_to_db,
            "test",
            ["list"],
            "x",
            "This is an add test",
            "This is a test",
        )
        self.assertTrue(
            database.delete_paper_entry_from_database(
                "test",
                ["list_add_and_remove"],
                "x",
                "This is an add test",
                "This is a test",
            )
        )

    def test_update_title(self):
        """Test whether the summary of an entry in the database can be updated safely. """
        config_reader = ConfigReader("../../database.crypt", "postgresql", "../../key")
        database = DatabaseConnector(
            config_reader.db_config,
            logging.DEBUG,
            "database_tester_logger",
            log_file="db_connector_test.log",
        )
        database.add_entry_to_db(
            "test",
            ["list_update_title"],
            "x",
            "This is an update title test",
            "This is a test",
        )
        paper_id = database.database_handler.fetch_from_db(
            "select id from papers where title='This is an update title test';"
        )[0][0]
        database.update_entry(
            "title",
            "updated title",
            "papers",
            paper_id
        )
        self.assertEqual(
            database.search_by_title(
                "updated title"
            )[0][0],
            "list_update_title",
        )
        database.update_entry(
            "contents",
            "updated contents",
            "papers",
            paper_id
        )
        self.assertEqual(
            database.database_handler.fetch_from_db(
                "select contents from papers where id=%s;",
                (paper_id, )
            )[0][0],
            "updated contents"
        )
        database.delete_paper_entry_from_database(
            "test",
            ["list_update_title"],
            "x",
            "updated title",
            "updated contents",
        )
        self.assertRaises(
            ValueError,
            database.update_entry,
            "test",
            "should not work",
            "papers",
            "This is a test",
        )
        self.assertRaises(
            ValueError,
            database.update_entry,
            "test",
            "should not work",
            "non-table",
            "This is a test",
        )

    def test_update_authors_papers(self):
        """ Test whether the author-paper relation of an entry cannot be changed."""
        config_reader = ConfigReader("../../database.crypt", "postgresql", "../../key")
        database = DatabaseConnector(
            config_reader.db_config,
            logging.DEBUG,
            "database_tester_logger",
            log_file="db_connector_test.log",
        )
        self.assertRaises(
            ValueError,
            database.update_entry,
            "test",
            "should not work",
            "authors_papers",
            "This is a test",
        )

    def test_update_authors(self):
        """ Test whether the authorship of a paper can be changed as expected.."""
        config_reader = ConfigReader("../../database.crypt", "postgresql", "../../key")
        database = DatabaseConnector(
            config_reader.db_config,
            logging.DEBUG,
            "database_tester_logger",
            log_file="db_connector_test.log",
        )

        database.add_entry_to_db(
            "test",
            ["list_update_authors"],
            "x",
            "This is an update author test",
            "This is a test",
        )

        author_id = database.database_handler.fetch_from_db(
                "select id from authors_id where author='list_update_authors'",
        )[0][0]
        papers = database.database_handler.fetch_from_db(
                "select paper_id from authors_papers where author_id=%s;",
                (author_id, )
        )
        database.update_entry(
            "author",
            "changed_authors",
            "authors_id",
            "list_update_authors"
        )
        author_id = database.database_handler.fetch_from_db(
                "select id from authors_id where author='changed_authors'",
        )[0][0]

        self.assertEqual(
            database.database_handler.fetch_from_db(
                "select  paper_id from authors_papers where author_id=%s;",
                (author_id, )
            ),
            papers
        )
        self.assertTrue(
            database.delete_paper_entry_from_database(
                "test",
                ["changed_authors"],
                "x",
                "This is an update author test",
                "This is a test",
            )
        )
        database.add_entry_to_db(
            "test",
            ["list_update_authors"],
            "x",
            "This is a test number 2",
            "This is a test",
        )
        database.add_entry_to_db(
            "another test",
            ["new list"],
            "u",
            "This is a another test",
            "something",
        )
        database.update_entry(
            "author",
            "new list",
            "authors_id",
            "list_update_authors"
        )
        self.assertTrue(
            database.delete_paper_entry_from_database(
                "test",
                ["new list"],
                "x",
                "This is a test number 2",
                "This is a test",
            )
        )
        self.assertTrue(
            database.delete_paper_entry_from_database(
                "another test",
                ["new list"],
                "u",
                "This is a another test",
                "something",
            )
        )
        self.assertRaises(
            ValueError,
            database.update_entry,
            "nonexistent column",
            "new list",
            "authors_id",
            "list_update_authors"
        )

    def test_update_bib(self):
        """ Test whether the content of one bibliography entry can be changed. """
        config_reader = ConfigReader("../../database.crypt", "postgresql", "../../key")
        database = DatabaseConnector(
            config_reader.db_config,
            logging.DEBUG,
            "database_tester_logger",
            log_file="db_connector_test.log",
        )
        database.add_entry_to_db(
            "test",
            ["list"],
            "x",
            "This is a bib test",
            "This is a test",
        )
        database.update_entry(
            "bibtex",
            "y",
            "bib",
            "x"
        )
        self.assertEqual(
            database.database_handler.fetch_from_db(
                "select bibtex from bib where bibtex_id='x';"
            )[0][0],
            "y"
        )
        self.assertRaises(
            ValueError,
            database.update_entry,
            "bibtex",
            "y",
            "bib",
            "x"
        )
        self.assertTrue(
            database.delete_paper_entry_from_database(
                "y",
                ["new list"],
                "x",
                "This is a bib test",
                "This is a test",
            )
        )
        self.assertRaises(
            ValueError,
            database.update_entry,
            "nonexistent",
            "y",
            "bib",
            "x"
        )

if __name__ == "__main__":
    unittest.main()