#!/usr/bin/env python
# -*- coding: UTF-8 -*-
__author__ = "NeMuX"
__copyright__ = "Copyright 2014"
__credits__ = ["NeMuX"]
__license__ = "GPL"
__version__ = "0.1"
__maintainer__ = "NeMuX"
__email__ = "jaraujo@globalcybersec.com"
__status__ = "Development"

import sys
import csv
import sqlite3

def cargar_vulnerabilidades(file,db='info/vulnerabilidades.db'):

    conn = sqlite3.connect(db)
    c = conn.cursor()

    conn.text_factory = "str"

    # Creamos la tabla

    c.execute('''CREATE TABLE IF NOT EXISTS vulnerabilidades
             (id INTEGER PRIMARY KEY AUTOINCREMENT, nessus_id INTEGER, qualys_id TEXT, acunetix_id TEXT,
              nombre_vulnerabilidad TEXT,descripcion TEXT, solucion TEXT, workaround TEXT,
              categoria TEXT, efecto TEXT, prioridad TEXT, cvss TEXT, plataforma TEXT,
              subplataforma TEXT, cve TEXT, referencias TEXT, exploit TEXT, estado TEXT,
              patrones TEXT, revisada NUMERIC, severidad NUMERIC)''')

    cvs_file = open(file)
    reader = csv.reader(cvs_file)

    rownum = 0
    colnum = 0

    for r in reader:
        #ignoramos el encabezado
        if rownum == 0:
            colnum = len(r)
            rownum = rownum + 1

        else:
            #Formamos la cadena del INSERT

            #query = "INSERT INTO vulnerabilidades (nessus_id, qualys_id, acunetix_id, nombre_vulnerabilidad, descripcion, solucion, workaround, " \
            #        "categoria, efecto, prioridad, cvss, plataforma, subplataforma, cve, referencias, exploit, estado, patrones, revisada) " \
            #        "VALUES (" + r[1] + ",\"" + r[3] + "\", \"" + r[4] + "\", \"" + r[6] + "\", \"" + r[7] + "\", \"" + r[8] + "\", \"" + r[9] \
            #        + "\", \"" + r[10] + "\", \"" + r[11] + "\", \"" + r[12] + "\", \"" + r[13] + "\", \"" + r[14] + "\", \"" + r[15] + "\", \"" + \
            #        r[16] + "\", \"" + r[17] + "\", \"" + r[18] + "\", \"" + r[19] + "\", \"" + r[20] + "\", \"" + r[21] + "\")"
            #print(query)
            #c.execute(query)

            #data = (r[1],r[3],r[4],r[6],r[7],r[8],r[9],r[10],r[11],r[12],r[13],r[14],r[15],r[16],r[17],r[18],r[19],r[20],r[21],r[22])
            data = (r[0],r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9],r[10],r[11],r[12],r[13],r[14],r[15],r[16],r[17],r[18],r[19])
            #print(str(data) + "\n")
            c.execute('''INSERT INTO vulnerabilidades (nessus_id, qualys_id, acunetix_id, nombre_vulnerabilidad, descripcion, solucion, workaround,
            categoria, efecto, prioridad, cvss, plataforma, subplataforma, cve, referencias, exploit, estado, patrones, revisada, severidad)
            VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',data)
            rownum = rownum + 1
    conn.commit()
    c.close()
    conn.close()
    print(str(rownum-1) + " Registros insertados...")

if __name__ == "__main__":
    if len(sys.argv)  > 1 :
        print("Procesando " + str(len(sys.argv) -1) + " archivos")
        for file in sys.argv[1:]:
            print("Cargando archivo -> " + file)
            cargar_vulnerabilidades(file)
        print("Hecho.")
    else:
        print(sys.argv[0] + ": falta un archivo como argumento" )


from django.conf.urls import url,include
from django.contrib import admin
from twapp import views

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^accounts/', include('allauth.urls')),
    url(r'^about/',views.about_page),
    url(r'^$',views.main_page),
    url(r'^tweets/',views.twitter_page),
    url(r'^videos/$',views.video_page),
    url(r'^download/$',views.video_download_page),
    url(r'^wiki/$',views.wiki_page),
    url(r'^news/$',views.news_page),
]


'''
Created on 20 Nov 2010

@author: jdrumgoole
'''
import os
import stat
import logging
from itertools import izip

from enum import Enum

from filetools.checksum import Checksum

from basetools.feedback import SimpleFeedback

def normalisePath( p ):
    return os.path.normpath( os.path.normcase( p ))

def splitPath( p ):

    pathElements = []
    ( tail, head ) = os.path.split( normalisePath( p ))

    while len( head) > 0 :
        pathElements.append( head )
        ( tail, head ) = os.path.split( tail )

    pathElements.reverse()

    return pathElements

class DirUtils( object ):

    def __init__( self, d ):
        '''
        Make a DirUtil object that provides utility functions over a single directory
        d = directory name (a string)
        '''

        self._d = d
        self._files = os.listdir( self._d )
        self._current = 0

    def __iter__(self):
        return self

    def refresh(self):
        self._files = os.listdir( self._d )

    def next(self):
        if self._current == len( self._files ) :
            self._current = 0
            raise StopIteration
        else:
            rv = self._files[ self._current ]
            self._current += 1
            return rv

    def dirname(self):
        '''
        return the dirname
        '''
        return self._d

    def __len__(self):
        '''
        Size of a directory in terms of no of files
        '''
        return len( self._files )

    def fileList(self, sortedList = False):
        '''
        Return a list of all the file (i.e. no dir) objects in a directory
        Don't sort return value unless sorted is true.
        '''
        result = []
        for i in self.listAll( sortedList = sortedList ):
            if os.path.isfile( i ):
                result.append( i )

        return result

    def dirList(self, sortedList = False):
        result = []
        for i in self.listAll( sortedList = sortedList ):
            if os.path.isdir( i ):
                result.append( i )

        return result

    def size(self):
        s=0
        for f in self.fileList() :
            s = s + os.path.getsize( f )

        return s

    def fileCount(self):
        return len( self.fileList())

    def dirCount(self):
        return len( self.dirList())

    def listAll(self, fullPath=True, sortedList = False ):
        result = []
        for i in os.listdir( self._d) :
            if fullPath :
                path = os.path.abspath( os.path.join( self._d, i ))
            else:
                path = i
            result.append( path )

        if sortedList :
            result.sort()

        return result

class FileCompare( object ):

    def __init__(self, f1, f2 ):
        self._f1 = normalisePath( f1 )
        self._f2 = normalisePath( f2 )

    def pathCompare( self ):
        return self._f1 == self._f2

    def reverseCompare(self):
        p1 = splitPath( self._f1 )
        p2 = splitPath( self._f2 )

        p1.reverse()
        p2.reverse()

        matchCount = 0
        for i, j in izip( p1, p2 ):
            if i == j :
                matchCount = matchCount + 1
            else:
                break

        p1.reverse() # put list back in order for return
        if matchCount > 0 :
            return p1[-matchCount:]
        else:
            return []

    def forwardCompare(self):
        '''
        Compare two target files from root down. Return number of path
        elements that are identical.
        '''

        p1 = splitPath( self._f1 )
        p2 = splitPath( self._f2 )

        matchCount = 0
        for i, j in izip( p1, p2 ):
            if i == j :
                matchCount = matchCount + 1
            else:
                break

        if matchCount > 0 :
            return p1[:matchCount]
        else:
            return []

    def sizeCompare(self):

        return os.path.getsize( self._f1 ) == os.path.getsize( self._f2 )

    def checksumCompare(self):
        return Checksum().blockComputeFile( self._f1 ) == Checksum().blockComputeFile(self._f2 )

    def binCompare(self):
        if self.pathCompare() : # identical files
            return True
        elif self.sizeCompare() and self.checksumCompare():
            return True
        else:
            return False

class DiffSet:

    def __init__(self, rhs, lhs):
        self._smallest = min( rhs, lhs )
        self._lhsSet = frozenset( lhs )
        self._rhsSet = frozenset( rhs )
        self._isEqual = ( self._lhsSet == self._rhsSet )
        self._lhsDiff = self._lhsSet.difference( self._rhsSet )
        self._rhsDiff = self._rhsSet.difference( self._lhsSet )

    def lhsDiff(self):
        return self._lhsDiff

    def rhsDiff(self):
        return self._rhsDiff

    def __len__(self):
        return len( self._lhsSet ) + len( self._rhsSet )

class DirCompare( object ) :
    '''
    Compare to directorys. Dirs are parsed during object creation
    '''

    def __init__(self, d1, d2 ) :

        self._d1 = DirUtils( d1 )
        self._d2 = DirUtils( d2 )
        self._d1List = self._d1.listAll(fullPath=False)
        self._d2List = self._d2.listAll(fullPath=False)
        self._d1List.sort()
        self._d2List.sort()

    def lengthCompare(self):
        return len( self._d1 ) == len( self._d2 )

    def filesCompare(self):
        pass

    def dirsCompare(self):
        pass

    def sizeCompare(self):

        return self._d1.size() == self._d2.size()

    def nameCompare(self):
        '''
        Compare two directories and return a list of files that match within the directory.
        For two identical directories the len(d1) = len(d2) = len( foundList) and
        sort( os.listDir( d1 ) = sort( os.listdir( d2)) = sort( foundList )
        '''

        ds = DiffSet( self._d1, self._d2 )

        return ds

class LogCmd :

    def __init__(self, logFilename, commentChar = "#" ):

        self._log = logging.getLogger( "LOGCMD" )
        self._log.setLevel( logging.INFO )
        self._LogFilename = logFilename
        self._commentChar = commentChar
        self._handler = logging.handlers.RotatingFileHandler(self._logFilename,
                                                             maxBytes =  10000,
                                                             backupCount = 3 )

        formatString = "%s %s : %s" % ("%(asctime)s", "%(name)s", "%(message)s",   )
        self._formatter = logging.Formatter( formatString )
        self._handler.setFormatter( self._formatter )
        self._log.addHandler( self._handler )

    def name(self):
        return self._logFilename

    def logCmd(self, s ):
        self._log.info( s )

    def logComment( self, c ):
        comment = "%s %s" % ( self._commentChar, c )
        self._log.info( comment )

class FileAttrs :

    DIR      ="dir"
    CHAR     ="char"
    FIFO     ="fifo"
    REGULAR  ="regular"
    BLOCK    = "block"
    LINK     = "link"
    SOCKET   = "socket"

    fileType = ( DIR, CHAR, FIFO, REGULAR, BLOCK, LINK, SOCKET )

    def __init__( self, name=None, size=0, computeChecksum=True ) :

        self._fileType = Enum( "dir",
                               "char",
                               "fifo",
                               "regular",
                               "block",
                               "link",
                               "socket" )

        self._name = name
        self._size = size
        self._creationDate = None
        self._modificationDate = None
        self._accessDate = None
        self._checksum = None
        self._type = None

        if name :
            st = os.stat( name )

            if stat.S_ISDIR( st[ stat.ST_MODE ]):
                self._type = self._fileType.dir
            elif stat.S_ISREG(st[ stat.ST_MODE ]):
                self._type = self._fileType.regular
                self._size = st[ stat.ST_SIZE ]
                if computeChecksum :
                    self._checksum =  Checksum().sha256( name )
            elif stat.S_ISLNK( st[ stat.ST_MODE ]):
                self._type = self._fileType.link
            elif stat.S_ISBLK( st[ stat.ST_MODE ]):
                self._type = self._fileType.block
            elif stat.S_ISSOCK( st[ stat.ST_MODE ]):
                self._type = self._fileType.sock
            elif stat.S_ISCHR( st[ stat.ST_MODE ]) :
                self._type = self._fileType.char
            elif stat.S_ISFIFO( st[ stat.ST_MODE ]) :
                self._type = self._fileType.char

            self._creationDate     = st[ stat.ST_CTIME ]
            self._modificationDate = st[ stat.ST_MTIME ]
            self._accessDate       = st[ stat.ST_ATIME ]

            #Sha256 from zero length file computed from http://www.xorbin.com/tools/sha256-hash-calculator
            self._zeroHash         = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
    def fileType(self):
        return self._type

    def creationDate(self):
        return self._creationDate

    def modificationDate(self):
        return self._modificationDate

    def accessDate(self ):
        return self._accessDate

    def size( self ):
        return self._size

    def isDir(self):
        return self._type == self._fileType.dir

    def isFile(self):
        return self._type == self._fileType.regular

    def dirname(self):
        return os.path.dirname( self._name )

    def basename(self):
        return os.path.basename( self._name )

    def name(self ):
        return self._name

    def setSize(self, size ):
        self._size = size

    def setName(self, name ):
        self._f = name

    def setChecksum(self, sum ):
        self._checksum = sum

    def checksum(self):
        if self._checksum == None :
            raise ValueError( "Checksum not set for %s" % self._name )
        return self._checksum

class FileOps :

    def __init__(self, bufferSize = 10000,
                 logCmd=None, flush = None,
                 feedback = SimpleFeedback() ):

        self._bufSize = bufferSize
        self._log = logCmd
        self._flush = flush
        self._feedback = feedback

    def isEqual(self, f1, f2 ):

        a1 = FileAttrs( f1 )
        a2 = FileAttrs( f2 )

        if a1.isDir() and a2.isDir() :
            return a1.basename() == a2.basename()

        if  a1.isFile() and a2.isFile() :
            return (( a1.basename() == a2.basename()) and
                    self.checksumIsEqual(f1, f2))
        else :
            return False

    def copyblock(self, src, sink, xfer  ):
        b = src.read( self._bufSize )
        sink.write( b )
        if self._flush:
            sink.flush()

        return xfer + len( b )

    def copyfile( self, f1, f2 ):

        xfer = 0

        srcAttrs = FileAttrs( f1 )
        with open( f1, "rb" ) as src:
            with open( f2, "wb" ) as sink:

                blocks = srcAttrs.size() / self._bufSize
                remnant= srcAttrs.size() % self._bufSize

                if remnant > 0 :
                    blocks = blocks + 1
                for i in xrange( blocks ):
                    xfer = self.copyblock( src, sink, xfer )
                    if self._feedback :
                        self._feedback.progress( f2, srcAttrs.size(), xfer )

        if self._log:
            self._log.logcopy( srcAttrs, FileAttrs( f2 ))

    def makedir(self, d  ):
        #print "makedir: '%s'" % d
        os.makedirs( d )
        if self._log:
            self._log.logmkdir( d )

    def checksumIsEqual(self, f1, f2 ):

        chk = Checksum()

        if os.path.isfile( f1 ) and os.path.isfile( f2 ) :
            return (( os.path.getsize( f1 ) == os.path.getsize( f2 )) and
                    ( chk.sha256( f2 ) == chk.sha256( f2 )))
        else:
            return False


"""mysite URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from checkbug.views import validatelogin,customers,programmer,solution, Solved, logout,delete,edit,fedit,insertdata
urlpatterns = [
    path('about/', validatelogin),
    path('about/client', customers),
    path('about/client/delete', delete),
    path('about/client/edit', edit),
    path('about/client/edit/fedit', fedit),
    path('about/programmer', programmer),
    path('about/programmer/solution', solution),
    path('about/programmer/Solved', Solved),
    path('admin/', admin.site.urls),
    path('about/client/logout', logout),
    path('about/client/insertdata', insertdata)
]


from setuptools import setup, find_packages

from bibtextools import __version__, __author__, __email__

with open("README.md") as rm:
    long_desc = rm.read()

#with open("requirements.txt") as req:
#    requirements = req.read().splitlines()

setup(
    name = "bibtextools",
    version = __version__,
    author = __author__,
    author_email = __email__,
    description = "Tools for working with BibTeX files",
    long_description_content_type="text/markdown",
    long_description=long_desc,
    keywords="bibtex latex biber bibliography",
    license='GPLv3',
    url='https://github.com/klb2/bibtex-tools',
    project_urls={
        #'Documentation': "https://digcommpy.readthedocs.io/",
        'Source Code': 'https://github.com/klb2/bibtex-tools',
        },
    packages=find_packages(),
    tests_require=['pytest', 'tox'],
    python_requires='>=3.7',
    install_requires=['bibtexparser',
                      'feedparser',
                      'click',
                     ],
    entry_points={"console_scripts": ["bibtex-tools=bibtextools.__main__:main"]},
)


"""Racemate URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, re_path, include
from rest_framework import routers
from rest_framework.schemas import get_schema_view

from api.views import MyUserViewSet, RunningGroupViewSet, PastTrainingViewSet
from calc.views import CalculatorView
from group.views import RunningGroupView, MemberView, CreateGroupView, ShowGroupsView, JoinGroupView, JoinConfirmView, \
    AdminConfirmView, AdminView, ShowAdminView
from messanger.views import ForumView, ForumChoiceView, SendMessageView, SendMessageGroupView, MessangerView, \
    MessangerAllView
from racemate.views import (LogoutView, LoginView, LandingView, LandingGeneratorView,
                            RegisterView, customhandler404, customhandler500, EditUserView, ContactView, AboutView)
from training.views import AddTrainingView, AddTreningView, TreningPlanWhiteView, TreningPlan18weeksView, \
    LoadTreningView, PlanChoiceView, DeleteTrainingView, PastTrainingDelete

from django.conf import settings
from django.conf.urls.static import static

router = routers.DefaultRouter()

router.register(r'users', MyUserViewSet)
router.register(r'groups', RunningGroupViewSet)
router.register(r'pasttrainings', PastTrainingViewSet)

schema_view = get_schema_view(title='Racemate API')

handler404 = customhandler404
handler500 = customhandler500

urlpatterns = [

    # racemate
    path('admin/', admin.site.urls),
    path('accounts/login/', LoginView.as_view(), name="login"),
    path('logout/', LogoutView.as_view(), name='logout'),
    path('register/', RegisterView.as_view(), name='register'),
    path('', LandingView.as_view(), name='landing-page'),
    re_path(r'start/(?P<id>(\d)+)/$', LandingGeneratorView.as_view(), name='generate_vdot'),
    path('edituser/', EditUserView.as_view(), name='edituser'),
    path('contact/', ContactView.as_view(), name='contact'),
    path('about/', AboutView.as_view(), name='about'),

    # group
    re_path(r'running_group/(?P<id>(\d)+)/$', RunningGroupView.as_view(), name='running-group'),
    re_path(r'member/(?P<id>(\d)+)/$', MemberView.as_view(), name='member'),
    path('creategroup/', CreateGroupView.as_view(), name='create-group'),
    path('showgroups/', ShowGroupsView.as_view(), name='show-groups'),
    path('joingroup/', JoinGroupView.as_view(), name='join-group'),
    re_path(r'joingroupconfirm/(?P<id>(\d)+)/$', JoinConfirmView.as_view(), name='join-group-confirm'),
    re_path(r'adminconfirm/(?P<id>(\d)+)/(?P<sender>(\d)+)$', AdminConfirmView.as_view(), name='adminconfirm'),
    re_path(r'adminview/(?P<id>(\d)+)$', AdminView.as_view(), name='adminview'),
    path('showadmin/', ShowAdminView.as_view(), name='showadmin'),

    # messanger
    re_path(r'forum/(?P<id>(\d)+)/$', ForumView.as_view(), name='forum'),
    path('forumchoice', ForumChoiceView.as_view(), name='forum-choice'),
    path('send_message/', SendMessageView.as_view(), name='sendmessage'),
    path('send_message_group', SendMessageGroupView.as_view(), name='sendmessagegroup'),
    re_path(r'^messanger/(?P<id>(\d)+)/$', MessangerView.as_view(), name='messanger'),
    # path('messangerall/', MessangerAllView.as_view(), name='messanger_all'),

    # training
    path('add_training/', AddTrainingView.as_view(), name='addtraining'),
    path('add_plan/', AddTreningView.as_view(), name='addplan'),  # nie zrobione
    path('treningplan_white/', TreningPlanWhiteView.as_view(), name='whiteplan'),
    path('treningplan_18weeks/', TreningPlan18weeksView.as_view(), name='18weeksplan'),
    path('loadtrening/', LoadTreningView.as_view(), name='loadtrening'),
    path('planchoice/', PlanChoiceView.as_view(), name='planchoice'),
    re_path(r'^deletetraining/(?P<id>(\d)+)/$', DeleteTrainingView.as_view(), name='deletetraining'),
    re_path(r'^(?P<pk>\d+)/delete/$', PastTrainingDelete.as_view(), name='delete'),

    # calc
    path('calculator/', CalculatorView.as_view(), name='calculator'),

    # api

    re_path(r'^api/v1/', include(router.urls), name='API'),
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),
    path('schema/', schema_view),

] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
# urlpatterns = format_suffix_patterns(urlpatterns)


import os
import sys

from flask import Flask, jsonify, request, abort, send_file
from dotenv import load_dotenv
from linebot import LineBotApi, WebhookParser
from linebot.exceptions import InvalidSignatureError
from linebot.models import MessageEvent, TextMessage, TextSendMessage, TemplateSendMessage, ButtonsTemplate, MessageAction

from fsm import TocMachine
from utils import send_text_message

# Let Flask set the environment variable depending on .env file.
load_dotenv()

machine_list = []
count_machine = 0
user_list = []

def create_new_machine():

    # Create a machine instance with the states configured.
    machine = TocMachine(
        states=["start_state", "state_for_menu", "state_for_homework_management", "state_for_exam_management", "state_for_reminder_management",
            "state_for_add_homework", "state_for_delete_homework", "state_for_examine_homework", "state_for_add_exam", "state_for_delete_exam", "state_for_examine_exam",
            "state_for_add_reminder", "state_for_delete_reminder", "state_for_examine_reminder"],
        transitions=[
            {
                # "Initial state" to "selection state".
                "trigger": "go_to_menu",
                "source": "start_state",
                "dest": "state_for_menu",
            },
            {
                # "selection state" to "homework management state".
                "trigger": "go_to_homework_management",
                "source": "state_for_menu",
                "dest": "state_for_homework_management",
            },
            {
                # "selection state" to "exam management state".
                "trigger": "go_to_exam_management",
                "source": "state_for_menu",
                "dest": "state_for_exam_management",
            },
            {
                # "selection state" to "reminder management state".
                "trigger": "go_to_reminder_management",
                "source": "state_for_menu",
                "dest": "state_for_reminder_management",
            },
            {
                # "homework management state" to "add homework state".
                "trigger": "go_to_add_homework",
                "source": "state_for_homework_management",
                "dest": "state_for_add_homework",
            },
            {
                # "homework management state" to "examine homework state".
                "trigger": "go_to_examine_homework",
                "source": "state_for_homework_management",
                "dest": "state_for_examine_homework",
            },
            {
                # "homework management state" to "delete homework state".
                "trigger": "go_to_delete_homework",
                "source": "state_for_homework_management",
                "dest": "state_for_delete_homework",
            },
            {
                # "exam management state" to "add exam state".
                "trigger": "go_to_add_exam",
                "source": "state_for_exam_management",
                "dest": "state_for_add_exam",
            },
            {
                # "exam management state" to "examine exam state".
                "trigger": "go_to_examine_exam",
                "source": "state_for_exam_management",
                "dest": "state_for_examine_exam",
            },
            {
                # "exam management state" to "delete exam state".
                "trigger": "go_to_delete_exam",
                "source": "state_for_exam_management",
                "dest": "state_for_delete_exam",
            },
            {
                # "reminder management state" to "add reminder state".
                "trigger": "go_to_add_reminder",
                "source": "state_for_reminder_management",
                "dest": "state_for_add_reminder",
            },
            {
                # "reminder management state" to "examine reminder state".
                "trigger": "go_to_examine_reminder",
                "source": "state_for_reminder_management",
                "dest": "state_for_examine_reminder",
            },
            {
                # "reminder management state" to "delete reminder state".
                "trigger": "go_to_delete_reminder",
                "source": "state_for_reminder_management",
                "dest": "state_for_delete_reminder",
            },
            {
                # Back to "homework management state".
                "trigger": "go_back_to_homework_management",
                "source": ["state_for_add_homework", "state_for_examine_homework", "state_for_delete_homework"],
                "dest": "state_for_homework_management",
            },
            {
                # Back to "exam management state".
                "trigger": "go_back_to_exam_management",
                "source": ["state_for_add_exam", "state_for_examine_exam", "state_for_delete_exam"],
                "dest": "state_for_exam_management",
            },
            {
                # Back to "reminder management state".
                "trigger": "go_back_to_reminder_management",
                "source": ["state_for_add_reminder", "state_for_examine_reminder", "state_for_delete_reminder"],
                "dest": "state_for_reminder_management",
            },
            {
                # Back to "state_for_menu".
                "trigger": "go_back_to_menu",
                "source": ["state_for_homework_management", "state_for_exam_management", "state_for_reminder_management", "state_for_add_homework",
                        "state_for_examine_homework", "state_for_delete_homework", "state_for_add_exam", "state_for_examine_exam", "state_for_delete_exam",
                        "state_for_add_reminder", "state_for_examine_reminder", "state_for_delete_reminder"],
                "dest": "state_for_menu"
            },
            {
                # Back to "initial state".
                "trigger": "go_back",
                "source": ["state_for_menu", "state_for_homework_management", "state_for_exam_management", "state_for_reminder_management", "state_for_add_homework",
                        "state_for_examine_homework", "state_for_delete_homework", "state_for_add_exam", "state_for_examine_exam", "state_for_delete_exam",
                        "state_for_add_reminder", "state_for_examine_reminder", "state_for_delete_reminder"],
                "dest": "start_state"
            },
        ],
        initial="start_state",
        auto_transitions=False,
        show_conditions=True,
    )

    # Add new machine to machine list.
    global machine_list
    machine_list.append(machine)
    global count_machine
    count_machine = count_machine + 1

# Use Flask to create a web application.
app = Flask(__name__, static_url_path="")

# get channel_secret and channel_access_token from your environment variable
channel_secret = os.getenv("LINE_CHANNEL_SECRET", None)
channel_access_token = os.getenv("LINE_CHANNEL_ACCESS_TOKEN", None)
if channel_secret is None:
    print("Specify LINE_CHANNEL_SECRET as environment variable.")
    sys.exit(1)
if channel_access_token is None:
    print("Specify LINE_CHANNEL_ACCESS_TOKEN as environment variable.")
    sys.exit(1)

# Create a LineBotApi instance.
line_bot_api = LineBotApi(channel_access_token)
parser = WebhookParser(channel_secret)

#***************************************************************************************

# Bind url "webhook" to this function.
@app.route("/webhook", methods=["POST"])
def webhook_handler():
    # Get the signature.
    signature = request.headers["X-Line-Signature"]

    # Get request body as text.
    body = request.get_data(as_text=True)
    app.logger.info(f"Request body: {body}")

    # Parse webhook body.
    try:
        events = parser.parse(body, signature)
    except InvalidSignatureError:
        abort(400)

    # If event is MessageEvent and message is TextMessage, then echo text.
    for event in events:
        if not isinstance(event, MessageEvent):
            continue
        if not isinstance(event.message, TextMessage):
            continue
        if not isinstance(event.message.text, str):
            continue

        #print(f"\nFSM STATE: {machine.state}")
        print(f"REQUEST BODY: \n{body}")

        # Check whether the user is registered.
        global user_list
        is_user_found = False
        user_index = (-1)
        for index in range(len(user_list)):
            if user_list[index] == event.source.user_id:
                is_user_found = True
                user_index = index

        # Add a new machine or not.
        if is_user_found == False:
            user_list.append(event.source.user_id)
            create_new_machine();

        # Set index for this user.
        global count_machine
        if is_user_found == False:
            user_index = count_machine - 1;

        # Check whether the event results in state transition.
        if event.message.text.lower() == "reset":
            machine_list[user_index].reset_states(event)
        elif machine_list[user_index].state == "start_state":
            machine_list[user_index].check_user_start(event)
        elif machine_list[user_index].state == "state_for_menu":
            machine_list[user_index].check_state_for_operation_menu(event)
        elif machine_list[user_index].state == "state_for_homework_management":
            machine_list[user_index].check_state_for_homework_operation(event)
        elif machine_list[user_index].state == "state_for_exam_management":
            machine_list[user_index].check_state_for_exam_operation(event)
        elif machine_list[user_index].state == "state_for_reminder_management":
            machine_list[user_index].check_state_for_reminder_operation(event)
        elif machine_list[user_index].state == "state_for_add_homework":
            machine_list[user_index].add_homework(event)
        elif machine_list[user_index].state == "state_for_add_exam":
            machine_list[user_index].add_exam(event)
        elif machine_list[user_index].state == "state_for_add_reminder":
            machine_list[user_index].add_reminder(event)
        elif machine_list[user_index].state == "state_for_examine_homework":
            machine_list[user_index].examine_homework(event)
        elif machine_list[user_index].state == "state_for_examine_exam":
            machine_list[user_index].examine_exam(event)
        elif machine_list[user_index].state == "state_for_examine_reminder":
            machine_list[user_index].examine_reminder(event)
        elif machine_list[user_index].state == "state_for_delete_homework":
            machine_list[user_index].delete_homework(event)
        elif machine_list[user_index].state == "state_for_delete_exam":
            machine_list[user_index].delete_exam(event)
        elif machine_list[user_index].state == "state_for_delete_reminder":
            machine_list[user_index].delete_reminder(event)

    return "OK"

#***************************************************************************************

# Bind url "show-fsm" to this function.
@app.route("/show-fsm", methods=["GET"])
def show_fsm():
    machine.get_graph().draw("fsm.png", prog="dot", format="png")
    return send_file("fsm.png", mimetype="image/png")

# If the script is executed positively instead of called by other script, run following code.
if __name__ == "__main__":
    port = os.environ.get("PORT", 8000)
    app.run(host="0.0.0.0", port=port, debug=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Extrae las lineas de texto como imágenes de un archivo HDF5 generado por el paquete
calamardo y ejecuta el Tesseract sobre ellas, generando una transcripción que
a su vez es almacenada en el HDF5
@author: nacho
"""
#
# paquetes base de Python3 (ya vienen con Python)
#
import time
import argparse
import subprocess
import os

# paquetes adicionales
#
import numpy as np
import skimage.io as imgio
#
# propios
#
import calamardo
import corrector

#---------------------------------------------------------------------------------------

def execute_tesseract(image,tmpdir='/tmp'):
    image = 255*(1-image) # binary to 8 bit grayscale
    imgio.imsave(os.path.join(tmpdir,'line.tif'),(255*image).astype(np.uint8))
    #command = 'tesseract --dpi 400 --psm 7 -l spa /tmp/line.tif /tmp/line'
    command = f'tesseract --dpi 400 --psm 13 -l spa  {tmpdir}/line.tif {tmpdir}/line'
    subprocess.run(command, shell=True,capture_output=True)
    with open(os.path.join(tmpdir,'line.txt'),'r') as f:
        line = f.readline().strip()
        return line

#---------------------------------------------------------------------------------------

if __name__ == '__main__':
    #
    # ARGUMENTOS DE LINEA DE COMANDOS
    #
    ap = argparse.ArgumentParser()
    ap.add_argument("--file", type=str, default="",
		    help="HDF5 file to process")
    ap.add_argument("--outdir", type=str, default='',
		    help="if specified, save training data in specified directory")
    ap.add_argument("--dict", type=str,
                    default="/datos/luisa/diccionarios/todo.txt",
                    help="Dictionary of valid words")
    ap.add_argument("--corpus", type=str,
                    default="../data/benito_perez_galdos_all.txt",
                    help="Text corpus to train language model")
    ap.add_argument("--tmpdir", type=str, default='/tmp',
		    help="directory to store temporary files")
    ap.add_argument("--force", type=bool, default=False,
		    help="overwrite previously generated transcriptions")
    #
    # INICIALIZACION
    #

    args = vars(ap.parse_args())
    hdf5_file = args["file"]
    outdir = args["outdir"]
    #
    if not len(hdf5_file):
        exit(1)

    file    = calamardo.CalamariFile(hdf5_file,'r+')
    n = file.len()

    save_dir  = len(outdir) > 0
    if save_dir:
        os.makedirs(outdir,exist_ok=True)

    corr = corrector.Corrector(args["dict"])
    accepted = 0
    total    = 0
    i = 0
    emptyimgcounter = 0
    for i in range(n):
        t0 = time.time()
        image = file.get_image(i)
        if not np.prod(image.shape) or np.max(image) == np.min(image):
            print('empty image')
            emptyimg = True
            emptyimgcounter += 1
            if emptyimgcounter >= 100:
                print('no more images.')
                break
            else:
                continue
        else:
            emptyimgcounter = 1

        raw_text = execute_tesseract(image,tmpdir=args['tmpdir'])
        print(f'{i:6d}:{raw_text:60s} --> ',end='')
        corrected_text, total_error, error_code = corr.correct_line(raw_text,ignore_tails=True)
        max_error = max(1,len(raw_text) // 10)
        print(f'{corrected_text:60s} | largo {len(raw_text):3d} maxerr {max_error:3d} err {total_error:3d}',end=' ')
        if total_error > max_error:
            print('-DISCARD- (garbage)')
            file.set_transcript(i,"<<DISCARDED>>")
        elif error_code == corrector.TOO_FEW_WORDS:
            print('-DISCARD- (too few words)')
            file.set_transcript(i,"<<DISCARDED>>")
        elif error_code == corrector.TOO_MANY_SHORT:
            print('-DISCARD- (too many short words)')
            file.set_transcript(i,"<<DISCARDED>>")
        else:
            print('-ACCEPT-')
            accepted += 1
            if save_dir:
                imgio.imsave(os.path.join(outdir,f'{i:08d}.tif'),(255*image).astype(np.uint8))
                ftxt = open(os.path.join(outdir,f'{i:08d}.gt.txt'),'w')
                print(corrected_text,file=ftxt)
                ftxt.close()

            file.set_transcript(i,corrected_text)
        total += 1

    paccepted = int(100.0 * accepted / total)
    pdiscarded = 100 - paccepted
    print('total lines ',total, ' accepted ',accepted, '(',paccepted,') discarded',total-accepted,'(',pdiscarded,')' )
    file.close()
    #
    # fin main
    #

#---------------------------------------------------------------------------------------


#everyone but freshman
import glob
import pandas as pd
import os
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from scipy import stats
from scipy.stats import f_oneway
from scipy.stats import tukey_hsd
import csv
from scikit_posthocs import posthoc_tukey
from statannotations.Annotator import Annotator
import math

def by_grades_2022(all_hrv):
    all_hrv_df = pd.read_excel(all_hrv)
    #all_hrv_df[['quality', 'HR', 'AVNN', 'SDNN','RMSSD','PNN50']] = all_hrv_df[['quality', 'HR', 'AVNN', 'SDNN','RMSSD','PNN50']].apply(pd.to_numeric)
    only_freshman = all_hrv_df[(all_hrv_df['date'].dt.year == 2022) & (all_hrv_df['grade'] == 2026)]
    #group the sport column by its unique values and count the occurrences
    counts = only_freshman.groupby('Sport').size()
    # identify 'Sports' values that appear less than 4 times
    mask = only_freshman['Sport'].isin(counts[counts < 3].index)
    # replace those values with 'other'
    only_freshman.loc[mask, 'Sport'] = 'Other'
    #replace "No_Sport" with "other"
    only_freshman.loc[only_freshman['Sport'] == 'No_Sport', 'Sport'] = 'Other'

    no_freshman = all_hrv_df[(all_hrv_df['date'].dt.year == 2022) & (all_hrv_df['grade'].isin([2025, 2024, 2023]))]
    #group the sport column by its unique values and count the occurrences
    counts = no_freshman.groupby('Sport').size()
    # identify 'Sports' values that appear less than 4 times
    mask = no_freshman['Sport'].isin(counts[counts < 3].index)
    # replace those values with 'other'
    no_freshman.loc[mask, 'Sport'] = 'Other'

    return only_freshman, no_freshman

def create_pie_chart(all_hrv):
    all_hrv_df = pd.read_excel(all_hrv)
    # extract year and create new column
    all_hrv_df['date_year'] = all_hrv_df['date'].dt.year
    all_hrv_df['name'] = all_hrv_df['name'].str.lower().str.replace(' ', '')
    # Group by name and year, and count the number of entries
    counts = all_hrv_df.groupby(['name', 'date_year']).size().unstack(fill_value=0)
    counts = pd.DataFrame(counts)
    counts.index.name = 'name'
    counts.columns.name = 'date_year'
    # Reset index to convert 'name' from index to a regular column
    counts = counts.reset_index()
    # Rename the columns to the desired format
    # count the number of rows that meet the specified conditions
    num_2021_gt_0_and_2022_eq_0 = len(counts[(counts[2021] > 0) & (counts[2022] == 0)])
    num_2021_eq_0_and_2022_gt_0 = len(counts[(counts[2021] == 0) & (counts[2022] > 0)])
    num_2021_and_2022_gt_0 = len(counts[(counts[2021] > 0) & (counts[2022] > 0)])
    # Print the results
    print(f"Number of rows with 2021 > 0 and 2022 = 0: {num_2021_gt_0_and_2022_eq_0}")
    print(f"Number of rows with 2021 = 0 and 2022 > 0: {num_2021_eq_0_and_2022_gt_0}")
    print(f"Number of rows with 2021 > 0 and 2022 > 0: {num_2021_and_2022_gt_0}")
    # numbers of subjects in each category
    numbers = [num_2021_gt_0_and_2022_eq_0, num_2021_eq_0_and_2022_gt_0, num_2021_and_2022_gt_0]
    # Labels for each number
    labels = ['Only 2021\n'+'('+str(num_2021_gt_0_and_2022_eq_0)+')', 'Only 2022\n'+'('+str(num_2021_eq_0_and_2022_gt_0)+')', 'Both years\n'+'('+str(num_2021_and_2022_gt_0)+')']
    # Creating the pie chart
    fig0, ax0 = plt.subplots(figsize=(10,10))
    ax0.pie(numbers, labels=labels, autopct='%1.1f%%')
    # Adding a title
    ax0.set_title('Subjects by Data Collection Year')
    fig0.savefig('years_piechart.png')
    #By gender
    fig1, ax1 = plt.subplots(figsize=(10,10))
    gender_counts = all_hrv_df['gender'].value_counts()
    fig1, ax1 = plt.subplots(figsize=(10,10))
    gender_counts = gender_counts.rename(index={'female':'Female','male':'Male','unknown': 'N.A'})
    ax1.pie(gender_counts, labels=gender_counts.index, autopct='%1.1f%%',)
    ax1.set_title('Gender Distribution')
    fig1.savefig('gender_piechart.png')
    # Sport pie chart
    sport_counts = all_hrv_df['Sport'].value_counts()
    all_hrv_df['Sport_reduced'] = all_hrv_df['Sport'].apply(lambda x: x if sport_counts[x] >= 15 else 'Other')
    counts_reduced = all_hrv_df['Sport_reduced'].value_counts()
    fig2, ax2 = plt.subplots(figsize=(10,10))
    ax2.pie(counts_reduced, labels=counts_reduced.index, autopct='%1.1f%%', labeldistance=1.1, pctdistance=0.8)
    ax2.text(-0.65, 1.37, '* Sports with less than 15 entries are in the "Other" category', fontsize=10, color='grey')
    ax2.set_title('Sports Distribution')
    plt.savefig('sports_piechart3.png')
    plt.show()

def group_data_by_variable(DataFrame, group_var, hr_var):
    grouped_data = []
    unique_groups = np.unique(DataFrame[group_var])
    if group_var == 'Sport':
        idx = np.where(unique_groups == 'Other')[0][0]
        unique_groups = np.concatenate((np.delete(unique_groups, idx), ['Other']))
        sport_order = {'Boys Basketball': 0, 'Girls Basketball': 1,'Boys Soccer': 2,
        'Girls Soccer': 3,'Cross Country': 4, 'Field Hockey': 5,'Football': 6,'Ice Hockey': 7,'Squash': 8,
        'Tennis': 9,'Volleyball': 10,'Water Polo': 11,'Basketball': 12, 'No_Sport': 13, 'Other': 14}
        unique_groups = sorted(unique_groups, key=lambda s: sport_order[s])
    for group in unique_groups:
        grouped_data.append(DataFrame[DataFrame[group_var] == group][hr_var])

    return unique_groups, grouped_data

def stats_and_plotting(unique_groups,grouped_data,DataFrame,hr_var,group_var, counter):
    if str(max(DataFrame['grade'])) == '2025':
        grades = 'no_freshman_'
        title_grades = 'No Freshman'
    else:
        grades = 'only_freshman_'
        title_grades = 'Only Freshman'
    save_path = '/Users/felixbrener/Documents/violin_swarm/'
    f_result= stats.f_oneway(*grouped_data)
    plt.figure(counter, figsize=(13, 8))
    counter = counter + 1
    #removing erronoius measurements
    #double check if there's a second zero for rmssd
    if hr_var == 'PNN50':
        DataFrame = DataFrame[DataFrame[hr_var] >= 0]
        minimumval = ' values less than\nzero '
    elif hr_var == 'RMSSD':
        DataFrame = DataFrame[DataFrame['AVNN'] > 0]
        minimumval = ' values less than\nzero '
    else:
        DataFrame = DataFrame[DataFrame[hr_var] > 0]
        minimumval = ' values of zero or\nless '
    #check again for values that should go in the other category
    #group the sport column by its unique values and count the occurrences
    counts = DataFrame.groupby('Sport').size()
    # identify 'Sports' values that appear less than 4 times
    mask = DataFrame['Sport'].isin(counts[counts < 3].index)
    # replace those values with 'other'
    DataFrame.loc[mask, 'Sport'] = 'Other'
    #if statement to check of all the unique grousp still exist in the data set
    #reseting unique groups now that more values have been removed
    unique_groups = np.unique(DataFrame[group_var])
    if group_var == 'Sport':
        sport_order = {'Boys Basketball': 0, 'Girls Basketball': 1,'Boys Soccer': 2,
        'Girls Soccer': 3,'Cross Country': 4, 'Field Hockey': 5,'Football': 6,'Ice Hockey': 7,'Squash': 8,
        'Tennis': 9,'Volleyball': 10,'Water Polo': 11,'Basketball': 12, 'No_Sport': 13, 'Other': 14}
        unique_groups = sorted(unique_groups, key=lambda s: sport_order[s])
    if  f_result.pvalue < 0.05:
        #Performing Tukey's HSD test
        tukey_df = posthoc_tukey(DataFrame, val_col=hr_var, group_col=group_var)
        #checking result of posthoc_tukey's
        #comverting the matrix to a non-redundant list of comparisons with the p-value
        remove = np.tril(np.ones(tukey_df.shape), k=0).astype("bool")
        tukey_df[remove] = np.nan
        #removing the lower half and diagonal of the matrix and turning the matrix format into a long dataframe
        molten_df = tukey_df.melt(ignore_index=False).reset_index().dropna()
        #filtering the rows where the value is less than 0.05
        filtered_rows = molten_df[molten_df['value'] < 0.05]
        #creatign swarm plots
        ax = sns.violinplot(data=DataFrame, x=group_var, y=hr_var, order=unique_groups, alpha = 0.5)
        ax = sns.swarmplot(data=DataFrame, x=group_var, y=hr_var, order=unique_groups, alpha = 0.7, color = '#FFFFFF', edgecolor='black')
        #ax = sns.swarmplot(data=DataFrame, x=group_var, y=hr_var, order=unique_groups, alpha = 0.65)
        # add pointplot with average values
        ax = sns.pointplot(data=DataFrame, x=group_var, y=hr_var, color="black", markers="d", scale=1.5, join=False, ci=None, size=10, order=unique_groups, linewidth=2, label='Group Average')

        if hr_var == 'RMSSD' or hr_var =='SDNN' or hr_var =='AVNN':
            ax.set(xlabel =group_var, ylabel = hr_var + "(ms)", title = hr_var +' by '+group_var)
        elif hr_var == 'HR':
            ax.set(xlabel =group_var, ylabel = hr_var + "(bpm)", title = hr_var +' by '+group_var)
        elif hr_var == 'PNN50':
            ax.set(xlabel =group_var, ylabel = hr_var + "(percentage)", title = hr_var +' by '+group_var)
        else:
            ax.set(xlabel =group_var, ylabel = hr_var, title = hr_var +' by '+group_var)
        if group_var == 'gender':
            ax.set(xlabel='Gender')
        ax.text(0, 1.006, "p-value annotation legend:\nNot Significant: p > 5.00e-02\n*: 1.00e-02 < p <= 5.00e-02\n**: 1.00e-03 < p <= 1.00e-02\n***: 1.00e-04 < p <= 1.00e-03\n****: p <= 1.00e-04", transform=ax.transAxes, fontsize=9)
        ax.text(0.8, 1.006, "n = " + str(len(DataFrame)) + " subjects\nAll subjects with "+str(hr_var)+ minimumval + "have been removed from the dataset", transform=ax.transAxes, fontsize=9)
        if len(filtered_rows) > 0:
            pairs = [(i[1]["index"], i[1]["variable"]) for i in filtered_rows.iterrows()]
            p_values = [i[1]["value"] for i in filtered_rows.iterrows()]

            annotator = Annotator(ax, pairs, data=DataFrame, x=group_var, y=hr_var, order=unique_groups)
            annotator.configure(text_format="star", loc="inside")
            annotator.set_pvalues_and_annotate(p_values)

        plt.title(hr_var +' by '+ group_var+'\n'+title_grades)
        if len(unique_groups) > 10:
            plt.xticks(rotation=15)
        if group_var == 'birth_year':
            labels_fixed = list(unique_groups[:-1]) + ['unknown']
            plt.xticks(range(len(unique_groups)), labels_fixed)
        # double the number of y-axis tic marks if there are less than 9
        if len(ax.get_yticks()) < 10:
            tick_interval = (ax.get_yticks()[-1] - ax.get_yticks()[0]) / (len(ax.get_yticks()) - 1)
            new_ticks = [ax.get_yticks()[0] + i*tick_interval/2 for i in range(2*len(ax.get_yticks())-1)]
            ax.set_yticks(new_ticks)
        plt.axhline(y=np.mean(DataFrame[hr_var]), color='#00E665', linestyle='--', label='Average '+ hr_var)
        # count the number of data points in each group
        counts = DataFrame[group_var].value_counts()
        # create a dictionary to map the counts to the group names
        count_dict = dict(zip(counts.index, counts.values))
        # add the counts to the x-tick labels
        if group_var == 'birth_year':
            unique_groups = ['unknown' if x == 2023 else x for x in unique_groups]
            count_dict['unknown'] = count_dict.pop(2023)
        xtick_labels = [f'{name}\n({count_dict[name]})' for name in unique_groups]
        ax.set_xticklabels(xtick_labels)
        plt.legend()
        if counter == 30:
            plt.show()
        else:
            plt.show(block = False)

    else:
        print('Unable to reject hypothesis')
        ax = sns.violinplot(data=DataFrame, x=group_var, y=hr_var, order=unique_groups, alpha = 0.5)
        ax = sns.swarmplot(data=DataFrame, x=group_var, y=hr_var, order=unique_groups, alpha = 0.7, color = '#FFFFFF', edgecolor='black')
        #ax = sns.swarmplot(data=DataFrame, x=group_var, y=hr_var, order=unique_groups, alpha = 0.65)
        # add pointplot with average values
        ax = sns.pointplot(data=DataFrame, x=group_var, y=hr_var, color="black", markers="d", scale=1.5, join=False, ci=None, size=10, order=unique_groups, linewidth=2, label='Group Average')

        if hr_var == 'RMSSD' or hr_var =='SDNN' or hr_var =='AVNN':
            ax.set(xlabel =group_var, ylabel = hr_var + "(ms)", title = hr_var +' by '+group_var)
        elif hr_var == 'HR':
            ax.set(xlabel =group_var, ylabel = hr_var + "(bpm)", title = hr_var +' by '+group_var)
        elif hr_var == 'PNN50':
            ax.set(xlabel =group_var, ylabel = hr_var + "(percentage)", title = hr_var +' by '+group_var)
        else:
            ax.set(xlabel =group_var, ylabel = hr_var, title = hr_var +' by '+group_var)
        if group_var == 'gender':
            ax.set(xlabel='Gender')
        plt.title(hr_var +' by '+ group_var+'\n'+title_grades)
        ax.text(0.89, 1.006, "n = " + str(len(DataFrame)) + " subjects", transform=ax.transAxes, fontsize=9)
        if len(unique_groups) > 10:
            plt.xticks(rotation=15)
        if group_var == 'birth_year':
            labels_fixed = list(unique_groups[:-1]) + ['unknown']
            plt.xticks(range(len(unique_groups)), labels_fixed)
        # double the number of y-axis tic marks if there are less than 9
        if len(ax.get_yticks()) < 10:
            tick_interval = (ax.get_yticks()[-1] - ax.get_yticks()[0]) / (len(ax.get_yticks()) - 1)
            new_ticks = [ax.get_yticks()[0] + i*tick_interval/2 for i in range(2*len(ax.get_yticks())-1)]
            ax.set_yticks(new_ticks)
        plt.axhline(y=np.mean(DataFrame[hr_var]), color='#00E665', linestyle='--', label='Average '+ hr_var)
        # count the number of data points in each group
        counts = DataFrame[group_var].value_counts()
        # create a dictionary to map the counts to the group names
        count_dict = dict(zip(counts.index, counts.values))
        # add the counts to the x-tick labels
        if group_var == 'birth_year':
            unique_groups = ['unknown' if x == 2023 else x for x in unique_groups]
            count_dict['unknown'] = count_dict.pop(2023)
        xtick_labels = [f'{name}\n({count_dict[name]})' for name in unique_groups]
        ax.set_xticklabels(xtick_labels)
        plt.legend()
        if counter == 30:
            plt.show(block = True)
        else:
            plt.show(block = False)
    # save the figure
    fig_name = "violin_swarm_" + grades + str(counter) + '.png'  # change the file name as per your preference
    plt.savefig(save_path + fig_name)
    return counter + 1

if __name__ == '__main__':

    all_hrv = '/Users/felixbrener/Documents/ARC/all_hrv.xlsx'
    hrv_list = ['HR','AVNN', 'SDNN', 'RMSSD','PNN50']
    group_list = ['Sport','birth_year','gender']
    counter = 0

#create_pie_chart(all_hrv)

for group in group_list:
    group_var = group
    for hrv in hrv_list:
        hr_var = hrv
        only_freshman, no_freshman = by_grades_2022(all_hrv)
        #finds all unique values in demographics data columns for upper grades
        #unique_groups, grouped_data = group_data_by_variable(no_freshman, group_var, hr_var)
        #performs one way ANOVa and tukey's HSD and then created box plots showing significant differences
        #counter = stats_and_plotting(unique_groups,grouped_data, no_freshman, hr_var,group_var, counter)
        #finds all unique values in demographics data columns for freshman
        unique_groups, grouped_data = group_data_by_variable(no_freshman, group_var, hr_var)
        #performs one way ANOVa and tukey's HSD and then created box plots showing significant differences
        counter = stats_and_plotting(unique_groups,grouped_data, no_freshman, hr_var,group_var, counter)
        #stats_and_plotting calls group_data_by_variable
        #group_data_by_variable calls add_demographics_data
        #group_data_by_variable()


from django.contrib import messages
from django.contrib.auth import logout
from django.http import HttpResponseRedirect

from apps.core.defs import is_valid_user
from bluedata.helpers import reverse_url

# Seta o contexto do List
#
def set_context_list(context, self):
    # Variables
    url = self.base_url
    title = self.template_title

    # Set menu on session
    self.request.session['menu'] = self.access['menu']
    self.request.session['item'] = self.access['item']

    # Templates
    context['template'] = 'keen.html'
    context['template_title'] = title
    context['template_subtitle'] = 'listar'
    context['template_url_clear'] = reverse_url('{}_list'.format(url))
    context['template_url_update'] = '{}_update'.format(url)
    context['template_table_head'] = 'bluedata_interface/template_table_head.html'
    context['template_table_row'] = 'bluedata_interface/template_table_row.html'

    btn_add = set_button('Adicionar', reverse_url('{}_create'.format(url)), 'btn-primary', 'la la-plus')
    context['template_buttons'] = [btn_add]

# Seta o contexto do Create
#
def set_context_create(context, self):
    # Variables
    url = self.base_url
    title = self.template_title

    # Set session and menu
    self.request.session['menu'] = self.access['menu']
    self.request.session['item'] = self.access['item']

    # Templates
    context['template'] = 'keen.html'
    context['template_title'] = title
    context['template_subtitle'] = 'adicionar'
    context['template_form_fields'] = 'bluedata_interface/template_form_render.html'
    context['template_btn_save'] = True
    context['template_scripts'] = None

    btn_back = set_button('Voltar', reverse_url('{}_list'.format(url)), 'btn-outline-secondary', 'la la-angle-left')
    context['template_buttons'] = [btn_back]

# Seta o contexto do Update
#
def set_context_update(context, self):
    # Variables
    url = self.base_url
    title = self.template_title

    # Set session and menu
    self.request.session['menu'] = self.access['menu']
    self.request.session['item'] = self.access['item']
    self.request.session['uuid'] = str(self.object.id)

    # Templates
    context['template'] = 'keen.html'
    context['template_title'] = title
    context['template_subtitle'] = 'alterar'
    context['template_url_delete'] = reverse_url('{}_delete'.format(url), [str(self.object.id)])
    context['template_url_active'] = reverse_url('{}_active'.format(url), [str(self.object.id)])
    context['template_form_fields'] = 'bluedata_interface/template_form_render.html'
    context['template_btn_save'] = True
    context['template_scripts'] = None

    btn_back = set_button('Voltar', reverse_url('{}_list'.format(url)), 'btn-outline-secondary', 'la la-angle-left')
    context['template_buttons'] = [btn_back]

# Seta o contexto do Delete
#
def set_context_delete(context, self):
    # Variables
    url = self.base_url
    title = self.template_title

    # Set session and menu
    self.request.session['menu'] = self.access['menu']
    self.request.session['item'] = self.access['item']

    # Templates
    context['template'] = 'keen.html'
    context['template_title'] = title
    context['template_subtitle'] = 'excluir'

    btn_back = set_button('Voltar', reverse_url('{}_update'.format(url), [str(self.object.id)]), 'btn-outline-secondary', 'la la-angle-left')
    context['template_buttons'] = [btn_back]

# Metodo get da BaseActiveView
#
def get_base_active_view(self, kwargs):
    # Set session and menu
    self.request.session['menu'] = self.access['menu']
    self.request.session['item'] = self.access['item']

    # Check permission
    if not is_valid_user(self.request.user, self.access):
        logout(self.request)
        return HttpResponseRedirect(reverse_url('account_login'))

    msg = toggle_active(self.model.objects.get(id=kwargs['pk']))
    set_message(self, msg)
    return HttpResponseRedirect(reverse_url('{}_list'.format(self.base_url)))

# Dado um modelo modifica o status is_active
#
def toggle_active(modelo):
    if modelo.is_active:
        modelo.is_active = False
        msg = 'Registro alterado para inativo'
    else:
        modelo.is_active = True
        msg = 'Registro alterado para ativo'
    modelo.save()
    return msg

# Seta a mensagem para o template
#
def set_message(self, msg):
    messages.add_message(self.request, messages.SUCCESS, msg)

# Seta um botão para o template
#
def set_button(title, url, color, icon):
    return {'title': title, 'url': url, 'color': color, 'icon': icon}


import urwid
import pyfiglet
import PieceRenderer
import generalFunctions
import fenReader

class BoardDrawer:
    def __init__(self):
        self.pieceTextWidgets = []
        self.letterTextWidgets = []
        self.numberTextWidgets = []
        self.pieceRenderer = PieceRenderer.PieceRenderer(16, 7)

        # Create piece widgets
        for i in range(8):
            self.pieceTextWidgets.append([])
            for j in range(8):
                field = urwid.Text("")

                # Sette bakgrunnsfarger:
                tomTxtW = self.pieceRenderer.renderBottomWhite()
                tomTxtB = self.pieceRenderer.renderBottomBlack()
                field.set_text(tomTxtW if (i + j) % 2 == 0 else tomTxtB)

                self.pieceTextWidgets[i].append(field)

        # Create letter widgets
        for i in range(1, 9):
            text = pyfiglet.Figlet().renderText(chr(0x40 + i))
            textWid = urwid.Text(text, align="center")
            self.letterTextWidgets.append(textWid)

        # Create number widgets
        for i in range(8, 0, -1):
            text = pyfiglet.Figlet().renderText(str(i))
            textWid = urwid.Text(text, align="center")
            self.numberTextWidgets.append(textWid)

        # Place widgets in piles and columns
        widgetRows = []
        for i in range(len(self.pieceTextWidgets)):
            row = self.pieceTextWidgets[i].copy()
            row.insert(0, self.numberTextWidgets[i])
            row.append(self.numberTextWidgets[i])
            widgetRows.append(urwid.Columns(row))
        numberRow = urwid.Columns(self.letterTextWidgets)
        numberRow = urwid.Padding(numberRow)
        numberRow.left = 16
        numberRow.right = 16
        widgetRows.append(numberRow)
        widgetRows.insert(0, numberRow)

        self.topWidget = urwid.Pile(widgetRows)
        self.topWidget = urwid.Padding(self.topWidget, width=16 * 10)
        self.topWidget = urwid.Filler(self.topWidget)

    def setPieces(self, fen):
        boardMatrix = fenReader.getBoardMatrix(fen)
        for i in range(8):
            for j in range(8):
                text = self.pieceRenderer.renderPiece(boardMatrix[i][j], "white" if (i + j) % 2 == 0 else "black")
                self.pieceTextWidgets[i][j].set_text(text)


import curses

class MainScreen(object):
	def __init__(self, width, height, EM):
		# height+1 because you cannot write to the bottom-right-most
		# spot of a pad/window, this is a workaround
		self.__scr = curses.newpad(height+1, width)
		#self.__EM = EM
		self.__coords = (0,0)
		self.__size = (width, height)

	def update(self, EM, EQ, whereto):
		self.__scr.clear()

		i = 0
		while EQ.can_get("move_camera", i):
			event = EQ.get("move_camera", i)
			i+=1
			self.update_coords(event["to"], (whereto[2]-whereto[0],
					whereto[3]-whereto[1]))

		entities = EM.get_prop("r_prio").items()
		priorities = sorted(entities, key=lambda x: x[1])
		for entity, priority in priorities:
			pos = EM.get_value(entity, "r_pos")
			char = EM.get_value(entity, "r_char")
			#if EM.has_prop(pos, "0_from"):
			#	val = EM.get_value(pos, "0_from")
			#	#if val < 10:
			#	char = str(val)[1:2]
			attrs = 0
			fg_colour = EM.get_value(entity, "r_colour",
					curses.COLOR_WHITE)
			bg_colour = curses.COLOR_CYAN
			if EM.has_prop(pos, "bg_colour"):
				bg_colour = EM.get_value(pos, "bg_colour")
				if EM.has_prop(pos, "bloody"):
					bg_colour = curses.COLOR_RED
			else:
				#this is a dumb effect
				attrs |= curses.A_REVERSE
			attrs |= self.get_color_pair(fg_colour, bg_colour)
			if pos is not None and char is not None:
				self.__scr.addstr(pos[1], pos[0], char, attrs)

		#self.__scr.addstr(10, 10,
		#		"%i, %i"%(self.__coords[0], self.__coords[1]))
		self.__scr.noutrefresh(self.__coords[1], self.__coords[0],
				whereto[1],whereto[0], whereto[3],whereto[2])

	def update_coords(self, centre, visible_size=None):
		if visible_size is None:
			visible_size = (curses.COLS, curses.LINES)
		x = centre[0] - visible_size[0]//2
		y = centre[1] - visible_size[1]//2
		x = max(0, min(x, self.__size[0]-visible_size[0]))
		y = max(0, min(y, self.__size[1]-visible_size[1]))
		self.__coords = (x,y)

	def get_color_pair(self, fg, bg):
		return curses.color_pair(1 + 8 * fg + bg)


import os
import signal
import subprocess
import sys
import time
import timeit
import logging
import platform
from textwrap import indent, shorten

from maple_test.configs import construct_logger, get_val
from maple_test.utils import PASS, FAIL, UNRESOLVED, NOT_RUN, ENCODING
from maple_test.utils import add_run_path

class TestError(Exception):
    pass

def run_command_win(cmd, work_dir, timeout, logger, env=None):
    """Run commands using subprocess on Windows"""
    new_env = add_run_path(str(work_dir))
    new_env.update(env)
    process_command = subprocess.Popen(
        cmd,
        shell=True,
        cwd=str(work_dir),
        env=new_env,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    logger.debug("execute cmd ===>>>: %s", cmd)
    return_code = com_out = com_err = None
    try:
        com_out, com_err = process_command.communicate(timeout=timeout)
    except subprocess.CalledProcessError as err:
        return_code, com_out, com_err = err.returncode, "", err
        logger.exception(err)
        return return_code, com_out, com_err
    except subprocess.TimeoutExpired:
        return_code, com_out, com_err = 3, "TimeOut", "TimeOut"
        return return_code, com_out, com_err
    else:
        return_code = process_command.returncode
        com_out = com_out.decode(ENCODING, errors="replace")
        com_err = com_err.decode(ENCODING, errors="replace")
        return return_code, com_out, com_err
    finally:
        process_command.kill()
        logger.debug("return code: %d", return_code)
        logger.debug("stdout : \n%s", indent(com_out, "+\t", lambda line: True))
        logger.debug("stderr : \n%s", indent(com_err, "@\t", lambda line: True))

def run_command_linux(cmd, work_dir, timeout, logger, env=None):
    """Run commands using subprocess on Linux"""
    new_env = add_run_path(str(work_dir))
    new_env.update(env)
    process_command = subprocess.Popen(
        cmd,
        shell=True,
        cwd=str(work_dir),
        env=new_env,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        close_fds=True,
        start_new_session=True,
    )
    logger.debug("execute cmd ===>>>: %s", cmd)
    return_code = com_out = com_err = None
    try:
        com_out, com_err = process_command.communicate(timeout=timeout)
    except subprocess.CalledProcessError as err:
        return_code, com_out, com_err = err.returncode, "", err
        logger.exception(err)
        return return_code, com_out, com_err
    except subprocess.TimeoutExpired:
        return_code, com_out, com_err = 3, "", "TimeOut"
        return return_code, com_out, com_err
    else:
        return_code = process_command.returncode
        com_out = com_out.decode(ENCODING, errors="replace")
        com_err = com_err.decode(ENCODING, errors="replace")
        return return_code, com_out, com_err
    finally:
        process_command.kill()
        try:
            os.killpg(process_command.pid, signal.SIGTERM)
        except ProcessLookupError:
            pass
        logger.debug("return code: %d", return_code)
        logger.debug("stdout : \n%s", indent(com_out, "+\t", lambda line: True))
        logger.debug("stderr : \n%s", indent(com_err, "@\t", lambda line: True))

def run_commands(
    position, old_result, commands, work_dir, timeout, log_config, env=None
):
    name = "{}_{}".format(log_config[1], int(time.time()))
    name = log_config[1]
    formatter = logging.Formatter(
        "%(asctime)s %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
    )
    logger = construct_logger(log_config[0], log_config[1], file_fmt=formatter)
    if not commands:
        err = "Run task exit unexpected : {}, Log file at: {}.log".format(
            old_result[-1], log_config[0].get("dir") / name
        )
        logger.error(err)
        return position, old_result
    remain_time = timeout
    result = (PASS, None)
    logger.debug("Work directory: {}".format(work_dir))

    if platform.system() == "Windows":
        run_command = run_command_win
    else:
        run_command = run_command_linux

    commands_result = []
    for command in commands:
        start = timeit.default_timer()

        return_code, com_out, com_err = run_command(
            command, work_dir, remain_time, logger, env
        )
        run_time = timeit.default_timer() - start
        remain_time = remain_time - run_time
        logger.debug(
            "Run time: {:.2}, remain time: {:.2}".format(run_time, remain_time)
        )
        command_result = {}
        command_result["cmd"] = command
        command_result["return_code"] = return_code
        command_result["stdout"] = com_out
        command_result["stderr"] = com_err
        commands_result.append(command_result)
        if return_code != 0:
            result = (FAIL, commands_result)
            err = "Failed, Log file at: {}.log".format(log_config[0].get("dir") / name)
            logger.error(err)
            break
        else:
            result = (PASS, commands_result)
    if result[0] == PASS:
        logger.debug("Task executed successfully")
    handlers = logger.handlers[:]
    for handler in handlers:
        handler.close()
        logger.removeHandler(handler)
    return position, result

def progress(results, progress_type):
    """Output test progress"""

    if progress_type == "silent":
        return 0
    if progress_type == "normal":
        time_gape = 1
        print_progress = sys.stdout.write
    else:
        time_gape = 10
        print_progress = print
    finished = 0
    total = len(results)
    while total != finished:
        time.sleep(time_gape)
        finished = sum([result.ready() for result in results])
        rate = finished / total
        print_progress(
            "\rRunning test cases: {:.2%} ({}/{}) ".format(rate, finished, total)
        )
        sys.stdout.flush()


from django.core.management.base import BaseCommand

from ideascube.search.utils import create_index_table
from ideascube.search.models import SEARCHABLE

class Command(BaseCommand):
    help = 'Reindex all the searchable objects'

    def handle(self, *args, **kwargs):
        create_index_table()
        for model in SEARCHABLE.values():
            count = 0
            for inst in model.objects.all():
                inst.index()
                count += 1
            if count:
                self.stdout.write('Indexed {} content.'.format(model.__name__))
        self.stdout.write('Done reindexing.')


# -*- coding: utf-8 -*-
"""
Created on Tue Apr 20 12:20:28 2021

@author: mrgna
"""

import numpy as np
from scipy import stats
from statsmodels.distributions.empirical_distribution import ECDF
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow_probability as tfp
tf.config.set_visible_devices([], 'GPU')

####### Comparisson of paths
class PathComparator():
    def __init__(self,paths1,paths2, T, names = ["paths1","paths2"]):
        self.n = paths1.shape[1] - 1
        self.T = T

        self.paths = [paths1,paths2]
        self.returns = [p[:,1:] / p[:,:-1] - 1 for p in self.paths]
        self.times = np.linspace(0,self.T,self.n+1)
        self.names = names

    def run_comparrison(self, plot = True):
        #Compare mean and std of paths

        self.means = [np.mean(p, 0) for p in self.paths]
        self.stds = [np.std(p, 0) for p in self.paths]

        if plot is True:
            [plt.plot(self.times,m) for m in self.means]
            plt.title(("Plot of means"))
            plt.show()

            [plt.plot(self.times,s) for s in self.stds]
            plt.show()

        ### QQ plot and ECDF
        self.idxs = [1,-1]
        self.ecdf_saved = []
        self.ecdf_ranges = []

        for idx in self.idxs:
            # QQ Plot

            percentiles = np.linspace(0.01,0.99,99)
            self.quantiles = [np.quantile(p[:,idx], percentiles) for p in self.paths]

            #qq plot with theoretical quantiles of simulated bs paths
            if plot is True:
                plt.scatter(self.quantiles[0], self.quantiles[1], s = 5)
                plt.plot(self.quantiles[0][[0,-1]],self.quantiles[0][[0,-1]],c='k')
                plt.show()

            # ECDF
            ecdfs = [ECDF(p[:,idx]) for p in self.paths]

            range1 = np.linspace(np.min(self.paths[0][:,idx]),np.max(self.paths[0][:,idx]),1000)

            tmp_ecdf_dist = np.abs(ecdfs[0](range1) - ecdfs[1](range1))
            max_dif_x = np.argmax(tmp_ecdf_dist)
            max_dif = np.max(np.abs(tmp_ecdf_dist))

            if plot is True:
                plt.plot(range1, ecdfs[0](range1))
                plt.plot(range1, ecdfs[1](range1))
                #plt.plot([range1[max_dif_x]]*2,[0,1])
                plt.show()

            self.ecdf_saved.append([ecdf(range1) for ecdf in ecdfs])
            self.ecdf_ranges.append(range1)

        #### Kolmogorov-Smirnov across time

        self.pvalues = [stats.ks_2samp(self.paths[0][:,i], self.paths[1][:,i], mode = 'exact')[1] for i in range(1,self.n+1)]

        if plot is True:
            plt.plot(self.times[1:], self.pvalues)
            plt.show()

        #### corr comparrison

        def calculate_cor_lag(x,lag):
            lead_x = x[:,lag:]
            lag_x = x[:,:-lag]
            cov_x = tfp.stats.correlation(lead_x,lag_x, 0, event_axis = None)
            return cov_x.numpy()

        self.lags = range(1,self.n)
        self.abs_corrs = [[np.mean(calculate_cor_lag(np.abs(lr), l)) for l in self.lags] for lr in self.returns]

        if plot is True:
            plt.plot(self.lags,self.abs_corrs[0])
            plt.plot(self.lags,self.abs_corrs[1])
            plt.show()

        self.corrs = [[np.mean(calculate_cor_lag(lr, l)) for l in self.lags] for lr in self.returns]

        if plot is True:
            plt.plot(self.lags,self.corrs[0])
            plt.plot(self.lags,self.corrs[1])
            plt.show()

if __name__ == '__main__':
    import BlackScholesModel
    import HestonModel
    import MarketGenerator

    BS = False

    real_n = 20
    n = 20
    real_T = 1/12
    dt = real_T / real_n
    cond_n = 0

    if BS:
        ########## Black shocles
        n = n
        T = dt * n

        s0 = 1
        mu = 0.05
        sigma1 = 0.3
        sigma2 = 0.3
        rate = 0.01

        corr = np.array([1])

        model1 = BlackScholesModel.BlackScholesModel(s0, mu, sigma1, np.ones((1,1)), rate, dt)
        model2 = BlackScholesModel.BlackScholesModel(s0, mu, sigma2, np.ones((1,1)), rate, dt)

        n_paths1 = 1000
        n_paths2 = 1000

        model1.reset_model(n_paths1)
        model2.reset_model(n_paths2)

        for _ in range(n):
            model1.evolve_s_b()
            model2.evolve_s_b()

        paths1 = model1.spot_hist[:,0,:]
        paths2 = model2.spot_hist[:,0,:]

        PathComparator(paths1,paths2,T).run_comparrison()

    ###### Heston
    else:
        n = n
        T = dt * n

        s0 = 1
        mu = 0.05
        v0 = 0.1
        kappa = 10
        theta = 0.1
        sigma = 1
        rho = -0.9
        rate = 0.01
        dt = T/n
        ddt = 0.01

        model1 = HestonModel.HestonModel(s0,mu,v0,kappa,theta,sigma,rho,rate,dt,ddt)
        model2 = HestonModel.HestonModel(s0,mu,v0,kappa,theta,sigma,rho,rate,dt,ddt)

        model1.use_v = True
        model2.use_v = True

        n_paths1 = 1000
        n_paths2 = 1000

        model1.reset_model(n_paths1)
        model2.reset_model(n_paths2)

        for _ in range(n):
            model1.evolve_s_b()
            model2.evolve_s_b()

        paths1 = model1.spot_hist[:,0,:]
        paths2 = model2.spot_hist[:,0,:]

        PathComparator(paths1,paths2,T).run_comparrison()

     ####### MG
    n_samples = 1000
    real_decoder = True

    MG = MarketGenerator.MarketGenerator(model1, n, cond_n)
    MG.create_vae(latent_dim = n, layers_units=[40], alpha = 0.95, beta = 100, real_decoder= real_decoder)
    MG.create_training_path(n_samples, overlap = True, seed = None, cheat = True)
    MG.train_vae(epochs = 500, batch_size = 128, lrs = [0.01,0.0001,0.00001])

    if BS is False:
        plt.plot(model1.spot_hist[:,1,:].T)
        plt.show()

    training_paths = MG.training_paths

    #get cond
    cond = MG.cond

    k = 20000 // n_samples
    tmp_cond = np.repeat(cond,k,axis = 0) if cond is not None else None
    mg_paths = MG.generate_paths(n_samples*k, cond = tmp_cond, save = False,
                                 std = 1, mean = 0, real_decoder = real_decoder)

    MG.plot_generated_paths(100, cond[:100,:] if cond_n > 0 else None)
    plt.plot(training_paths[:100,:].T)
    plt.show()

    pc = PathComparator(training_paths,mg_paths,T)
    pc.run_comparrison()

    ###### Test on longer
    if real_n > n:
        n_long_samples = 20000
        if cond_n == 0:
            cond = None
        else:
            if BS:
                    model1.reset_model(1)
                    for i in range(cond_n):
                        model1.evolve_s_b()
                    spots = model1.spot_hist[:,0,:]
                    cond = spots[:,1:] / spots[:,:-1] - 1
                    cond = np.repeat(cond,n_long_samples,axis = 0)

            else:
                if cond_n > 0:
                    new_v0 = v0
                    model1.reset_model(1000)
                    for i in range(100):
                        model1.evolve_s_b()
                    idx = np.argmin(np.abs(model1.v - new_v0)) #v0
                    spots = model1.spot_hist[:,0,:]
                    cond = spots[idx:idx+1,1:] / spots[idx:idx+1,:-1] - 1
                    cond = cond[:,-cond_n:]
                    cond = np.repeat(cond,n_long_samples,axis = 0)

                    model1.v0 = new_v0
                else:
                    cond = None

        long_mg_paths = MG.generate_longer_paths(n_long_samples, length = real_n, cond = cond, save = False,
                                                 std = 1, mean = 0, real_decoder = real_decoder)

        MG.plot_generated_paths(100, cond[:100,:] if cond_n > 0 else None, real_n)

        model1.reset_model(20000)
        for i in range(real_n):
            model1.evolve_s_b()
        long_model_paths = model1.spot_hist[:,0,:]

        pc_long = PathComparator(long_model_paths,long_mg_paths,T)
        pc_long.run_comparrison()

        if BS is False:
            model1.v0 = v0


from functools import wraps
import logging
from stocker.stock_exceptions import StockInformationMissingException
from typing import Tuple, Optional, Type

""" Dektoratorji za metode razreda Stock """
logger = logging.getLogger(__name__)

def error_call(func_to_call, error_to_raise):
	# type: (callable, Type[StockInformationMissingException]) -> Optional
	def _ignore(_e):
		logger.info(f"Ignored exception {_e}")

	def _raise(_e):
		raise _e

	d = {
		"ignore": _ignore,
		"raise": _raise
	}
	d.get(func_to_call, lambda x: x())(error_to_raise)

def ignore_stock_exceptions(*ignore_exceptions):
	# type: (Tuple[str]) -> callable
	""" Ignores given stock exceptions """

	def decorator(func):
		@wraps(func)
		def wrapper(*args, **kwargs):
			try:
				return func(*args, **kwargs)
			except BaseException as be:
				if type(be) in ignore_exceptions:
					return
				raise be

		return wrapper

	return decorator

def debug(func):
	""" Debug helper function, logs inputs and outputs of functions"""
	def get_string(o):
		""" represents object with shortest notation """
		ret_s = str(o)
		ret_s = ret_s.replace("\n", "")
		if type(o) == list:
			return f"[{','.join([get_string(elem) for elem in o])}]"
		elif type(o) == tuple:
			return f"({','.join([get_string(elem) for elem in o])})"
		if len(ret_s) >= 15:
			return str(type(o))
		return ret_s

	@wraps(func)
	def wrapper(*args, **kwargs):
		args_repr = [get_string(o) for o in args]
		kwarg_repr = [f"{k}={get_string(v)}" for k, v in kwargs.items()]
		logger.debug(" ".join(
			("Calling function", func.__name__, f"with args   ({', '.join(args_repr)})")
		))
		logger.debug(" ".join(
			("                ", " "*len(func.__name__), f"with kwargs ({', '.join(kwarg_repr)})")
		))
		_ret = func(*args, **kwargs)
		logger.debug(" ".join(
			("Returned by     ", func.__name__, f" ({get_string(_ret)})")
		))
		return _ret

	return wrapper

def default_perform_on(*default_work_columns, default_errors='raise'):
	# type: (Tuple[str], Optional[str]) -> callable
	""" Gives function an option to perform other columns """
	column_indexes = [(slice(None, None), i) for i in range(len(default_work_columns))]
	if len(column_indexes) == 1:
		# If there is only one value, it doesnt have to be unpacked, rather forwarded
		column_indexes = column_indexes[0]

	def decorator(func):
		@wraps(func)
		def wrapper(*args, work_dataframe=None, work_columns=default_work_columns, errors=default_errors, **kwargs):
			_work_columns = [work_columns] if isinstance(work_columns, str) else list(
				work_columns)  # makes sure there is a list
			if work_dataframe is None:
				if any([_wc not in args[0].dataframe for _wc in _work_columns]):
					r = error_call(errors, StockInformationMissingException)
					if r is None:
						return
				_work_dataframe = args[0].dataframe[_work_columns].copy()
			else:
				_work_dataframe = work_dataframe.copy()
				if any([_wc not in _work_dataframe for _wc in _work_columns]):
					error_call(errors, StockInformationMissingException)
				_work_columns = _work_dataframe.columns.to_list()
			_ret_dataframe = func(*args, work_dataframe=_work_dataframe, col_idxs=column_indexes, **kwargs)
			_ret_dataframe.drop(columns=_work_columns, errors='ignore', inplace=True)
			return _ret_dataframe

		return wrapper
	return decorator

def join_with_dataframe_as(*default_column_names, default_make_unique=True, default_rsuffix=None):
	# type: (Optional[Tuple[str]], Optional[bool], Optional[str]) -> callable
	""" Saves the output series to stock column """

	def decorator(func):
		def prepare_names(names_in, args, make_unique=True, rsuffix=None):
			""" Assures that correctly formatted string tuple is returned """
			if isinstance(names_in, str):
				names_in = (names_in,)

			if make_unique:
				_args = [str(x) for x in args]
				_rsuffix = prepare_names(rsuffix, [], make_unique=False) if rsuffix is not None else []
				return ["_".join([n] + _args + _rsuffix) for n in names_in]
			else:
				return list(names_in)

		@wraps(func)
		def wrapper(*args, column_names=default_column_names, make_unique_name=default_make_unique,
					rsuffix=default_rsuffix, no_join=False, **kwargs):
			_ret_series = func(*args, **kwargs)
			_col_names = column_names

			# Try with column_names, if not present use col_names from decorator, or the dataframe columns
			_col_names = prepare_names(_col_names, args[1:], make_unique=make_unique_name, rsuffix=rsuffix)
			# logger.debug(f"Col names        {str(list(_ret_series.columns))} -> {str(_col_names)}")
			_ret_series.columns = _col_names
			args[0].dataframe.drop(columns=_col_names, errors='ignore', axis=1, inplace=True)
			if not no_join:
				args[0].dataframe = args[0].dataframe.join(_ret_series)
			return _ret_series

		return wrapper

	return decorator

def save_to_group(default_group_name):
	# type: (str) -> callable
	""" Saves the dataframe column names to list of similar attributes. It also creates a getter function for the list if
	it is not present yet. """

	def decorator(func):
		@wraps(func)
		def wrapper(*args, group_name=default_group_name, label=None, **kwargs):
			_ret_dataframe = func(*args, **kwargs)
			_group_name = group_name
			if _group_name is None:
				# User specified, it is not allowed to add it to the group
				return _ret_dataframe
			col_names = list(_ret_dataframe.columns.to_list())
			params_called_with = list(args[1:])
			if label is not None:
				if type(label) in [list, tuple]:
					params_called_with += label
				else:
					params_called_with += [label]
			args[0].groups[_group_name].append((col_names, tuple(params_called_with)))
			return _ret_dataframe

		return wrapper

	return decorator


import torch
from torch import nn

class SimpleModel(nn.Module):
    def __init__(self):
        super(SimpleModel, self).__init__()
        self.sequential_convs = nn.Sequential(
            nn.Conv2d(1,96,(5,5),padding=2),
            nn.ReLU(),
            nn.MaxPool2d((2,2),stride=2,padding=0),
            nn.Conv2d(96,48,(1,1)),
            nn.ReLU(),
            nn.Conv2d(48,96,(3,3),padding=1),
            nn.ReLU(),
            nn.MaxPool2d((2,2),stride=2,padding=0),
            nn.Conv2d(96,48,(1,1)),
            nn.ReLU(),
            nn.Conv2d(48,96,(3,3),padding=1),
            nn.ReLU(),
            nn.MaxPool2d((2,2),stride=2,padding=0),
            nn.Conv2d(96,48,(1,1)),
            nn.ReLU(),
            nn.Flatten(),
            nn.Linear(768,128),
            nn.ReLU(),
            nn.Linear(128,101)
        )

    def forward(self, x):
        x = self.sequential_convs(x)
        return x


from django.shortcuts import render
from .serializers import QuestSerializer, QuestCompliteSerializer
from .models import Quest, QuestComplite
from rest_framework.viewsets import GenericViewSet
from rest_framework import mixins
from rest_framework.permissions import IsAuthenticated
from rest_framework.authentication import TokenAuthentication
from rest_framework.response import Response
from core.models import UserProfile
from lvl.models import LevelInformation

# Create your views here.
class QuestViewSet(mixins.ListModelMixin, mixins.RetrieveModelMixin, GenericViewSet):
    serializer_class = QuestSerializer
    queryset = Quest.objects.all()
    permission_classes = [IsAuthenticated]
    authentication_classes = [TokenAuthentication]

class QuestCompliteViewSet(mixins.CreateModelMixin, GenericViewSet):
    serializer_class = QuestCompliteSerializer
    queryset = QuestComplite.objects.all()
    permission_classes = [IsAuthenticated]
    authentication_classes = [TokenAuthentication]

    def create(self, request, *args, **kwargs):
        user = UserProfile.objects.get(user=request.user)
        # request.user.profile
        quest_id = request.data.get("quest")
        answer = request.data.get("answer")
        quest = Quest.objects.get(id=quest_id)
        if QuestComplite.objects.filter(quest=quest_id).exists():
            return Response("you have already did this quest")
        else:
            if quest.quest_answer == answer:
                user.gained_xp += quest.xp
                level_info = LevelInformation.objects.first()
                lvl = user.gained_xp // level_info.needed_xp
                user.level += lvl
                if lvl >= 1:
                    user.gained_xp -= level_info.needed_xp * lvl
                user.save()
                return super().create(request, *args, **kwargs)
            else:
                return Response("this answer is incorrect")


import abc
import datetime
import json
from typing import Tuple, Dict, Union, Callable, List, Optional, Any
import copy

import pandas as pd
from numpy import NaN
import sqlalchemy as sa

from qualipy.exceptions import InvalidReturnValue

def convert_value_to_varchar(value):
    if isinstance(value, dict):
        return json.dumps(value)
    return value

class BaseType(object):
    def __init__(self):
        pass

    def __repr__(self):
        class_name = str(self.__class__).split(".")[-1]
        class_name = class_name.replace("'>", "")
        return class_name

class BaseData(object):
    def __init__(self, data, config=None, stratify=None):
        self.data = data
        self.stratify = stratify

    def get_data(self, backend_used=None):
        return self.data

    def set_fallback_data(self):
        pass

class MetricResult:

    types_repr = {
        float: "float",
        int: "int",
        bool: "bool",
        dict: "dict",
        str: "str",
        "custom": "list",
    }

    def __init__(
        self,
        value,
        metric: str,
        date: datetime.date,
        column_name: str,
        return_format: str,
        type,
        run_name: str = None,
        arguments: str = None,
        meta: dict = None,
    ) -> None:
        self.value = value
        self.metric = metric
        self.date = date
        self.column_name = column_name
        self.return_format = return_format
        self.type = type
        self.run_name = run_name
        self.arguments = arguments
        self.meta = meta

    @staticmethod
    def create_arg_string(keyword_arguments: Dict[str, Any]) -> str:
        if keyword_arguments:
            return str(keyword_arguments)
        return NaN

    def set_return_value_type(self):
        if str(self.value) == "nan":
            pass
        elif self.return_format in [int, float, str, dict, bool]:
            try:
                self.value = self.return_format(self.value)
            except TypeError as e:
                raise InvalidReturnValue(
                    "Invalid return self.value: {}, was expecting"
                    " '{}'".format(e, str(self.return_format))
                )
        elif self.return_format == "custom":
            if not isinstance(self.value, list):
                raise InvalidReturnValue("Improperly formatted custom return type")
            else:
                return
        else:
            raise InvalidReturnValue(
                "Unsupported type: '{}'".format(str(self.return_format))
            )

    def update_keys(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)

    def __repr__(self) -> str:
        return str(self.to_dict())

    def to_dict(self) -> dict:
        if self.meta is not None:
            meta = json.dumps(self.meta)
        else:
            meta = self.meta
        return {
            "value": self.value,
            "metric": self.metric,
            "date": self.date,
            "column_name": self.column_name,
            "return_format": self.types_repr[self.return_format],
            "type": self.type,
            "run_name": self.run_name,
            "arguments": self.arguments,
            "meta": meta,
        }

class BackendBase(abc.ABC):
    def __init__(self, config):
        self.config = config

    @abc.abstractmethod
    def set_schema(self, data, columns):
        return

    def generate_description(
        self,
        function: Callable,
        data,
        column: str,
        date: datetime.datetime,
        function_name: str,
        viz_type: str = "numerical",
        return_format: str = float,
        run_name: str = None,
        kwargs: Dict[str, Any] = None,
        overwrite_kwargs: Dict[str, Any] = None,
    ) -> MetricResult:
        kwargs = {} if kwargs is None else kwargs
        original_kwargs = copy.copy(kwargs)
        if overwrite_kwargs is not None:
            for arg in function.allowed_arguments:
                if arg in overwrite_kwargs:
                    kwargs[arg] = overwrite_kwargs[arg]
        value = function(data, column, **kwargs)
        argument_str = MetricResult.create_arg_string(original_kwargs)
        metric_res = MetricResult(
            value=value,
            metric=function_name,
            arguments=argument_str,
            date=date,
            column_name=column,
            return_format=return_format,
            type=viz_type,
            run_name=run_name,
        )
        return metric_res

    @abc.abstractmethod
    def get_dtype(self, data, column):
        return

    @abc.abstractmethod
    def check_type(self, data, column, desired_type, force=False):
        return

    @abc.abstractmethod
    def generate_column_general_info(self, specs, data, time_of_run):
        return

    @abc.abstractmethod
    def generate_data(self, *args, **kwargs):
        return

    def write(self, conn, measures, project, batch_name, schema=None):
        data = pd.DataFrame([i.to_dict() for i in measures])
        data["insert_time"] = datetime.datetime.now().replace(tzinfo=None)
        data["batch_name"] = batch_name

        data.value = data.value.apply(convert_value_to_varchar)

        data.to_sql(
            project.project_name,
            con=conn,
            if_exists="append",
            index=False,
            schema=schema,
        )