# coding: utf-8

# In[1]:

from sklearn.cluster import KMeans
import numpy as np
import csv
import math

# In[2]:

C_Lambda = 0.03 #where used?# In closedformsolution
TrainingPercent = 80
ValidationPercent = 10
TestPercent = 10
M = 10
PHI = []
IsSynthetic = False #where used? To remove columns with zero variance

# In[3]:

def GetTargetVector(filePath):
    t = []
    with open(filePath, 'rU') as f: #rU is a file mode. Means read as text file with newline delimiter
        reader = csv.reader(f) #returns an iterator over each row in file
        for row in reader:  #row is a string list of the values that were seperated by comma ie.. "a,b" as ["a","b"]
            t.append(int(row[0])) #1st column is the target column. Hence, only it is appended as an int.
    #print("Raw Training Generated..")
    return t

#what is role of IsSynthetic?#
#why columns 5,6,7,8,9 are deleted if IsSynthetic==False?#
#These columns have only value 0 which implies the variance in these columns are zero. Zero variance can't be tolerated
#while computing inverse covariance matrix.Hence removed
def GenerateRawData(filePath, IsSynthetic):
    dataMatrix = []
    with open(filePath, 'rU') as fi:
        reader = csv.reader(fi)
        for row in reader:
            dataRow = []
            for column in row:
                dataRow.append(float(column)) #each column value of a row is stored as float value in dataRow list
            dataMatrix.append(dataRow)

    if IsSynthetic == False :
        dataMatrix = np.delete(dataMatrix, [5,6,7,8,9], axis=1) #axis=0 means 2nd arg is row number.axis=1 column number
    dataMatrix = np.transpose(dataMatrix) #transpose of dataMatrix
    #print ("Data Matrix Generated..")
    return dataMatrix

def GenerateTrainingTarget(rawTraining,TrainingPercent = 80): #rawTraining is target vector
    TrainingLen = int(math.ceil(len(rawTraining)*(TrainingPercent*0.01))) #computes num.rows * 0.8
    t           = rawTraining[:TrainingLen] #extracts row numbers 1 to num.rows*0.8 or row index 0 to (num.rows*0.8) -1
    #print(str(TrainingPercent) + "% Training Target Generated..")
    return t

def GenerateTrainingDataMatrix(rawData, TrainingPercent = 80): #rowData is data matrix
    T_len = int(math.ceil(len(rawData[0])*0.01*TrainingPercent)) #same as GenerateTrainingTarget but on data matrix
    d2 = rawData[:,0:T_len] #T_len used on column bounding as data matrix is transposed.
    #print(str(TrainingPercent) + "% Training Data Generated..")
    return d2

def GenerateValData(rawData, ValPercent, TrainingCount):
    valSize = int(math.ceil(len(rawData[0])*ValPercent*0.01)) # num.rows out of total num.rows equivalent to valPercent
    V_End = TrainingCount + valSize
    dataMatrix = rawData[:,TrainingCount+1:V_End] #rows right after traing data for the size of valsize is extracted
    #print (str(ValPercent) + "% Val Data Generated..")
    return dataMatrix

def GenerateValTargetVector(rawData, ValPercent, TrainingCount):
    valSize = int(math.ceil(len(rawData)*ValPercent*0.01)) #same as GenerateValData but to find target vector
    V_End = TrainingCount + valSize
    t =rawData[TrainingCount+1:V_End]
    #print (str(ValPercent) + "% Val Target Data Generated..")
    return t

#what is bigsigma?#sigma^2#
def GenerateBigSigma(Data, MuMatrix,TrainingPercent,IsSynthetic): #Raw data 41Xn, MuMatrix 10X41
    BigSigma    = np.zeros((len(Data),len(Data))) #creates 41X41 matrix of zeros
    DataT       = np.transpose(Data) #nX41
    TrainingLen = math.ceil(len(DataT)*(TrainingPercent*0.01)) #first 80 percent lines
    varVect     = []
    for i in range(0,len(DataT[0])): #41 times
        vct = []
        for j in range(0,int(TrainingLen)): #55k times
            vct.append(Data[i][j])
        varVect.append(np.var(vct)) #variance of each feature

    for j in range(len(Data)):
        BigSigma[j][j] = varVect[j] #reassigning diagonal values in zero matrix
    if IsSynthetic == True:
        BigSigma = np.dot(3,BigSigma)
    else:
        BigSigma = np.dot(200,BigSigma) #multiplying each covariance with 200 #why?#
    ##print ("BigSigma Generated..")
    #print("varVect: ",varVect)
    return BigSigma

#Computes Gaussian power value#
def GetScalar(DataRow,MuRow, BigSigInv):  #1X41,1X41,41X41
    R = np.subtract(DataRow,MuRow) #x-Mu for each column #1X41
    T = np.dot(BigSigInv,np.transpose(R)) #(x-Mu)T/sigma^2 #41X1
    L = np.dot(R,T) #(x-Mu)T/sigma^2 X (x-Mu) #1X1 #Scalar
    return L

#Computes Phi#
def GetRadialBasisOut(DataRow,MuRow, BigSigInv):
    phi_x = math.exp(-0.5*GetScalar(DataRow,MuRow,BigSigInv))
    return phi_x

#Generates phi matrix#
def GetPhiMatrix(Data, MuMatrix, BigSigma, TrainingPercent = 80):#Raw data 41Xn, Mumatrix 10X41, bigsigma 41X41
    DataT = np.transpose(Data) #nX41
    TrainingLen = math.ceil(len(DataT)*(TrainingPercent*0.01)) #80% of total records
    PHI = np.zeros((int(TrainingLen),len(MuMatrix)))  #55kX10 zero matrix
    BigSigInv = np.linalg.inv(BigSigma) #bigsigma inverse
    for  C in range(0,len(MuMatrix)): #10
        for R in range(0,int(TrainingLen)): #55k
            PHI[R][C] = GetRadialBasisOut(DataT[R], MuMatrix[C], BigSigInv)#e^(-((x-Mu)T/2*sigma^2) * (x-Mu))
    #print ("PHI Generated..")
    return PHI

#computes the weight vector
def GetWeightsClosedForm(PHI, T, Lambda): #10X55k, 55k X 1
    Lambda_I = np.identity(len(PHI[0])) #identity matrix 10X10
    for i in range(0,len(PHI[0])):
        Lambda_I[i][i] = Lambda #diagonal values as lambda
    PHI_T       = np.transpose(PHI) #10X55k
    PHI_SQR     = np.dot(PHI_T,PHI) #10X10
    PHI_SQR_LI  = np.add(Lambda_I,PHI_SQR) #adding regularization factor lambda
    PHI_SQR_INV = np.linalg.inv(PHI_SQR_LI)
    INTER       = np.dot(PHI_SQR_INV, PHI_T) #phi inverse #10X55k
    W           = np.dot(INTER, T) #weights 10X1
    ##print ("Training Weights Generated..")
    return W

#This generates target output for given PHI matrix and weights computed earlier
def GetValTest(VAL_PHI,W):
    Y = np.dot(W,np.transpose(VAL_PHI)) #computes dot product of W vector and VAL_PHI matrix
    ##print ("Test Out Generated..")
    return Y

#Given computed and actual target value vectors, computes ERMS error
def GetErms(VAL_TEST_OUT,ValDataAct):
    sum = 0.0
    t=0
    accuracy = 0.0
    counter = 0
    val = 0.0
    for i in range (0,len(VAL_TEST_OUT)):
        sum = sum + math.pow((ValDataAct[i] - VAL_TEST_OUT[i]),2) #(computed - actual)^2
        if(int(np.around(VAL_TEST_OUT[i], 0)) == ValDataAct[i]): #if integer round off of computed value = actual value
            counter+=1                                           #increment counter by 1
    accuracy = (float((counter*100))/float(len(VAL_TEST_OUT))) #Values matched divided by total values in percentage
    ##print ("Accuracy Generated..")
    ##print ("Validation E_RMS : " + str(math.sqrt(sum/len(VAL_TEST_OUT))))
    return (str(accuracy) + ',' +  str(math.sqrt(sum/len(VAL_TEST_OUT)))) #accuracy and RMS value of error

# ## Fetch and Prepare Dataset

# In[4]:

RawTarget = GetTargetVector('Querylevelnorm_t.csv')
RawData   = GenerateRawData('Querylevelnorm_X.csv',IsSynthetic) #transpose of data as matrix

# ## Prepare Training Data

# In[5]:

TrainingTarget = np.array(GenerateTrainingTarget(RawTarget,TrainingPercent))
TrainingData   = GenerateTrainingDataMatrix(RawData,TrainingPercent)
print(TrainingTarget.shape)
print(TrainingData.shape)

# ## Prepare Validation Data

# In[6]:

ValDataAct = np.array(GenerateValTargetVector(RawTarget,ValidationPercent, (len(TrainingTarget))))
ValData    = GenerateValData(RawData,ValidationPercent, (len(TrainingTarget)))
print(ValDataAct.shape)
print(ValData.shape)

# ## Prepare Test Data

# In[7]:

TestDataAct = np.array(GenerateValTargetVector(RawTarget,TestPercent, (len(TrainingTarget)+len(ValDataAct))))
TestData = GenerateValData(RawData,TestPercent, (len(TrainingTarget)+len(ValDataAct)))
print(ValDataAct.shape)
print(ValData.shape)

# ## Closed Form Solution [Finding Weights using Moore- Penrose pseudo- Inverse Matrix]

# In[8]:

kmeans = KMeans(n_clusters=M, random_state=0).fit(np.transpose(TrainingData)) #finding M clusters in training data
Mu = kmeans.cluster_centers_ #Getting cluster centers #for each centroid 41 values ie.. it finds 10 centroids for each feature
BigSigma     = GenerateBigSigma(RawData, Mu, TrainingPercent,IsSynthetic) #returns bigsigma for training set 41X41
TRAINING_PHI = GetPhiMatrix(RawData, Mu, BigSigma, TrainingPercent) #55k X 10
W            = GetWeightsClosedForm(TRAINING_PHI,TrainingTarget,(C_Lambda)) #10X1 weight vector obtained
TEST_PHI     = GetPhiMatrix(TestData, Mu, BigSigma, 100) #using same bigsigma as in training set
VAL_PHI      = GetPhiMatrix(ValData, Mu, BigSigma, 100) #using same bigsigma as in training set

# In[9]:

print(Mu.shape)
print(BigSigma.shape)
print(TRAINING_PHI.shape)
print(W.shape)
print(VAL_PHI.shape)
print(TEST_PHI.shape)

# ## Finding Erms on training, validation and test set

# In[10]:

TR_TEST_OUT  = GetValTest(TRAINING_PHI,W) #returns target output value for the train dataset
VAL_TEST_OUT = GetValTest(VAL_PHI,W) #returns target output value for the validation dataset
TEST_OUT     = GetValTest(TEST_PHI,W) #returns target output value for the test dataset

TrainingAccuracy   = str(GetErms(TR_TEST_OUT,TrainingTarget))
ValidationAccuracy = str(GetErms(VAL_TEST_OUT,ValDataAct))
TestAccuracy       = str(GetErms(TEST_OUT,TestDataAct))

# In[11]:

print ('UBITname      = baalajiv')
print ('Person Number = 50287414')
print ('----------------------------------------------------')
print ("------------------LeToR Data------------------------")
print ('----------------------------------------------------')
print ("-------Closed Form with Radial Basis Function-------")
print ('----------------------------------------------------')
print ("M = " + str(M)+ "\nLambda = " + str(C_Lambda))
print ("E_rms Training   = " + str(float(TrainingAccuracy.split(',')[1])))
print ("E_rms Validation = " + str(float(ValidationAccuracy.split(',')[1])))
print ("E_rms Testing    = " + str(float(TestAccuracy.split(',')[1])))

# ## Gradient Descent solution for Linear Regression

# In[12]:

print ('----------------------------------------------------')
print ('--------------Please Wait for 2 mins!----------------')
print ('----------------------------------------------------')

# In[13]:

W_Now        = np.dot(220, W) #randomly 220 times the weights computed using closed form method
La           = 2 #Lambda value for SGD
learningRate = 0.01
L_Erms_Val   = []
L_Erms_TR    = []
L_Erms_Test  = []
W_Mat        = []

for i in range(0,400):

    #print ('---------Iteration: ' + str(i) + '--------------')
    Delta_E_D     = -np.dot((TrainingTarget[i] - np.dot(np.transpose(W_Now),TRAINING_PHI[i])),TRAINING_PHI[i])#-(target-computed)* phi #negative as change must be in opposite direction of gradient
    La_Delta_E_W  = np.dot(La,W_Now) #(lambda) * Ew
    Delta_E       = np.add(Delta_E_D,La_Delta_E_W)  #Total delta E = delta Ed + delta Ew
    Delta_W       = -np.dot(learningRate,Delta_E) #changind weights based on their partial differential values times learning rate
    W_T_Next      = W_Now + Delta_W #The change in weights are subtracted from previous weights
    W_Now         = W_T_Next #updating weights

    #-----------------TrainingData Accuracy---------------------#
    TR_TEST_OUT   = GetValTest(TRAINING_PHI,W_T_Next) #computes target output values for given phi matrix and weights
    Erms_TR       = GetErms(TR_TEST_OUT,TrainingTarget) #gets training set accuracy given computed and actual target values
    L_Erms_TR.append(float(Erms_TR.split(',')[1])) #takes ERMS value and discards accuracy from output of previous line

    #-----------------ValidationData Accuracy---------------------#
    VAL_TEST_OUT  = GetValTest(VAL_PHI,W_T_Next) #computes target output values for given phi matrix and weights
    Erms_Val      = GetErms(VAL_TEST_OUT,ValDataAct) #gets validation set accuracy given computed and actual target values
    L_Erms_Val.append(float(Erms_Val.split(',')[1]))#takes ERMS value and discards accuracy from output of previous line

    #-----------------TestingData Accuracy---------------------#
    TEST_OUT      = GetValTest(TEST_PHI,W_T_Next) #computes target output values for given phi matrix and weights
    Erms_Test = GetErms(TEST_OUT,TestDataAct) #gets test set accuracy given computed and actual target values
    L_Erms_Test.append(float(Erms_Test.split(',')[1]))#takes ERMS value and discards accuracy from output of previous line

# In[14]:

print ('----------Gradient Descent Solution--------------------')
print ("M = " + str(M) + "\nLambda  = " + str(La) + "\neta=0.01")
print ("E_rms Training   = " + str(np.around(min(L_Erms_TR),5)))
print ("E_rms Validation = " + str(np.around(min(L_Erms_Val),5)))
print ("E_rms Testing    = " + str(np.around(min(L_Erms_Test),5)))

Learning to Rank using Linear Regression.py


# -*- coding: utf-8 -*-
"""
This module provides a tool for creating labeled annotations for a SAR image.
"""

__classification__ = "UNCLASSIFIED"
__author__ = "Thomas McCullough"

import os
from typing import Union

import tkinter
from tkinter import ttk
from tkinter.messagebox import showinfo, askyesno, askyesnocancel
from tkinter.filedialog import askopenfilename, asksaveasfilename

from sarpy.annotation.label import LabelSchema

from tk_builder.base_elements import StringDescriptor, TypedDescriptor, BooleanDescriptor

from tk_builder.widgets.basic_widgets import Frame, Label, Entry, Button
from tk_builder.widgets.derived_widgets import TreeviewWithScrolling

from sarpy_apps.supporting_classes import file_filters

###########
# Treeview for a label schema, and associated selection widget

class SchemaViewer(TreeviewWithScrolling):
    """
    For the purpose of viewing the schema definition.
    """

    def __init__(self, master, label_schema=None, **kwargs):
        """

        Parameters
        ----------
        master
            The GUI element which is the master of this node.
        label_schema : None|LabelSchema
            The label schema.
        kwargs
            The keyword argument collection
        """

        self._label_schema = None
        self.master = master

        kwargs['column'] = ('Name', )
        TreeviewWithScrolling.__init__(self, master, **kwargs)
        self.heading('#0', text='Name')
        self.heading('#1', text='ID')

        self.fill_from_label_schema(label_schema)

    def _empty_entries(self):
        """
        Empty all entries - for the purpose of reinitializing.

        Returns
        -------
        None
        """

        self.delete(*self.get_children())

    def delete_entry(self, the_id):
        """
        Delete the given entry.

        Parameters
        ----------
        the_id : str
        """

        # noinspection PyBroadException
        try:
            self.delete(the_id)  # try to delete, and just skip if it fails
        except Exception:
            pass

        if the_id in self._label_schema.labels:
            self._label_schema.delete_entry(the_id, recursive=True)

    def rerender_entry(self, the_id):
        """
        Re(create) the entry for the given id.

        Parameters
        ----------
        the_id : str
        """

        if self._label_schema is None:
            self._empty_entries()
            return

        if the_id is None or the_id == '':
            self.fill_from_label_schema(self._label_schema)
        else:

            # noinspection PyBroadException
            try:
                self.delete(the_id)
            except Exception:
                pass

            if the_id not in self._label_schema.labels:
                return
            parent_id = self._label_schema.get_parent(the_id)
            the_index = self._label_schema.subtypes[parent_id].index(the_id)
            the_label = self._label_schema.labels[the_id]
            self.insert(parent_id, the_index, the_id, text=the_label, values=(the_id,))
            children = self._label_schema.subtypes.get(the_id, None)
            if children is not None:
                for child_id in children:
                    self.rerender_entry(child_id)

    def fill_from_label_schema(self, schema):
        """
        Fill contents from a label schema.

        Parameters
        ----------
        schema : None|LabelSchema
        """

        self._empty_entries()
        self._label_schema = schema
        if self._label_schema is None:
            return
        for element_id in self._label_schema.subtypes['']:
            self.rerender_entry(element_id)

class _SchemaSelectionWidget(object):
    """
    Class for interactive label schema selection widget.
    """

    def __init__(self, label_schema, start_item=None):
        """

        Parameters
        ----------
        label_schema : str|LabelSchema
        start_item : None|str
            The initial item selected
        """

        # validate label schema input
        if isinstance(label_schema, str):
            label_schema = LabelSchema.from_file(label_schema)
        if not isinstance(label_schema, LabelSchema):
            raise TypeError(
                'label_schema must be either a path to am appropriate .json file or a '
                'LabelSchema object. Got type {}'.format(type(label_schema)))

        # initialize the selected value option
        self._selected_value = ''

        self.root = tkinter.Toplevel()
        self.root.geometry('250x400')
        self.root.wm_title('Select Label Schema Entry')

        self.viewer = SchemaViewer(self.root, label_schema=label_schema)
        self.viewer.frame.pack(side=tkinter.TOP, expand=tkinter.TRUE, fill=tkinter.BOTH)
        self.set_selected_value(start_item)
        self.submit_button = Button(self.root, text='Submit', command=self.set_value)
        self.submit_button.pack(side=tkinter.BOTTOM, expand=tkinter.TRUE)
        self.root.grab_set()
        self.root.wait_window()

    def set_value(self):
        self._selected_value = self.viewer.focus()
        self.root.destroy()

    @property
    def selected_value(self):
        """
        str: The id of the selected element.
        """

        return self._selected_value

    def set_selected_value(self, selected_value):
        """
        Set the treeview selection to the provided id.

        Parameters
        ----------
        selected_value : None|str

        Returns
        -------
        None
        """

        self.viewer.set_selection_with_expansion(selected_value)

def select_schema_entry(label_schema, start_id=None):
    """
    Get user selected label schema entry from treeview widget.

    Parameters
    ----------
    label_schema : str|LabelSchema
        The path to a LabelSchema json file, or a LabelSchema object.
    start_id : None|str
        The starting point for the selection.

    Returns
    -------
    str
    """

    selection_widget = _SchemaSelectionWidget(label_schema, start_item=start_id)
    value = selection_widget.selected_value
    return value

#########
# the overall schema editor

class AppVariables(object):
    current_id = StringDescriptor('current_id', default_value=None)  # type: Union[None, str]
    unsaved_edits = BooleanDescriptor('unsaved_edits', default_value=False)  # type: bool
    browse_directory = StringDescriptor(
        'browse_directory', default_value=os.path.expanduser('~'),
        docstring='The directory for browsing for file selection.')  # type: str
    label_file_name = StringDescriptor('label_file_name', default_value=None)  # type: Union[None, str]
    label_schema = TypedDescriptor('label_schema', LabelSchema)  # type: LabelSchema

class LabelEntryPanel(Frame):
    """
    Panel for viewing and editing the details of a given label schema entry.
    """

    def __init__(self, master, app_variables, **kwargs):
        """

        Parameters
        ----------
        master : tkinter.Tk|tkinter.Toplevel
        app_variables : AppVariables
        kwargs
            keyword arguments passed through for frame
        """

        self._app_variables = app_variables
        self._current_id = None
        self._parent_id = None
        self._new_entry = False
        self.id_changed = None  # state variable for external usage

        self.master = master
        Frame.__init__(self, master, **kwargs)
        self.header_message = Label(self, text='', padding=5)
        self.header_message.grid(row=0, column=0, sticky='NSEW', padx=3, pady=3)

        self.frame2 = Frame(self, borderwidth=1, relief=tkinter.RIDGE)
        self.id_label = Label(self.frame2, text='ID:', borderwidth=1, relief=tkinter.RIDGE, padding=5, width=10)
        self.id_label.grid(row=0, column=0, sticky='NW', padx=3, pady=3)
        self.id_entry = Entry(self.frame2, text='')
        self.id_entry.grid(row=0, column=1, sticky='NEW', padx=3, pady=3)

        self.name_label = Label(self.frame2, text='Name:', borderwidth=1, relief=tkinter.RIDGE, padding=5, width=10)
        self.name_label.grid(row=1, column=0, sticky='NW', padx=3, pady=3)
        self.name_entry = Entry(self.frame2, text='')
        self.name_entry.grid(row=1, column=1, sticky='NEW', padx=3, pady=3)

        self.parent_label = Label(self.frame2, text='Parent:', borderwidth=1, relief=tkinter.RIDGE, padding=5, width=10)
        self.parent_label.grid(row=2, column=0, sticky='NW', padx=3, pady=3)
        self.parent_button = Button(self.frame2, text='<Choose>')
        self.parent_button.grid(row=2, column=1, sticky='NEW', padx=3, pady=3)

        self.frame2.grid_columnconfigure(1, weight=1)
        self.frame2.grid(row=1, column=0, sticky='NSEW', padx=3, pady=3)

        self.frame3 = Frame(self, borderwidth=1, relief=tkinter.RIDGE)
        self.cancel_button = Button(self.frame3, text='Cancel')
        self.cancel_button.pack(side=tkinter.RIGHT)
        self.save_button = Button(self.frame3, text='Save')
        self.save_button.pack(side=tkinter.RIGHT)
        self.frame3.grid(row=2, column=0, sticky='NSEW', padx=3, pady=3)
        self.grid_rowconfigure(1, weight=1)
        self.grid_columnconfigure(0, weight=1)
        self.pack(fill=tkinter.BOTH, expand=tkinter.TRUE)

        # callbacks
        self.parent_button.config(command=self.parent_callback)
        self.cancel_button.config(command=self.cancel_callback)
        # save_button bound by controlling parent

        self.update_label_schema()

    @property
    def label_schema(self):
        """
        None|LabelSchema : The label schema.
        """

        return self._app_variables.label_schema

    def update_label_schema(self):
        self.update_current_id()

    @property
    def current_id(self):
        """
        None|str: The current id.
        """

        return self._current_id

    def _set_parent_text(self):
        if self._parent_id is None or self._parent_id == '':
            self.parent_button.set_text('<Top Level>')
        else:
            self.parent_button.set_text(self.label_schema.labels[self._parent_id])

    def update_current_id(self):
        self._current_id = self._app_variables.current_id
        if self.label_schema is None:
            self._new_entry = False
            self._parent_id = None
            self.header_message.set_text('No label schema defined.')
            self.id_entry.set_text('')
            self.id_entry.config(state='disabled')
            self.name_entry.set_text('')
            self.name_entry.config(state='disabled')
            self.parent_button.set_text('')
            self.parent_button.config(state='disabled')
        elif self._current_id is None:
            self._parent_id = None
            self._new_entry = True

            self.header_message.set_text(
                'New entry - <ID> is immutable once initialized and <Name> is for simple interpretation.')

            id_suggestion = self.label_schema.suggested_next_id
            str_id_suggestion = '<ID>' if id_suggestion is None else str(id_suggestion)

            self.id_entry.set_text(str_id_suggestion)
            self.id_entry.config(state='normal')
            self.name_entry.set_text('<Name>')
            self.name_entry.config(state='normal')
            self._set_parent_text()
            self.parent_button.config(state='normal')
        else:
            self._new_entry = False
            self._parent_id = self.label_schema.get_parent(self._current_id)

            self.header_message.set_text(
                '<ID> is immutable, <Name> for simple interpretation.')
            self.id_entry.set_text(self._current_id)
            self.id_entry.config(state='disabled')
            self.name_entry.set_text(self.label_schema.labels[self._current_id])
            self.name_entry.config(state='normal')
            self._set_parent_text()
            self.parent_button.config(state='normal')

    def parent_callback(self):
        """
        Edit or populate the parent id.
        """

        if self.label_schema is None:
            return

        self._parent_id = select_schema_entry(self.label_schema, start_id=self._parent_id)
        self._set_parent_text()

    def cancel_callback(self):
        if self.label_schema is None:
            return

        self.update_current_id()
        self.close_window()
        self.master.grab_release()

    def save_function(self):
        self.id_changed = None
        if self.label_schema is None:
            return True

        the_id = self.id_entry.get()
        the_name = self.name_entry.get()
        the_parent = '' if self._parent_id is None else self._parent_id

        if self._new_entry:
            # if this is a new entry, then verify that both id and name are set
            if the_id == '<ID>' or the_name == '<Name>':
                showinfo('Entries Not Initialized', message='Both `ID` and `Name` must be set.')
                return False

            try:
                self.label_schema.add_entry(the_id, the_name, the_parent=the_parent)
                self.id_changed = the_id
            except Exception as e:
                showinfo("Creation Error", message="Creation error - {}".format(e))
                return False

            self._new_entry = False
            self.update_current_id()
        else:

            try:
                if self.label_schema.change_entry(the_id, the_name, the_parent):
                    self.id_changed = the_id
            except Exception as e:
                showinfo("Edit Error", message="Editing error - {}".format(e))
                return False

        return True

    def hide_on_close(self):
        self.master.protocol("WM_DELETE_WINDOW", self.close_window)

    def close_window(self):
        self.master.withdraw()

class SchemaEditor(Frame):
    """
    An editor for a label schema
    """

    def __init__(self, master, label_schema=None, **kwargs):
        """

        Parameters
        ----------
        master : tkinter.Tk|tkinter.Toplevel
        label_schema : None|str|LabelSchema
        kwargs
            keyword arguments for Frame
        """

        self.variables = AppVariables()
        self.master = master
        Frame.__init__(self, master, **kwargs)

        self.frame1 = Frame(self, borderwidth=1, relief=tkinter.RIDGE)

        self.version_label = Label(
            self.frame1, text='Version Number:', relief=tkinter.RIDGE, justify=tkinter.LEFT, padding=5, width=18)
        self.version_label.grid(row=0, column=0, sticky='NEW')
        self.version_entry = Entry(self.frame1, text='')
        self.version_entry.grid(row=0, column=1, sticky='NEW')

        self.version_date_label = Label(
            self.frame1, text='Version Date:', relief=tkinter.RIDGE, justify=tkinter.LEFT, padding=5, width=18)
        self.version_date_label.grid(row=1, column=0, sticky='NEW')
        self.version_date_entry = Entry(self.frame1, text='')
        self.version_date_entry.grid(row=1, column=1, sticky='NEW')

        self.classification_label = Label(
            self.frame1, text='Classification:', relief=tkinter.RIDGE, justify=tkinter.LEFT, padding=5, width=18)
        self.classification_label.grid(row=2, column=0, sticky='NEW')
        self.classification_entry = Entry(self.frame1, text='')
        self.classification_entry.grid(row=2, column=1, sticky='NEW')

        self.confidence_label = Label(
            self.frame1, text='Confidence Values:', relief=tkinter.RIDGE, justify=tkinter.LEFT, padding=5, width=18)
        self.confidence_label.grid(row=3, column=0, sticky='NEW')
        self.confidence_entry = Entry(self.frame1, text='')
        self.confidence_entry.grid(row=3, column=1, sticky='NEW')

        self.geometries_label = Label(
            self.frame1, text='Geometries:', relief=tkinter.RIDGE, justify=tkinter.LEFT, padding=5, width=18)
        self.geometries_label.grid(row=4, column=0, sticky='NEW')
        self.geometries_entry = Entry(self.frame1, text='')
        self.geometries_entry.grid(row=4, column=1, sticky='NEW')

        self.frame1.grid_columnconfigure(1, weight=1)
        self.frame1.pack(side=tkinter.TOP, fill=tkinter.X)

        self.frame2 = Frame(self, borderwidth=1, relief=tkinter.RIDGE)
        self.new_button = Button(self.frame2, text='New Entry')
        self.new_button.pack(side=tkinter.LEFT, padx=5, pady=5)
        self.edit_button = Button(self.frame2, text='Edit Entry')
        self.edit_button.pack(side=tkinter.LEFT, padx=5, pady=5)
        self.delete_button = Button(self.frame2, text='Delete Entry')
        self.delete_button.pack(side=tkinter.LEFT, padx=5, pady=5)
        self.move_up_button = Button(self.frame2, text='Move Entry Up')
        self.move_up_button.pack(side=tkinter.LEFT, padx=5, pady=5)
        self.move_down_button = Button(self.frame2, text='Move Entry Down')
        self.move_down_button.pack(side=tkinter.LEFT, padx=5, pady=5)
        self.frame2.pack(side=tkinter.TOP, fill=tkinter.X)

        self.schema_viewer = SchemaViewer(self)
        self.schema_viewer.frame.pack(side=tkinter.BOTTOM, expand=tkinter.TRUE, fill=tkinter.BOTH)
        # NB: it's already packed inside a frame

        self.pack(expand=tkinter.YES, fill=tkinter.BOTH)

        # set up the menu bar
        menu = tkinter.Menu()
        filemenu = tkinter.Menu(menu, tearoff=0)
        filemenu.add_command(label="Open Schema", command=self.callback_open)
        filemenu.add_command(label="New Schema", command=self.callback_new_schema)
        filemenu.add_separator()
        filemenu.add_command(label="Save", command=self.save)
        filemenu.add_command(label="Save As", command=self.callback_save_as)
        filemenu.add_separator()
        filemenu.add_command(label="Exit", command=self.exit)
        menu.add_cascade(label="File", menu=filemenu)
        self.master.config(menu=menu)

        # setup entry configs and some validation callbacks
        self.schema_viewer.bind('<<TreeviewSelect>>', self.item_selected_on_viewer)

        self.version_entry.config(
            validate='focusout', validatecommand=self.register(self._version_entry_validate))
        self.version_date_entry.config(state='disabled')
        self.classification_entry.config(
            validate='focusout', validatecommand=self.register(self._classification_validate))
        self.confidence_entry.config(
            validate='focusout', validatecommand=self.register(self._confidence_validate))
        self.geometries_entry.config(
            validate='focusout', validatecommand=(self.register(self._geometry_validate), ),
            invalidcommand=(self.register(self._geometry_invalid), ))
        self.edit_button.config(command=self.callback_edit_entry)
        self.new_button.config(command=self.callback_new_entry)
        self.delete_button.config(command=self.callback_delete_entry)
        self.move_up_button.config(command=self.callback_move_up)
        self.move_down_button.config(command=self.callback_move_down)

        # setup the entry panel
        self.entry_popup = tkinter.Toplevel(self.master)
        self.entry = LabelEntryPanel(self.entry_popup, self.variables)
        self.entry.hide_on_close()
        self.entry_popup.withdraw()

        self.entry.save_button.config(command=self.callback_save_entry)

        self.set_label_schema(label_schema)

    @property
    def label_schema(self):
        """
        None|LabelSchema: The label schema
        """

        return self.variables.label_schema

    # some entry validation methods
    def _version_entry_validate(self):
        the_value = self.version_entry.get().strip()
        if the_value != '' and self.label_schema.version != the_value:
            self.variables.unsaved_edits = True
            self.label_schema._version = the_value
            self.label_schema.update_version_date(value=None)
            self.version_date_entry.set_text(self.label_schema.version_date)
        return True

    def _classification_validate(self):
        the_value = self.classification_entry.get().strip()
        if self.label_schema.classification != the_value:
            self.variables.unsaved_edits = True
            self.label_schema._classification = the_value
        return True

    def _confidence_validate(self):
        the_value = self.confidence_entry.get().strip()
        if the_value == '':
            the_values = None
        else:
            if ',' in the_value:
                temp_values = [entry.strip() for entry in the_value.split(',')]
            else:
                temp_values = the_value.split()

            # noinspection PyBroadException
            try:
                the_values = [int(entry) for entry in temp_values]
            except Exception:
                the_values = temp_values

            # reformat the contents
            self.confidence_entry.delete(0, len(the_value))
            self.confidence_entry.insert(0, ', '.join('{}'.format(entry) for entry in the_values))

        if self.label_schema.confidence_values != the_values:
            self.variables.unsaved_edits = True
            self.label_schema.confidence_values = the_values
        return True

    def _geometry_validate(self):
        the_value = self.geometries_entry.get().strip()

        if the_value == '':
            the_values = None
        else:
            if ',' in the_value:
                the_values = [entry.strip().lower() for entry in the_value.split(',')]
            else:
                the_values = [entry.lower() for entry in the_value.split()]

        if (self.label_schema.permitted_geometries == the_values) or \
                (self.label_schema.permitted_geometries is None and the_values is None):
            return True
        try:
            self.label_schema.permitted_geometries = the_values
            self.variables.unsaved_edits = True

            # reformat the contents
            self.geometries_entry.delete(0, len(the_value))
            self.geometries_entry.insert(0, ', '.join(self.label_schema.permitted_geometries))

            return True
        except ValueError:
            return False

    def _geometry_invalid(self):
        showinfo(
            'geometry type guidance',
            message='The contents for geometry should be a space delimited\n'
                    'combination of {"point", "line", "polygon"}')
        self.geometries_entry.focus_set()
        temp = self.geometries_entry.get()
        self.geometries_entry.delete(0, len(temp))
        if self.label_schema is None or self.label_schema.permitted_geometries is None:
            self.geometries_entry.insert(0, '')
        else:
            self.geometries_entry.insert(0, ' '.join(self.label_schema.permitted_geometries))

    # some helper methods
    def _set_focus_on_entry_popup(self):
        self.entry_popup.deiconify()
        self.entry_popup.focus_set()
        self.entry_popup.lift()
        self.entry_popup.grab_set()

    def _verify_selected(self):
        if self.variables.current_id is None:
            showinfo('No Element Selected', message='Choose element from Viewer')
            return False
        return True

    def _check_save_state(self):
        """
        Checks the save state.

        Returns
        -------
        bool
            Continue (True) or abort (False)
        """

        if not self.variables.unsaved_edits:
            return True

        result = askyesnocancel(
            title="Unsaved Edits",
            message="There are unsaved edits. Save before opening a new file?")
        if result is None:
            return False

        if result is True:
            self.save()
        return True

    def _update_schema_display(self):
        if self.variables.label_schema is None:
            self.version_entry.config(state='disabled')
            self.version_entry.set_text('')
            self.version_date_entry.set_text('')
            self.classification_entry.config(state='disabled')
            self.classification_entry.set_text('')
            self.confidence_entry.config(state='disabled')
            self.confidence_entry.set_text('')
            self.geometries_entry.config(state='disabled')
            self.geometries_entry.set_text('')
        else:
            self.version_entry.config(validate='none')
            self.version_entry.set_text(self.label_schema.version)
            self.version_entry.config(validate='focusout', state='normal')

            self.version_date_entry.set_text(self.label_schema.version_date)

            self.classification_entry.config(validate='none')
            self.classification_entry.set_text(self.label_schema.classification)
            self.classification_entry.config(validate='focusout', state='normal')

            self.confidence_entry.config(validate='none')
            if self.label_schema.confidence_values is None:
                self.confidence_entry.set_text('')
            else:
                self.confidence_entry.set_text(
                    ' '.join('{}'.format(entry) for entry in self.label_schema.confidence_values))
            self.confidence_entry.config(validate='focusout', state='normal')

            self.geometries_entry.config(validate='none')
            if self.label_schema.permitted_geometries is None:
                self.geometries_entry.set_text('')
            else:
                self.geometries_entry.set_text(
                    ' '.join(self.label_schema.permitted_geometries))
            self.geometries_entry.config(validate='focusout', state='normal')

        self.schema_viewer.fill_from_label_schema(self.variables.label_schema)
        self.entry.update_label_schema()

    def prompt_for_filename(self):
        schema_file = asksaveasfilename(
            initialdir=self.variables.browse_directory,
            filetypes=[file_filters.json_files, file_filters.all_files])

        if schema_file == '' or schema_file == ():
            # closed or cancelled
            return False

        self.variables.browse_directory = os.path.split(schema_file)[0]
        self.variables.label_file_name = schema_file
        # todo: what if this file name exists? It should prompt already?
        return True

    def set_label_schema(self, label_schema):
        """
        Sets the label schema value.

        Parameters
        ----------
        label_schema : None|str|LabelSchema
        """

        if label_schema is None:
            self.variables.label_file_name = None
            self.variables.label_schema = LabelSchema()
        elif isinstance(label_schema, str):
            the_file = label_schema
            label_schema = LabelSchema.from_file(the_file)
            self.variables.label_file_name = the_file
            self.variables.label_schema = label_schema
            browse_dir = os.path.split(os.path.abspath(the_file))[0]
            self.variables.browse_directory = browse_dir
        elif isinstance(label_schema, LabelSchema):
            self.variables.label_file_name = None
            self.variables.label_schema = label_schema
        else:
            raise TypeError(
                'input must be the path for a label schema file or a LabelSchema instance')

        self.variables.unsaved_edits = False
        self.variables.current_id = None
        self._update_schema_display()

    def set_current_id(self, value):
        if value == '':
            value = None
        if (value is None and self.variables.current_id is None) or \
                (value == self.variables.current_id):
            return

        self.variables.current_id = value
        self.entry.update_current_id()
        if value is not None:
            self.schema_viewer.set_selection_with_expansion(value)

    # callbacks and bound methods
    def save(self):
        """
        Save any current progress.
        """

        if self.variables.label_file_name is None:
            if not self.prompt_for_filename():
                return  # they opted to not pick a file

        self.label_schema.to_file(self.variables.label_file_name)
        self.variables.unsaved_edits = False

    def exit(self):
        """
        Exit the application.

        Returns
        -------
        None
        """

        if self.variables.unsaved_edits:
            save_state = askyesno('Save Progress', message='There are unsaved edits. Save?')
            if save_state is True:
                self.save()
        self.master.destroy()

    def callback_open(self):
        if not self._check_save_state():
            return

        schema_file = askopenfilename(
            initialdir=self.variables.browse_directory,
            filetypes=[file_filters.json_files, file_filters.all_files])
        if schema_file == '' or schema_file == ():
            # closed or cancelled
            return

        self.set_label_schema(schema_file)

    def callback_new_schema(self):
        if not self._check_save_state():
            return

        self.set_label_schema(LabelSchema())

    def callback_save_as(self):
        if self.prompt_for_filename():
            # they chose the new filename
            self.save()

    def callback_edit_entry(self):
        if not self._verify_selected():
            return
        self._set_focus_on_entry_popup()

    def callback_new_entry(self):
        self.variables.current_id = None
        self.entry.update_current_id()
        self._set_focus_on_entry_popup()

    def callback_delete_entry(self):
        if not self._verify_selected():
            return

        selected = self.variables.current_id
        # does the selected entry have any children?
        children = self.label_schema.subtypes.get(selected, None)
        if children is not None and len(children) > 0:
            response = askyesnocancel('Delete Children?', message='Selected entry has children. Delete all children?')
            if response is not True:
                return
        self.schema_viewer.delete_entry(selected)
        self.variables.unsaved_edits = True
        self.set_current_id(None)

    def callback_move_up(self):
        if not self._verify_selected():
            return
        selected = self.variables.current_id

        result = self.label_schema.reorder_child_element(selected, spaces=-1)
        if result:
            self.schema_viewer.rerender_entry(selected)
            self.variables.unsaved_edits = True

    def callback_move_down(self):
        if not self._verify_selected():
            return
        selected = self.variables.current_id

        result = self.label_schema.reorder_child_element(selected, spaces=1)
        if result:
            self.schema_viewer.rerender_entry(selected)
            self.variables.unsaved_edits = True

    def callback_save_entry(self):
        if self.entry.save_function():
            self.schema_viewer.rerender_entry(self.entry.id_changed)
            self.set_current_id(self.entry.id_changed)
            self.entry.close_window()
            self.entry_popup.grab_release()

    # noinspection PyUnusedLocal
    def item_selected_on_viewer(self, event):
        item_id = self.schema_viewer.focus()
        if item_id == '':
            return
        self.set_current_id(item_id)

def main(label_schema=None):
    """
    Main method for initializing the tool

    Parameters
    ----------
    label_schema : None|str|LabelSchema
    """

    root = tkinter.Tk()

    the_style = ttk.Style()
    the_style.theme_use('classic')

    # noinspection PyUnusedLocal
    app = SchemaEditor(root, label_schema=label_schema)

    root.mainloop()

if __name__ == '__main__':
    if __name__ == '__main__':
        import argparse

        parser = argparse.ArgumentParser(
            description="Open the labeling schema editing tool with optional input file.",
            formatter_class=argparse.RawTextHelpFormatter)

        parser.add_argument(
            'input', metavar='input', default=None,  nargs='?',
            help='The path to the existing schema file for opening.')
        args = parser.parse_args()

        main(label_schema=args.input)


# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
# See the file 'LICENSE' for copying permission.

from django.db import models

from api_app.choices import PythonModuleBasePaths
from api_app.models import AbstractReport, PythonConfig, PythonModule
from api_app.playbooks_manager.models import PlaybookConfig
from api_app.visualizers_manager.exceptions import VisualizerConfigurationException
from api_app.visualizers_manager.validators import validate_report

class VisualizerReport(AbstractReport):
    config = models.ForeignKey(
        "VisualizerConfig", related_name="reports", null=False, on_delete=models.CASCADE
    )
    report = models.JSONField(default=list, validators=[validate_report])
    name = models.CharField(null=True, blank=True, default=None, max_length=50)

    class Meta:
        ordering = ["pk"]

class VisualizerConfig(PythonConfig):
    playbooks = models.ManyToManyField(
        PlaybookConfig,
        related_name="visualizers",
    )
    python_module = models.ForeignKey(
        PythonModule,
        on_delete=models.PROTECT,
        related_name="%(class)ss",
        limit_choices_to={
            "base_path__in": [
                PythonModuleBasePaths.Visualizer.value,
            ]
        },
    )

    @classmethod
    @property
    def plugin_type(cls) -> str:
        return "3"

    @classmethod
    @property
    def config_exception(cls):
        return VisualizerConfigurationException

    @classmethod
    @property
    def serializer_class(cls):
        from api_app.visualizers_manager.serializers import VisualizerConfigSerializer

        return VisualizerConfigSerializer


import re  # Python Regex library
import dns.resolver  # Used to confirm that an email domain exists.

def check_email(email):
    # Simple Regex for syntax checking
    regex = '^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,})$'

    # Email syntax check
    match = re.match(regex, email)
    if match == None:
        return False

    try:
        # Get domain for DNS lookup
        splitAddress = email.split('@')
        domain = str(splitAddress[-1])

        # MX record lookup
        records = dns.resolver.query(domain, 'MX')
        mxRecord = records[0].exchange
        return True
    except:
        return False


"""MEG-DATA Analysis Module - Single Subject"""

import os
import csv
import datetime

import numpy as np
import scipy.io as sio
from scipy.stats.stats import pearsonr

from fooof import FOOOFGroup

from om.core.utils import clean_file_list, get_cur_subj, extract_fooof_group
from om.core.errors import DataNotComputedError, UnknownDataSourceError, InconsistentDataError

###################################################################################################
###################################################################################################

class MegSubj(object):
    """Class for a single subject of fooof results for MEG Source PSDs.

    Attributes
    ----------
    dat_source : {'OMEGA', 'HCP'}
        Which database the MEG subject comes from.
    db : OMDB() object
        Database object of paths for the omegamappin project.
    subnum : int
        Subject number of the subject of MEG data that is loaded.
    n_psds : int
        Number of PSDs loaded, corresponds to number of vertices.
    exponents : 1d array
        Aperiodic exponent values for each vertex.
    centers : 2d array
        Oscillation centers, for each vertex [n_verts, n_oscs].
    powers : 2d array
        Oscillation powers, for each vertex [n_verts, n_oscs].
    bws : 2d array
        Oscillation bandwidths, for each vertex [n_verts, n_oscs].
    centers_all : 1d array
        Vector of all oscillation centers, collapsed across vertices.
    powers_all : 1d array
        Vector of all oscillation powers, collapsed across vertices.
    bws_all : 1d array
        Vector of all oscillation bandwidths, collapsed across vertices.
    n_oscs : int
        Number of oscillations extracted for the subject, across all vertices.
    centers_hist : 1d array
        Center frequencies binned into histogram format.
    bands : Osc() object
        Oscillation band definitions.
    oscs : dict
        Oscillations, split up by band, per vertex.
    peaks : dict
        Peak frequencies within each oscillation band.
    comment : str
        Note on the current data, typically subject number, can be used for plotting.
    sex : {'M', 'F'}
        Sex of the current subject.
    age : int
        Age of the current subject.
    osc_count : 1d array
        Number of oscillations extracted per vertex.
    has_data : boolean
        Whether or not data has been loaded to current object.
    has_vertex_oscs : boolean
        Whether data is defined for oscillations at each vertex (not in bands).
    has_vertex_exponents : boolean
        Whether aperiodic exponent values per vertex are loaded.
    has_all_osc : boolean
        Whether data has been flattened into all oscillations.
    has_vertex_bands : boolean
        Whether data has been converted into band specific oscillations, per vertex.
    has_peak_freqs : boolean
        Whether peak frequencies, within oscillatory bands, have been calculated.
    has_demo : boolean
        Whether demographic data has been loaded.
    """

    def __init__(self, db, dat_source, osc=None):
        """Initialize object with omegamappin database.

        Parameters
        ----------
        db : OMDB() object
            Database object for omegamappin project.
        osc : Osc() object, optional
            Oscillation band definitions.
        """

        # Store which db is set
        self.dat_source = dat_source

        # Add database object
        self.db = db

        # Initialize subject number
        self.subnum = int()
        self.n_psds = int()

        # Initialize data arrays
        self.exponents = np.array([])
        self.centers = np.array([])
        self.powers = np.array([])
        self.bws = np.array([])

        # Initialize arrays for all-osc data
        self.centers_all = np.array([])
        self.powers_all = np.array([])
        self.bws_all = np.array([])
        self.n_oscs = np.array([])

        # Initialize list to store all centers histogram
        #  This is used for the subject specific oscillation analysis/plots
        self.centers_hist = []

        # Initialize dictionary to store osc bands
        if osc:
            self.bands = osc.bands
        else:
            self.bands = dict()

        # Initialize dictionary for oscillation bands
        self.oscs = dict()

        # Initialize dictionary for peak frequencies
        self.peaks = dict()

        # Set plot comment
        self.comment = ''

        # Set boolean for what has been run
        self.has_data = False
        self.has_vertex_oscs = False
        self.has_vertex_exponents = False
        self.has_all_osc = False
        self.has_vertex_bands = False
        self.has_peak_freqs = False
        self.has_demo = False

        # Initialize demographic variables
        self.sex = str()
        self.age = int()

        # Initialize oscillation count
        self.osc_count = int()

    def set_bands(self, osc):
        """

        Parameters
        ----------
        osc : ?
            xx
        """

        if self.bands and (self.has_vertex_bands or self.peaks):
            raise InconsistentDataError("Can't change band definitions after they have been used.")
        else:
            self.bands = osc.bands

    def import_fooof(self, subnum, get_demo=True):
        """Import fooof results to MegSubj object.

        Parameters
        ----------
        subnum : int
            Number of the subject to import.
        get_demo : boolean, optional (default = True)
            Whether to load demographic data from csv file.
        """

        # Check if object already has data
        if self.has_data:
            raise InconsistentDataError('Subject object already contains data. Can not add')

        # Set subject number for current data object
        self.subnum = subnum
        self.comment = 'S-' + str(self.subnum)

        # Set up paths, get list of files for available subjects
        files = os.listdir(os.path.join(self.db.fooof_path, self.dat_source))
        files = clean_file_list(files, 'fooof_Vertex')

        # Get specific file name, and set up full file path for specified subject
        cur_subj_file = get_cur_subj(subnum, files)
        cur_subj_path = os.path.join(self.db.fooof_path, self.dat_source, cur_subj_file)

        # Load FOOOF data file
        fg = FOOOFGroup()
        fg.load(cur_subj_path)
        self.centers, self.powers, self.bws, self.exponents, self.n_psds = extract_fooof_group(fg)

        # Update which data is loaded
        self.has_vertex_oscs = True
        self.has_vertex_exponents = True

        # Check how many oscillations per vertex
        self.osc_count = np.zeros([self.n_psds])
        for i in range(self.n_psds):
            self.osc_count[i] = len(np.nonzero(self.centers[i, :])[0])

        # Get demographic data
        if get_demo:
            self.has_demo = True
            self.sex, self.age = _get_demo_csv(self.subnum, self.db.meg_path, self.dat_source)

        # Update boolean to say current subject has data attached
        self.has_data = True

    def all_oscs(self, verbose=True):
        """Flatten osc data to vectors.

        Parameters
        ----------
        verbose : boolean
            Whether to print out status as function runs.

        Notes
        -----
        - When imported, oscillation data is in matrix form [n_vertex, osc_slots].
            This functions converts these matrices into 1-D vectors.
        - This function loses information about which vertex oscillations occur at.
        """

        # Flatten osc data into vectors. Uses C-style row-major order
        self.centers_all = self.centers.flatten('C')
        self.powers_all = self.powers.flatten('C')
        self.bws_all = self.bws.flatten('C')

        # Flattened vectors will have lots of zeros. Get only non-zero indices.
        # TODO: note - currently checks based on powers : hack for dropping 0 amplitudes
        non_zeros = np.nonzero(self.powers_all)
        self.centers_all = self.centers_all[non_zeros]
        self.powers_all = self.powers_all[non_zeros]
        self.bws_all = self.bws_all[non_zeros]

        # Check for nans in BW estimation
        # NOTE: Updated fooof sometimes returns NaN for bw. Check and discard those.
        nans = np.isnan(self.bws_all)
        n_nans = sum(nans)

        # If there are nans, print how many and remove them
        if n_nans > 0:
            if verbose:
                print('Subj:', str(self.subnum), 'Found', str(n_nans), ' NaNs. Removing.')

            # Remove osc's with nan values
            non_nans = np.logical_not(nans)
            self.centers_all = self.centers_all[non_nans]
            self.powers_all = self.powers_all[non_nans]
            self.bws_all = self.bws_all[non_nans]

        # Create the all-centers histogram
        self.centers_hist, _ = np.histogram(self.centers_all, bins=np.arange(3, 40.25, 0.25))

        # Get the number of oscillations
        self.n_oscs = len(self.centers_all)

        # Update boolean that all-oscillations has been computed
        self.has_all_osc = True

    def osc_bands_vertex(self):
        """Groups oscillations at each vertex in distinct frequency bands.

        Stores band specific oscillations in (self.){thetas, alphas, betas, lowgammas}.
        """

        # Check that oscillation bands are defined
        if not self.bands:
            raise DataNotComputedError('Oscillation bands not specified, can not proceed.')

        # Initialize matrices to store oscillations in each band
        for band in self.bands:
            self.oscs[band] = np.zeros([self.n_psds, 4])

        # Loop through each vertex
        for i in range(self.n_psds):

            # Get centers, powers and bws from individual vertex
            centers_temp = self.centers[i, :]
            powers_temp = self.powers[i, :]
            bws_temp = self.bws[i, :]

            # Loop through each oscillation band
            for band in self.bands:

                # Get oscillations in specific band
                self.oscs[band][i, :] = _get_single_osc(centers_temp, powers_temp, bws_temp,
                                                        self.bands[band][0], self.bands[band][1])

        # Update boolean to note that current subject has band specific oscs calculated.
        self.has_vertex_bands = True

    def peak_freq(self, dat, avg='mean'):
        """Calculates the peak frequency for each oscillatory band.

        Parameters
        ----------
        dat : {'all', 'band'}
            Which data to use to calculate peak frequency.
        avg : {'mean', 'median'}, optional
            Which type of averaging to do.
        """

        # Check that oscillation bands are defined
        if not self.bands:
            raise DataNotComputedError('Oscillation bands not specified, can not proceed.')

        # Calculate peak frequency from all oscillations, across all vertices, within band range
        if dat is 'all':

            # Check all osc data has been computed
            if not self.has_all_osc:
                raise DataNotComputedError('All Osc data has not been computed. Can not continue.')

            # Loop through each band, calculating peak frequency
            for band in self.bands:
                self.peaks[band] = _osc_peak_all(
                    self.centers_all, self.bands[band][0], self.bands[band][1], avg)

        # Calculate peak frequency from within band specific maps, which have enforced
        #  a single oscillation per band, per vertex, choosing highest power oscillations
        elif dat is 'band':

            if not self.has_vertex_bands:
                raise DataNotComputedError('Bands not computed per vertex, can not continue.')

            for band in self.bands:

                non_zero_inds = np.nonzero(self.oscs[band][:, 0])

                if avg is 'mean':
                    self.peaks[band] = np.mean(self.oscs[band][non_zero_inds, 0])
                elif avg is 'median':
                    self.peaks[band] = np.median(self.oscs[band][non_zero_inds, 0])

        # Update tracking of what data has been loaded/computed
        self.has_peak_freqs = True

    def calc_osc_param_corrs(self):
        """Calculates correlations between oscillatory parameters.

        Returns
        -------
        corrs_mat : ?
            xx
        ps_mat : ?
            xx
        labels : ?
            xx
        """

        # Check all osc data has been computed
        if not self.has_all_osc:
            raise DataNotComputedError('All Osc data has not been computed. Can not continue.')

        # Set labels for the things being correlated
        labels = ['Centers', 'Powers', 'Bandwidths']

        # Check how many categories there are
        n_label = len(labels)

        # Initialize matrices to store R and p values
        corrs_mat = np.zeros([n_label, n_label])
        ps_mat = np.zeros([n_label, n_label])

        # Calculate correlations between all parameters
        corrs_mat[0, 1], ps_mat[0, 1] = pearsonr(self.centers_all, np.log10(self.powers_all))
        corrs_mat[0, 2], ps_mat[0, 2] = pearsonr(self.centers_all, np.log10(self.bws_all))
        corrs_mat[1, 2], ps_mat[1, 2] = pearsonr(np.log10(self.powers_all),
                                                 np.log10(self.bws_all))

        # Duplicate matrix across the diagonal
        corrs_mat = corrs_mat + corrs_mat.T
        ps_mat = ps_mat + ps_mat.T

        return corrs_mat, ps_mat, labels

    def set_fooof_viz(self):
        """Saves a matfile of freq info to be loaded with Brainstorm for visualization.

        Notes
        -----
        Requires that ... data be
        """

        # Set up paths to save to
        save_name = str(self.subnum) + '_fooof_Viz'
        save_file = os.path.join(self.db.viz_path, save_name)

        # Initialize dictionary, save basic information and exponent data
        save_dict = {}
        save_dict['subnum'] = self.subnum
        save_dict['dat_source'] = self.dat_source
        save_dict['save_time'] = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        save_dict['exponents'] = self.exponents

        # Save out oscillation data
        for band in self.bands:
            save_dict[band.lower()] = self.oscs[band]

        # Save the dicionary out to a .mat file
        sio.savemat(save_file, save_dict)

############################################################################################
########################## OMEGAMAPPIN - OM_MD - PUBLIC FUNCTIONS ##########################
############################################################################################

def print_corrs_mat(rs_mat, ps_mat, labels):
    """Prints out correlations, from a given matrix.

    Parameters
    ----------
    rs_mat : 2d array
        Matrix of R values.
    ps_mat : 2d array
        Matrix of p-values.
    labels : list of str
        Labels for the rows & columns.
    """

    # Check how size of the matrix there are
    n_label = len(labels)

    # Loop through the matrix to print out
    for x_dat in range(n_label):
        for y_dat in range(n_label):

            # Skip bottom triangle and diagonal
            if x_dat == y_dat or y_dat < x_dat:
                continue

            # Print out correlation
            print('Corr of ', '{:18}'.format(labels[x_dat]+'-'+labels[y_dat]),
                  ' is ', '{:+1.4f}'.format(rs_mat[x_dat, y_dat]), '    with p-val of ',
                  '{:1.5f}'.format(ps_mat[x_dat, y_dat]))

def print_corrs_vec(rs_vec, ps_vec, labels, desc):
    """Prints out correlations, from a given vector.

    Parameters
    ----------
    rs_vec : 1d array
        Vector of R values.
    ps_vec : 1d array
        Vector of p-values.
    labels : list of str
        Labels for the columns.
    desc : str
        Label for the row.
    """

    # Check the length of the vector
    n_label = len(labels)

    # Loop through vectors, printing out the correlations.
    for cur_label in range(n_label):
        print('Corr of ', '{:20}'.format(labels[cur_label]+'-'+desc), ' is ',
              '{:+1.4f}'.format(rs_vec[cur_label]), '    with p-val of ',
              '{:1.5f}'.format(ps_vec[cur_label]))

#################################################################################################
############################ OMEGAMAPPIN - OM_MD - PRIVATE FUNCTIONS ############################
#################################################################################################

def _get_single_osc(centers, powers, bws, osc_low, osc_high):
    """ Searches for an oscillations of specified frequency band.

    Returns a single oscillation in that band.
    Helper function for osc_per_vertex in MegSubj.

    Parameters
    ----------
    centers : 1d array
        Vector of oscillation centers.
    powers : 1d array
        Vector of oscillation powers.
    bws : 1d array
        Vector of oscillation bandwidths.
    osc_low : float
        Lower bound of frequency band to extract.
    osc_high : float
        Upper bound of frequency band to extract.

    Returns
    -------
    osc_out : array
        Osc data, form - (centers, powers, bws, # oscillations).
    """

    # Find indices of oscillations in the specified range
    osc_inds = (centers >= osc_low) & (centers <= osc_high)

    # Get cen, pow & bw for oscillations in specfied range
    osc_cens = centers[osc_inds]
    osc_pows = powers[osc_inds]
    osc_bws = bws[osc_inds]

    # Get number of oscillations in the frequency band.
    n_oscs = len(osc_cens)

    # Get highest power oscillation in band
    cen, power, bw = _get_single_osc_power(osc_cens, osc_pows, osc_bws)

    return np.array([cen, power, bw, n_oscs])

def _get_single_osc_power(osc_cens, osc_pows, osc_bws):
    """Return the highest power oscillation in a given range.

    Parameters
    ----------
    osc_cens : 1d array
        Vector of oscillation centers.
    osc_pows : 1d array
        Vector of oscillation powers.
    osc_bws : 1d array
        Vector of oscillation bandwidths.

    Returns
    -------
    center : float
        Center frequency value of highest power oscillation.
    power : float
        Power value of highest power oscillation.
    bw : float
        Bandwidth of highest power oscillation.
    """

    # Return zeros if there are no oscillations in given vectors
    if len(osc_cens) == 0:
        return 0., 0., 0.

    # If singular oscillation, return that oscillation
    elif len(osc_cens) == 1:
        return osc_cens, osc_pows, osc_bws

    # If multiple oscillations, return the one with the highest power
    else:
        high_ind = np.argmax(osc_pows)
        return osc_cens[high_ind], osc_pows[high_ind], osc_bws[high_ind]

def _get_demo_csv(subnum, meg_path, dat_source, use_restricted=True):
    """Get demographic information from csv file for specified subject.

    Parameters
    ----------
    subnum : int
        Subject number to get demographic info for.
    meg_path : str
        String for path to csv file.
    dat_source: {'OMEGA', 'HCP'}
        Which database subject is from.

    Returns
    -------
    sex : {'M', 'F'}
        Sex ['M'/'F'] of specified subject.
    age : int
        Age (in whole years) of specified subject.
    """

    # Set up paths for demographic info csv file
    if dat_source is 'OMEGA':
        num_ind = 1
        sex_ind = 4
        age_ind = 7
        res = ''
    elif dat_source is 'HCP':
        if use_restricted:
            num_ind = 0
            sex_ind = 6
            age_ind = 1
            res = '_RESTRICTED_wgender'
        else:
            res = ''
            num_ind = 0
            sex_ind = 3
            age_ind = 4

    else:
        raise UnknownDataSourceError('Unrecognized database source to load from.')

    # Set up name of CSV file to use
    csv_file = os.path.join(meg_path, '00-' + dat_source + '_Subjects' + res + '.csv')

    # Open csv file, loop through looking for right row, grab age & sex information
    with open(csv_file, 'r') as f_name:
        reader = csv.reader(f_name, delimiter=',')

        for row in reader:

            # Find subject line in file
            if row[num_ind] == str(subnum):

                #
                if dat_source is 'OMEGA':
                    sex = row[sex_ind]
                    age = int(row[age_ind])

                elif dat_source is 'HCP' and not use_restricted:
                    sex = row[sex_ind]
                    age_temp = (row[age_ind]).split('-')
                    age = (int(age_temp[0]) + int(age_temp[1]))/2

                else:
                    sex = row[sex_ind]
                    age = int(row[age_ind])

                break

    return sex, age

def _osc_peak_all(centers, osc_low, osc_high, avg='mean'):
    """Find the peak-frequency of a vector of center frequencies.

    Parameters
    ----------
    centers : 1d vector
        Vector of center frequencies to use.
    osc_low : float
        Lower bound of frequency range to check.
    osc_high : float
        Upper bound of frequency range to check.
    avg : {'mean', 'median'}, optional (default = 'mean')
        What kind of average to take.

    Returns
    -------
    peak : float
        Peak frequency value, the average frequency within a given range.
    """

    # Pull out all center frequencies between given range
    osc_inds = (centers > osc_low) & (centers < osc_high)
    osc_cens = centers[osc_inds]

    # Take the average of the center frequencies
    if avg is 'mean':
        peak = np.mean(osc_cens)
    elif avg is 'median':
        peak = np.median(osc_cens)

    return peak


import tushare as ts
import pandas as pd
import matplotlib.pyplot as plt
plt.figure(figsize=(40,30))
def get_stock_price(code0,start0,end0,ktype0):
    data = ts.get_hist_data(code0,start=start0,end=end0,ktype=ktype0)
    data.loc[data['p_change']<-4,'p_change']=1
    y=data['p_change']
    fig=plt.plot(y,label='p_change')
    y1=data['close']
    fig=plt.plot(y1,label='close_price')
    plt.legend()
    plt.xlabel('TIME')
    plt.xticks(rotation=45)
    plt.show()
start = '2016-01-01'
end = '2018-03-30'
code = '600315'
ktype = 'W'

get_stock_price(code,start,end,ktype)


import pytest
from flask import Flask
from api import app

@pytest.fixture
def client():
    app.config['TESTING'] = True
    with app.test_client() as client:
        yield client

def test_get_domains(client):
    response = client.get('/domains')
    assert response.status_code == 200
    # Assert other conditions based on the expected behavior of the API

# Write more test cases for other API endpoints


from pathlib import Path

import cv2
import face_recognition
import numpy as np
import tensorflow as tf
from keras.models import model_from_json

def main():
    BASE_DIR = Path(__file__).resolve().parent
    MODEL = "mobilenetv2"
    MODELS_DIR = BASE_DIR / "models"

    MODEL_WEIGHTS = MODELS_DIR / f"{MODEL}.h5"
    MODEL_JSON = MODELS_DIR / f"{MODEL}.json"
    IMAGE_SIZE = 224
    SOURCE_INPUT = 0
    SCALE_FACTOR = 2

    input_stream = cv2.VideoCapture(SOURCE_INPUT)

    fer_model = model_from_json(open(MODEL_JSON, "r").read())
    fer_model.load_weights(MODEL_WEIGHTS)

    emotions_labels = [
        "nervoso",
        "nojo",
        "medo",
        "feliz",
        "neutro",
        "triste",
        "surpreso",
    ]

    while True:
        bol, frame = input_stream.read()

        frame_resized = cv2.resize(
            frame,
            (0, 0),
            fx=1 / SCALE_FACTOR,
            fy=1 / SCALE_FACTOR,
            interpolation=cv2.INTER_AREA,
        )

        face_locations = face_recognition.face_locations(
            img=frame_resized,
            number_of_times_to_upsample=1,
            model="hog",
        )

        for face, face_location in enumerate(face_locations):
            top, right, bottom, left = face_location

            top *= SCALE_FACTOR
            right *= SCALE_FACTOR
            bottom *= SCALE_FACTOR
            left *= SCALE_FACTOR

            print(f"Face {face+1}, top: {top}, right: {right}, bottom: {bottom}, left: {left}")

            face_image = frame[top:bottom, left:right]
            face_image = cv2.resize(face_image, (IMAGE_SIZE, IMAGE_SIZE))

            face_image = tf.keras.preprocessing.image.img_to_array(face_image)

            img_pixels = np.expand_dims(face_image, axis=0)
            img_pixels /= 255

            predictions = fer_model.predict(img_pixels)
            max_index = np.argmax(predictions[0])

            emotion = emotions_labels[max_index]

            cv2.putText(
                frame,
                emotion.upper(),
                (left, top - 20),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.6,
                (0, 255, 0),
                2,
            )

            cv2.rectangle(
                frame,
                (left, top),
                (right, bottom),
                (255, 0, 0),
                2,
            )

        cv2.imshow("Video Stream", frame)

        if cv2.waitKey(1) & 0xFF == ord("q"):
            break

    input_stream.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    main()


# TRAINING THE CNN MODEL
# Note: Will most likely have to plot loss and accuracy after
# code since live noggin plots can't be used with this keras setup

# (EPOCHS) 25
# (BATCH-SIZE) 100
# (LEARNING RATE) 0.001

from tensorflow.keras.optimizers import Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow import convert_to_tensor
from .model import *
import keras.backend as K
import numpy as np
import pickle
import os
genres = ('blues', 'classical', 'country', 'disco', 'hiphop', 'metal', 'pop', 'reggae', 'rock')

# <<< VERSION ONE >>> (PNGS)
"""
# directory of training data (spectrograms)
train_dir = "recognition/data/spectrograms/"
# augments image data for batch generation
#   rescale: scales rgb images data down so model can process
train_datagen = ImageDataGenerator(rescale=1./255)
# navigates to directory and generates batches of augmented data
#   target_size: dimensions to resize images (288, 432)
#   color_mode: color channels of images (rgb is 4 channels)
#   class_mode: type of label arrays returned (categorical is 2D one-hot encoded labels)
#   batch_size: size of batches of data (100)
# returns iterable yielding tuples of (x, y), where x is np array containing
# batch of images and y is np array containing respective batch of labels
train_generator = train_datagen.flow_from_directory(train_dir, target_size=(288,432), color_mode="rgb", class_mode='categorical', batch_size=100)
# repeat same process for validation data (testing_data)
validation_dir = "recognition/data/testing_data/"
valid_datagen = ImageDataGenerator(rescale=1./255)
valid_generator = valid_datagen.flow_from_directory(validation_dir, target_size=(288,432), color_mode='rgb', class_mode='categorical', batch_size=100)
"""

def get_f1(y_true, y_pred):
    """
    docstring
    """
    # computing accuracy (f1)
    true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives+K.epsilon())
    recall = true_positives / (possible_positives+K.epsilon())
    f1_val = 2 * (precision*recall) / (precision+recall+K.epsilon())
    return f1_val

# <<< VERSION ONE >>> (PNGS)
"""
def train():
    # create model
    model = GenreModel(input_shape=(288, 432, 3), classes=9)
    # create optimizer
    optim = Adam(learning_rate=0.001)
    # configure the model for training
    model.compile(optimizer = optim, loss='categorical_crossentropy', metrics=['accuracy', get_f1])
    # training: fits the model on data yielded batch-by-batch by a python generator
    history = model.fit_generator(train_generator, epochs=100, validation_data=valid_generator)
    # saves history as pickle dictionary
    with open('recognition/data/history_backup/history_dict.pkl', 'wb') as file_pi:
        pickle.dump(history.history, file_pi)
    # return model and history
    return (model, history)
"""

# <<< VERSION TWO >>> (ARRAYS)
def train():
    one_hots = {"blues":[1,0,0,0,0,0,0,0,0], "classical":[0,1,0,0,0,0,0,0,0],
            "country":[0,0,1,0,0,0,0,0,0], "disco":[0,0,0,1,0,0,0,0,0],
            "hiphop":[0,0,0,0,1,0,0,0,0], "metal":[0,0,0,0,0,1,0,0,0],
            "pop":[0,0,0,0,0,0,1,0,0], "reggae":[0,0,0,0,0,0,0,1,0],
            "rock":[0,0,0,0,0,0,0,0,1]}
    x_train = []
    y_train = []
    x_test = []
    y_test = []
    for g in genres:
        print("Training: " + str(g))
        # iterate through the newly sliced 5400 five-second sound files
        for filename in os.listdir(os.path.join("recognition/data/spectrograms/", f"{g}")):
            # training data
            filedata = np.load(f"recognition/data/spectrograms/{g}/{filename}")
            filedata = np.resize(filedata,(128, 216))
            x_train.append(filedata)
            # training labels
            y_train.append(one_hots[g])
        print("Validation: " + str(g))
        # iterate through the newly sliced 5400 five-second sound files
        for filename in os.listdir(os.path.join("recognition/data/testing_data/", f"{g}")):
            # testing data
            filedata = np.load(f"recognition/data/testing_data/{g}/{filename}")
            filedata = np.resize(filedata,(128, 216))
            x_test.append(filedata)
            # testing labels
            y_test.append(one_hots[g])
    #x_train = np.concatenate([x_train[i] for i in range(len(x_train))])
    print(len(x_train))
    x_train = np.array(x_train)
    print(x_train.shape)
    y_train = np.array(y_train)
    print(y_train)
    print(len(x_train))
    x_test = np.array(x_test)
    print(x_train.shape)
    y_test = np.array(y_test)
    print(y_train)
    validation_data=(x_test, y_test)
    #x_train = np.array(x_train)
    #y_train = np.array(y_train)
    #x_test = np.array(x_test)
    #y_test = np.array(y_test)
    #x_train = convert_to_tensor(x_train)
    #y_train = convert_to_tensor(y_train)
    #x_test = convert_to_tensor(x_test)
    #y_test = convert_to_tensor(y_test)

    # create model
    model = GenreModel(input_shape=(128, 216, 1), classes=9)
    # create optimizer
    optim = Adam(learning_rate=0.001)
    # configure the model for training
    model.compile(optimizer=optim, loss='categorical_crossentropy', metrics=['accuracy', get_f1])
    # training: fits the model on data yielded batch-by-batch by a python generator
    print(type(x_train))
    print(type(y_train))
    print(type(x_test))
    print(type(y_test))
    print(type(validation_data))
    history = model.fit(x_train, y_train, batch_size=100, epochs=25, validation_data=validation_data)
    # saves history as pickle dictionary
    with open('recognition/data/history_backup/history_dict.pkl', 'wb') as file_pi:
        pickle.dump(history.history, file_pi)
    # return model and history
    return (model, history)


from re import I
from django.shortcuts import render
from rest_framework.decorators import action
from .models import Listing, Transaction, UserProfile, Review
from .serializers import ListingSerializer, TransactionSerializer, TutorSerializer, UserProfileSerializer, ReviewSerializer
from rest_framework.permissions import IsAuthenticated, IsAuthenticatedOrReadOnly
from rest_framework import serializers, viewsets, mixins, generics, status, views
from rest_framework.response import Response
from api.permissions import ProfilePermission, ListingPermission, TransactionPermission
from rest_framework.permissions import AllowAny
from django.contrib.auth import get_user_model
from rest_framework.generics import get_object_or_404
from allauth.account.admin import EmailAddress
from allauth.account.utils import send_email_confirmation
from rest_framework.exceptions import APIException
# Create your views here.

User = get_user_model()

class UserProfileAPIView(generics.RetrieveUpdateAPIView):
	serializer_class = UserProfileSerializer
	permission_classes = [ProfilePermission]

	def get_queryset(self):
			return UserProfile.objects.all()

class NewEmailConfirmation(views.APIView):
	permission_classes = [AllowAny]

	def post(self, request):
		user = get_object_or_404(User, email=request.data['email'])
		emailAddress = EmailAddress.objects.filter(user=user, verified=True).exists()

		if emailAddress:
			return Response({'message': 'This email is already verified'}, status=status.HTTP_400_BAD_REQUEST)
		else:
			try:
				send_email_confirmation(request, user=user)
				return Response({'message': 'Email confirmation sent'}, status=status.HTTP_201_CREATED)
			except APIException:
				return Response({'message': 'This email does not exist, please create a new account'}, status=status.HTTP_403_FORBIDDEN)

class ListingViewSet(viewsets.ModelViewSet):
	serializer_class = ListingSerializer
	permission_classes = [ListingPermission, IsAuthenticatedOrReadOnly]

	def get_queryset(self):
		user = self.request.GET.get('user')
		mod_code = self.request.GET.get('mod_code')
		if user:
			return Listing.objects.filter(owner=user).order_by("date_created")
		elif mod_code:
			return Listing.objects.filter(mod_code__icontains=mod_code).order_by("-owner__xp")
		else:
			return Listing.objects.all()

class TransactionViewSet(viewsets.ModelViewSet):
	serializer_class = TransactionSerializer
	permission_classes = [TransactionPermission, IsAuthenticatedOrReadOnly]

	def get_queryset(self):
			return Transaction.objects.all()

	@action(detail=False)
	def accept(self, request):
		# pk here is the student id
		listing_id = self.request.GET.get('listing')
		student = self.request.GET.get('student')
		if not student or not listing_id:
			return Response({"message": "please provide valid listing and student id"}, status=status.HTTP_400_BAD_REQUEST)
		listing = get_object_or_404(Listing, id=listing_id)
		if listing.owner.pk != request.user.id:
			return Response({"message": "you are not the owner of this listing!"}, status=status.HTTP_401_UNAUTHORIZED)
		try:
			transaction = listing.transactions.get(student=student)
		except Transaction.DoesNotExist:
			return Response({"message": "transaction does not exist"}, status=status.HTTP_400_BAD_REQUEST)

		transaction.is_accepted = True
		transaction.save(update_fields=['is_accepted'])
		return Response({"message": "accepted student (" + str(transaction.student) + ")"}, status=status.HTTP_200_OK)

	@action(detail=True)
	def students(self, request, pk=None):
		user = request.user.id
		listing = get_object_or_404(Listing, id=pk)
		if user != listing.owner.pk:
			return Response(status=status.HTTP_401_UNAUTHORIZED)
		transactions = listing.transactions.all()
		# requested = transactions.filter(is_accepted=False)
		# accepted = transactions.filter(is_accepted=True)
		# requested_serializer = TransactionSerializer(requested, many=True)
		# accepted_serializer = TransactionSerializer(accepted, many=True)
		tx_serializer = TransactionSerializer(transactions, many=True)
		return Response( tx_serializer.data
		# 	{
		# # 	"requested": requested_serializer.data,
		# # 	"accepted": accepted_serializer.data
		# }
		)

	@action(detail=False)
	def tutors(self, request):
		pk = request.user.id
		if not pk:
			return Response(status=status.HTTP_401_UNAUTHORIZED)
		print(pk)
		transactions = Transaction.objects.filter(student=pk)
		print(transactions)
		tutor_serializer = TutorSerializer(transactions, many=True)
		# requested = transactions.filter(is_accepted=False)
		# accepted = transactions.filter(is_accepted=True)
		# requested_serializer = TutorSerializer(requested,many=True)
		# accepted_serializer = TutorSerializer(accepted,many=True)
		return Response(
			tutor_serializer.data
		# 	{
		# 	"requested": requested_serializer.data,
		# 	"accepted": accepted_serializer.data
		# }
		)

	@action(detail=True)
	def status(self, request, pk=None):
		# pk here is listing id
		listing = get_object_or_404(Listing, id=pk)
		user = request.user.id
		if listing.owner.pk == user:
			return Response ({
				"user_type": "owner"
			})

		try:
			transaction = listing.transactions.get(student=user)
		except Transaction.DoesNotExist:
			return Response({"user_type": ""})

		is_accepted = transaction.is_accepted
		gave_review = transaction.gave_review
		return Response({
			"user_type": "student",
			"is_accepted": is_accepted,
			"gave_review": gave_review
		})

class ReviewVewSet(viewsets.ViewSet):
	permission_classes = [IsAuthenticatedOrReadOnly]

	def retrieve(self, request, pk=None):
		queryset = Review.objects.filter(tutor=pk)
		serializer = ReviewSerializer(queryset, many=True)
		return Response(serializer.data)

	def create(self, request):
		if not request.user.id:
			return Response(status=status.HTTP_401_UNAUTHORIZED)
		request.data["student"] = request.user.id
		serializer = ReviewSerializer(data=request.data)
		if serializer.is_valid():
			serializer.save()
			return Response(serializer.data, status=status.HTTP_201_CREATED)
		return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


#!/usr/bin/python3
"""Defines a Module for the HBNBCommand"""

import cmd
import os
import importlib
import inspect
import pkgutil
from models import storage
from models.base_model import BaseModel
from models.user import User
from models.state import State
from models.city import City
from models.place import Place
from models.amenity import Amenity
from models.review import Review

class HBNBCommand(cmd.Cmd):
    """HBNBCommand class"""
    prompt = '(hbnb) '

    def __init__(self, *args, **kwargs) -> None:
        """
        Initializes a new instance of HBNBCommand class.

        Args:
            *args: Variable length argument list.
            **kwargs: Arbitrary keyword arguments.

        Attributes:
            __class_names (dict): Dictionary containing the names of the
                classes that inherit from BaseModel and their respective
                module names.
        """
        super().__init__(*args, **kwargs)
        self.__class_names = self.get_class_names()

    def get_class_names(self):
        """
        Dynamically discover class names that inherit from BaseModel.
        """
        class_names = {}
        for _, file_name, _ in pkgutil.iter_modules(['models']):
            module_name = os.path.splitext(file_name)[0]
            module = importlib.import_module('models.' + module_name)

            for name, obj in inspect.getmembers(module):
                if inspect.isclass(obj) and issubclass(obj, BaseModel):
                    class_names[name] = 'models.' + module_name
        return class_names

    def default(self, line):
        """Process a command line based on predefined syntax.

        This method parses the given command line and executes the
        corresponding command based on the syntax rules.

        Usage: <class name>.<command name>(<arg1>, <arg2>, ...)

        """
        cmd_dict: dict = {"all": self.do_all,
                          "count": self.do_count,
                          "show": self.do_show,
                          "destroy": self.do_destroy,
                          "update": self.do_update,
                          "create": self.do_create}
        parts = line.split(".")

        if len(parts) == 2 and parts[0] in self.__class_names:
            class_name = parts[0]
            rest_parts = parts[1].split("(")

            if len(rest_parts) == 2 and rest_parts[0] in cmd_dict:
                cmd_name = rest_parts[0]
                arg_str = rest_parts[1].rstrip(")")
                args = [class_name] + arg_str.split(",")
                cmd_str = " ".join(args)
                return cmd_dict[cmd_name](cmd_str)
            else:
                print("*** Unknown Syntax {}".format(line))

    def do_greet(self, line) -> None:
        '''Pints a greet message
        '''
        print("Hello")

    def do_quit(self, line) -> bool:
        """Quit: command to exit the program
        """
        return True

    def do_EOF(self, line) -> bool:
        """EOF: end-of-file mark the exits the program
        """
        "Exit"
        return True

    def emptyline(self) -> None:
        """Deals with the empty lines
        """
        pass

    def do_create(self, line) -> None:
        """
        Usage: create <class name>

        Creates a new instance of BaseModel, saves it (JSON file) & prints id

        Ex: $ create BaseModel
        """
        args = line.split()

        if len(args) == 0:
            print("** class name missing **")
        else:
            class_name = args[0]
            if class_name not in self.__class_names:
                print("** class doesn't exist **")
            else:
                try:
                    module_name = self.__class_names[class_name]
                    module = importlib.import_module(module_name)
                    cls = getattr(module, class_name)
                except AttributeError:
                    print("** class doesn't exist **")
                    return

                new_obj = cls()
                new_obj.save()
                print(new_obj.id)

    def do_show(self, line) -> None:
        """
        Usage: show <class name> <id>

        Prints the string rep. of an instance based on the class name & id.

        Ex: $ show BaseModel 1234-1234-1234.
        """
        args = line.split()

        if len(args) == 0:
            print("** class name missing **")
        elif args[0] not in self.__class_names:
            print("** class doesn't exist **")
        elif len(args) == 1:
            print("** instance id missing **")
        else:
            obj_dict = storage.all()
            obj_key = "{}.{}".format(args[0], args[1])
            obj = obj_dict.get(obj_key, None)
            if obj is None:
                print("** no instance found **")
            else:
                print(obj)

    def do_destroy(self, line) -> None:
        """
        Usage: destroy <class name> <id>

        Deletes an instance based on the class name and id
        (save the change into the JSON file).

        Ex: $ destroy BaseModel 1234-1234-1234.
        """
        args = line.split()

        if len(args) == 0:
            print("** class name missing **")
        elif args[0] not in self.__class_names:
            print("** class doesn't exist **")
        elif len(args) == 1:
            print("** instance id missing **")
        else:
            obj_dict = storage.all()
            obj_key = "{}.{}".format(args[0], args[1])
            obj = obj_dict.get(obj_key, None)
            if obj is None:
                print("** no instance found **")
            else:
                obj_dict.pop(obj_key)
                storage.save()
                return

    def do_all(self, line) -> None:
        """
        Usage: all <class name>

        Prints all string rep. of all instances based or not on the class name

        Ex: $ all BaseModel or $ all.
        """
        args = line.split()

        if len(args) > 0 and args[0] not in self.__class_names:
            print("** class doesn't exist **")
        else:
            obj_list = []
            for obj in storage.all().values():
                if len(args) == 0 or obj.__class__.__name__ == args[0]:
                    obj_list.append(obj.__str__())
            if len(obj_list) == 0:
                print("[]")
            else:
                print("[{}]".format(", ".join(obj_list)))

    def do_update(self, line) -> None:
        """
        Usage: update <class name> <id> <attribute name> "<attribute value>"

        Updates an instance based on the class name and id by adding or
        updating attribute(name & value) or using a dictionary representation
        save the change into the JSON file)

        Example:
        $ update BaseModel 1234-1234-1234 email "aibnb@mail.com"
        $ update User 1234-1234-1234-1234 {'first_name': 'John', 'age': 89}
        """
        args = line.split()
        obj_dict = storage.all()

        if len(args) == 0:
            print("** class name missing **")
        elif args[0] not in self.__class_names:
            print("** class doesn't exist **")
        elif len(args) == 1:
            print("** instance id missing **")
        elif "{}.{}".format(args[0], args[1]) not in obj_dict.keys():
            print("** no instance found **")
        elif len(args) == 2:
            print("** attribute name missing **")
        elif len(args) == 3:
            print("** value missing **")
        else:
            obj_key = "{}.{}".format(args[0], args[1])
            obj = obj_dict.get(obj_key, None)
            attr_name = args[2]
            attr_value = args[3]

            try:
                if attr_value.isdigit():
                   attr_value = int(attr_value)
                elif float(attr_value):
                    attr_value = float(attr_value)
            except ValueError:
                pass

            class_attr = type(obj).__dict__
            if attr_name in class_attr.keys():
                try:
                    attr_value = type(class_attr[attr_name])(attr_value)
                except Exception:
                    print(f"** value type error: {attr_name} **")
                    print(f"Got: {type(attr_value)}")
                    print(f"Expected: {type(class_attr[attr_name])}")
                    return

            setattr(obj, attr_name, attr_value)
            storage.save()

    def do_count(self, line):
        """
        Usage: count <class_name>

        Count the number of instances of a given class.

        Ex: $ count BaseModel or $ count User
        """
        args = line.split()

        if len(args) == 0:
            print("** class name missing **")
            return

        count = 0
        for obj in storage.all().values():
            if obj.__class__.__name__ == args[0]:
                count += 1
        print(count)

if __name__ == '__main__':
    HBNBCommand().cmdloop()