# -*- coding: utf-8 -*-
"""
Created on Sat Mar 14 09:44:10 2020
@author: Vitor Vilas-Boas
"""
""" 72 trials per classe * 2 sessions
    T = startTrial=0; cue=2; startMI=3.25; endMI=6; endTrial=7.5-8.5

    Dataset description MNE (Linux) (by vboas): more info in http://bbci.de/competition/iv/desc_2a.pdf
    Meta-info (Training data _T):
     	1=1023 (rejected trial)
     	2=768 (start trial)
     	3=1072 (Unknown/ Eye Moviments)
     	4=769 (Class 1 - LH - cue onset)
     	5=770 (Class 2 - RH - cue onset)
     	6=771 (Class 3 - Foot - cue onset)
     	7=772 (Class 4 - Tongue - cue onset)
     	8=277 (Eye closed) [suj 4 = 32766 (Start a new run)]
     	9=276 (Eye open)   [suj 4 = None ]
     	10=32766 (Start a new run) [suj 4 = None ]
    Meta-info (Test data _E):
     	1=1023 (rejected trial)
     	2=768 (start trial)
     	3=1072 (Unknown/ Eye Moviments)
     	4=783 (Cue unknown/undefined)
     	5=277 (Eye closed)
     	6=276 (Eye open)
     	7=32766 (Start a new run)

    Dataset description MNE (MAC & Windows) (by vboas): more info in http://bbci.de/competition/iv/desc_2a.pdf
    Meta-info (Training data _T):
     	1=1023 (rejected trial)
     	2=1072 (Unknown/ Eye Moviments)
     	3=276 (Eye open)                      [suj 4 = 32766 (Start a new run)]
     	4=277 (Eye closed)                    [suj 4 = 768 (start trial)]
     	5=32766 (Start a new run)             [suj 4 = 769 (Class 1 - LH - cue onset)]
     	6=768 (start trial)                   [suj 4 = 770 (Class 2 - RH - cue onset)]
     	7=769 (Class 1 - LH - cue onset)      [suj 4 = 771 (Class 3 - Foot - cue onset)]
     	8=770 (Class 2 - RH - cue onset)      [suj 4 = 772 (Class 4 - Tongue - cue onset)]
     	9=771 (Class 3 - Foot - cue onset)    [suj 4 = None ]
     	10=772 (Class 4 - Tongue - cue onset) [suj 4 = None ]
    Meta-info (Test data _E):
     	1=1023 (rejected trial)
     	2=1072 (Unknown/ Eye Moviments)
     	3=276 (Eye open)
     	4=277 (Eye closed)
     	5=32766 (Start a new run)
     	6=768 (start trial)
     	7=783 (Cue unknown/undefined)
"""

import os
import mne
import pickle
import warnings
import numpy as np
from scipy.io import loadmat, savemat
from datetime import datetime

warnings.filterwarnings("ignore", category=DeprecationWarning)
mne.set_log_level(50, 50)

path = '/mnt/dados/eeg_data/IV2a/' ## >>> ENTER THE PATH TO THE DATASET HERE

path_out = path + 'npy/'
if not os.path.isdir(path_out): os.makedirs(path_out)

for suj in range(1,10):
    raw = mne.io.read_raw_gdf(path + 'A0' + str(suj) + 'T.gdf').load_data()
    dt = raw.get_data()[:22] # [channels x samples]
    et_raw = mne.events_from_annotations(raw)  # raw.find_edf_events()

    et = np.delete(et_raw[0], 1, axis=1) # remove MNE zero columns
    et = np.delete(et,np.where(et[:,1] == 1), axis=0) # remove rejected trial
    et = np.delete(et,np.where(et[:,1] == 3), axis=0) # remove eye movements/unknown
    et = np.delete(et,np.where(et[:,1] == 8), axis=0) # remove eyes closed
    et = np.delete(et,np.where(et[:,1] == 9), axis=0) # remove eyes open
    et = np.delete(et,np.where(et[:,1] == 10), axis=0) # remove start of a new run/segment
    et[:,1] = np.where(et[:,1] == 2, 0, et[:,1]) # start trial t=0
    et[:,1] = np.where(et[:,1] == 4, 1, et[:,1]) # LH
    et[:,1] = np.where(et[:,1] == 5, 2, et[:,1]) # RH
    et[:,1] = np.where(et[:,1] == 6, 3, et[:,1]) # Foot
    et[:,1] = np.where(et[:,1] == 7, 4, et[:,1]) # Tongue
    for i in range(0, len(et)):
        if et[i,1] == 0: et[i,1] = (et[i+1,1]+10) # labeling start trial [11 a 14] according cue [1,2,3,4]

    raw = mne.io.read_raw_gdf(path + 'A0' + str(suj) + 'E.gdf').load_data()
    trues = np.ravel(loadmat(path + 'true_labels/A0' + str(suj) + 'E.mat')['classlabel'])
    dv = raw.get_data()[:22] # [channels x samples]
    ev_raw = mne.events_from_annotations(raw)  # raw.find_edf_events()
    ev = np.delete(ev_raw[0], 1, axis=1) # remove MNE zero columns
    ev = np.delete(ev,np.where(ev[:,1] == 1), axis=0) # remove rejected trial
    ev = np.delete(ev,np.where(ev[:,1] == 3), axis=0) # remove eye movements/unknown
    ev = np.delete(ev,np.where(ev[:,1] == 5), axis=0) # remove eyes closed
    ev = np.delete(ev,np.where(ev[:,1] == 6), axis=0) # remove eyes open
    ev = np.delete(ev,np.where(ev[:,1] == 7), axis=0) # remove start of a new run/segment
    ev[:,1] = np.where(ev[:,1] == 2, 0, ev[:,1]) # start trial t=0
    ev[np.where(ev[:,1] == 4),1] = trues # change unknown value labels(4) to value in [1,2,3,4]
    for i in range(0, len(ev)):
        if ev[i,1] == 0: ev[i,1] = (ev[i+1,1]+10) # labeling start trial [11 a 14] according cue [1,2,3,4]

    info = {'fs':250, 'class_ids':[1,2,3,4], 'trial_tcue':2.0, 'trial_tpause':6.0, 'trial_mi_time':4.0,
            'trials_per_class':72, 'eeg_channels':dt.shape[0], 'ch_labels':raw.ch_names,
            'datetime':datetime.now().strftime('%d-%m-%Y_%Hh%Mm')}

    # ### to 8 channels m1 grid
    # grid_8ch = [7,8,9,10,11,13,15,17]
    # dt, dv = dt[grid_8ch], dv[grid_8ch]
    # info['ch_labels'] = ['EEG-C3','EEG-C1','EEG-Cz','EEG-C2','EEG-C4','EEG-CP3','EEG-CPz','EEG-CP4']
    # info['eeg_channels'] = len(grid_8ch)

    #%% save npy session files
    np.save(path_out + 'A0' + str(suj) + 'T', [dt,et,info], allow_pickle=True)
    np.save(path_out + 'A0' + str(suj) + 'E', [dv,ev,info], allow_pickle=True)

    #%% prepare agregate file (single npy file with all sessions)
    ev1 = np.copy(ev)
    ev[:,0] += len(dt.T) # ev pos + last dt pos (et is continued by ev)
    events = np.r_[et, ev]
    data = np.c_[dt, dv]
    info['trials_per_class'] = 144

    #%% save npy agregate file
    np.save(path_out + 'A0' + str(suj), [data,events,info], allow_pickle=True)
    # pickle.dump([data,events,info], open(path_out + ' A0' + str(suj) + '.pkl', 'wb'))
    # savemat(path_out + 'A0' + str(suj) + '.mat', mdict={'data':data,'events':events,'info':info})


# -*- coding: utf-8 -*-
#
# A script to check for annular ring violations
# both for TH pads and vias
# requirements: KiCAD pcbnew >= 4.0
# annular.py release "1.5.1"
#
# annular.py checking PCB for Annular Ring in Vias and TH Pads
# (SMD, Connector and NPTH are skipped)
# default Annular Ring >= 0.15 both for TH Pads and Vias
#

#### plugins errors
#import pcbnew;pcbnew.GetWizardsBackTrace()

global mm_ius, DRL_EXTRA, AR_SET, AR_SET_V, DRL_EXTRA_ius, MIN_AR_SIZE, MIN_AR_SIZE_V, found_violations, LogMsg, ___version___

___version___="1.7.2"

#wx.LogMessage("My message")
mm_ius = 1000000.0
# (consider always drill +0.1)
DRL_EXTRA=0.1
DRL_EXTRA_ius=DRL_EXTRA * mm_ius

AR_SET = 0.125   #minimum annular accepted for pads
MIN_AR_SIZE = AR_SET * mm_ius

AR_SET_V = 0.125  #minimum annular accepted for vias
MIN_AR_SIZE_V = AR_SET_V * mm_ius

import sys
import wx
import wx.richtext
#import subprocess
import os
import pcbnew
from pcbnew import *
# import base64
# from wx.lib.embeddedimage import PyEmbeddedImage
import time

from . import AnnularDlg
from . import AnnularResultDlg

sys.path.append(os.path.dirname(__file__))

debug = False
def wxLogDebug(msg,dbg):
    """printing messages only if show is omitted or True"""
    if dbg == True:
        wx.LogMessage(msg)
#
def find_pcbnew_w():
    windows = wx.GetTopLevelWindows()
    pcbneww = [w for w in windows if "pcbnew" in w.GetTitle().lower()]
    if len(pcbneww) != 1:
        return None
    return pcbneww[0]
#

class AnnularResult_Dlg(AnnularResultDlg.AnnularResultDlg):
    # from https://github.com/MitjaNemec/Kicad_action_plugins
    # hack for new wxFormBuilder generating code incompatible with old wxPython
    # noinspection PyMethodOverriding
    def SetSizeHints(self, sz1, sz2):
        if wx.__version__ < '4.0':
            self.SetSizeHintsSz(sz1, sz2)
        else:
            super(AnnularResult_Dlg, self).SetSizeHints(sz1, sz2)

    def onOK(self, event):
        self.Destroy()
        #return self.EndModal(wx.ID_OK) # if modal_result == wx.ID_OK:

    def OnClickCopy(self, event):
        self.m_richTextResult.SelectAll()
        self.m_richTextResult.Copy()
        #global LogMsg
        #copy2clip(LogMsg)
        self.copy_btn.SetLabel("Text Copied")

    # def onDeleteClick(self, event):
    #     return self.EndModal(wx.ID_DELETE)
    #
    # def onConnectClick(self, event):
    #     return self.EndModal(wx.ID_REVERT)

    def __init__(self,  parent):
        import wx
        AnnularResultDlg.AnnularResultDlg.__init__(self, parent)
        #self.GetSizer().Fit(self)
        self.SetMinSize(self.GetSize())
                #### ----- connections
        # Connect Events
        self.Bind(wx.EVT_BUTTON, self.onOK, self.ok_btn)
        #self.ok_btn.Bind(wx.EVT_BUTTON, self.EndModal(wx.ID_OK))
        self.Bind(wx.EVT_BUTTON, self.OnClickCopy, self.copy_btn)
        self.ok_btn.SetFocus()
        # Tooltips
        self.copy_btn.SetToolTip( wx.ToolTip(u"Copy Text to Clipboard" ))
        self.ok_btn.SetToolTip( wx.ToolTip(u"Exit" ))

        #def onOK(self, event):
        #    return self.EndModal(wx.ID_OK) # if modal_result == wx.ID_OK:

        #def onConnectClick(self, event):
        #    return self.EndModal(wx.ID_REVERT)

        #self.m_buttonDelete.Bind(wx.EVT_BUTTON, self.onDeleteClick)
        #self.m_buttonReconnect.Bind(wx.EVT_BUTTON, self.onConnectClick)
        #if wx.__version__ < '4.0':
        #    self.m_buttonReconnect.SetToolTipString( u"Select two converging Tracks to re-connect them\nor Select tracks including one round corner to be straighten" )
        #    self.m_buttonRound.SetToolTipString( u"Select two connected Tracks to round the corner\nThen choose distance from intersection and the number of segments" )
        #else:
        #    self.m_buttonReconnect.SetToolTip( u"Select two converging Tracks to re-connect them\nor Select tracks including one round corner to be straighten" )
        #    self.m_buttonRound.SetToolTip( u"Select two connected Tracks to round the corner\nThen choose distance from intersection and the number of segments" )

class Annular_Dlg(AnnularDlg.AnnularDlg):
    # from https://github.com/MitjaNemec/Kicad_action_plugins
    # hack for new wxFormBuilder generating code incompatible with old wxPython
    # noinspection PyMethodOverriding
    def SetSizeHints(self, sz1, sz2):
        if wx.__version__ < '4.0':
            self.SetSizeHintsSz(sz1, sz2)
        else:
            super(Annular_Dlg, self).SetSizeHints(sz1, sz2)

    def __init__(self,  parent):
        import wx
        AnnularDlg.AnnularDlg.__init__(self, parent)
        #self.GetSizer().Fit(self)
        self.SetMinSize(self.GetSize())
        #c1.Bind(wx.EVT_CHECKBOX, self.OntextMetric, c1)
        #self.m_checkBoxPHD.Bind(wx.EVT_CHECKBOX, self.OnClickCheck, self.m_checkBoxPHD)
        self.m_checkBoxPHD.Bind(wx.EVT_CHECKBOX, self.OnClickCheck)
        self.m_bitmapAR.SetBitmap(wx.Bitmap(os.path.join(os.path.dirname(__file__), "./annular-help.png")))

        #self.Bind(wx.EVT_CHECKBOX, self.OnClickCheck)
        #self.m_buttonDelete.Bind(wx.EVT_BUTTON, self.onDeleteClick)
        #self.m_buttonReconnect.Bind(wx.EVT_BUTTON, self.onConnectClick)
        #if wx.__version__ < '4.0':
        #    self.m_buttonReconnect.SetToolTipString( u"Select two converging Tracks to re-connect them\nor Select tracks including one round corner to be straighten" )
        #    self.m_buttonRound.SetToolTipString( u"Select two connected Tracks to round the corner\nThen choose distance from intersection and the number of segments" )
        #else:
        #    self.m_buttonReconnect.SetToolTip( u"Select two converging Tracks to re-connect them\nor Select tracks including one round corner to be straighten" )
        #    self.m_buttonRound.SetToolTip( u"Select two connected Tracks to round the corner\nThen choose distance from intersection and the number of segments" )
    def OnClickCheck(self, event):
        #self.Destroy()
        if self.m_checkBoxPHD.IsChecked():
            #self.Destroy()
            self.m_staticTextPHD.Enable()
            self.m_textCtrlPHD.Enable()
        else:
            self.m_staticTextPHD.Disable()
            self.m_textCtrlPHD.Disable()

    # def onDeleteClick(self, event):
    #     return self.EndModal(wx.ID_DELETE)
    #
    # def onConnectClick(self, event):
    #     return self.EndModal(wx.ID_REVERT)

# Python plugin stuff
class annular_check( pcbnew.ActionPlugin ):
    """
    A script to check for annular ring violations
    both for TH pads and vias
    requirements: KiCAD pcbnew >= 4.0
        AR_SET    minimum annular accepted for pads
        AR_SET_V  minimum annular accepted for vias
    """
    global ___version___
    def defaults( self ):
        """
        Method defaults must be redefined
        self.name should be the menu label to use
        self.category should be the category (not yet used)
        self.description should be a comprehensive description
          of the plugin
        """
        self.name = "Annular checker \nversion "+___version___
        self.category = "Checking PCB"
        self.description = "Automaticaly check annular on an existing PCB"
        #self.pcbnew_icon_support = hasattr(self, "show_toolbar_button")
        self.show_toolbar_button = True
        self.icon_file_name = os.path.join(os.path.dirname(__file__), 'annular.png')

    def Run( self ):
        import sys,os
        #mm_ius = 1000000.0
        # _pcbnew_frame = [x for x in wx.GetTopLevelWindows() if x.GetTitle().lower().startswith('pcbnew')][0]

        #while True:
        #    time.sleep(1)
        #    pcbnew_window = find_pcbnew_window()
        #    if not pcbnew_window:
        #        continue
        pcbnew_window = find_pcbnew_w()
        #aParameters = RoundTrackDlg(None)
        aParameters = Annular_Dlg(pcbnew_window) # _pcbnew_frame)
        aParameters.m_LabelTitle.SetLabel("version:  "+___version___)
        aParameters.m_textCtrlARP.SetToolTip( wx.ToolTip(u"Annular Ring for Pads (mm)" ))
        aParameters.m_staticTextPHD.SetToolTip( wx.ToolTip(u"Drill extra margin (mm)" ))
        aParameters.m_textCtrlARV.SetToolTip( wx.ToolTip(u"Annular Ring for Vias (mm)" ))
        aParameters.m_staticTextARV.SetToolTip( wx.ToolTip(u"Annular Ring for Vias (mm)" ))
        aParameters.m_textCtrlPHD.SetToolTip( wx.ToolTip(u"Drill extra margin (mm)" ))
        aParameters.m_staticTextARP.SetToolTip( wx.ToolTip(u"Annular Ring for Pads (mm)" ))
        aParameters.m_checkBoxPHD.SetToolTip( wx.ToolTip(u"use drill size as finished hole size\nadding an extra drill margin" ))
        aParameters.m_textCtrlPHD.SetValue('0.1')
        aParameters.m_textCtrlARP.SetValue('0.125')
        aParameters.m_textCtrlARV.SetValue('0.125')
        aParameters.Show()
        modal_result = aParameters.ShowModal()
        #import pcbnew;pcbnew.GetWizardsBackTrace()
        if modal_result == wx.ID_OK:
            global mm_ius, DRL_EXTRA, AR_SET, AR_SET_V, DRL_EXTRA_ius, MIN_AR_SIZE, MIN_AR_SIZE_V
            phd = float(aParameters.m_textCtrlPHD.GetValue().replace(',','.'))
            ar  = float(aParameters.m_textCtrlARP.GetValue().replace(',','.'))
            arv = float(aParameters.m_textCtrlARV.GetValue().replace(',','.'))
            if aParameters.m_checkBoxPHD.IsChecked():
                DRL_EXTRA=phd
                DRL_EXTRA_ius=DRL_EXTRA * mm_ius
            else:
                DRL_EXTRA=0
                DRL_EXTRA_ius=DRL_EXTRA * mm_ius
            AR_SET = ar   #minimum annular accepted for pads
            MIN_AR_SIZE = AR_SET * mm_ius

            AR_SET_V = arv  #minimum annular accepted for vias
            MIN_AR_SIZE_V = AR_SET_V * mm_ius
            #snap2grid(gridSizeMM,use_grid_origin)
            calculate_AR()
        else:
            None  # Cancel

def annring_size(pad):
    # valid for oval pad/drills
    annrX=(pad.GetSize()[0] - (pad.GetDrillSize()[0]+DRL_EXTRA_ius))/2
    annrY=(pad.GetSize()[1] - (pad.GetDrillSize()[1]+DRL_EXTRA_ius))/2
    #annr=min(pad.GetSize()) - max(pad.GetDrillSize())
    #if annr < MIN_AR_SIZE:
    #print annrX
    #print annrY
    #print pad.GetSize()[0]/mm_ius
    #print pad.GetSize()[0]#/mm_ius
    #print pad.GetDrillSize()[0]#/mm_ius
    #print DRL_EXTRA_ius
    #print pad.GetDrillSize()[0]/mm_ius
    #print (pad.GetDrillSize()[0]+DRL_EXTRA_ius)/mm_ius
    #print annrX/mm_ius
    return min(annrX,annrY)

def annringNP_size(pad):
    # valid for oval pad/drills
    annrX=(pad.GetSize()[0] - (pad.GetDrillSize()[0]))/2
    annrY=(pad.GetSize()[1] - (pad.GetDrillSize()[1]))/2
    #annr=min(pad.GetSize()) - max(pad.GetDrillSize())
    #if annr < MIN_AR_SIZE:
    #print annrX
    #print annrY
    #print pad.GetSize()[0]/mm_ius
    #print pad.GetSize()[0]#/mm_ius
    #print pad.GetDrillSize()[0]#/mm_ius
    #print DRL_EXTRA_ius
    #print pad.GetDrillSize()[0]/mm_ius
    #print (pad.GetDrillSize()[0]+DRL_EXTRA_ius)/mm_ius
    #print annrX/mm_ius
    #return min(annrX,annrY)
    return annrX,annrY

def vias_annring_size(via):
    # calculating via annular
    annr=(via.GetWidth() - (via.GetDrillValue()+DRL_EXTRA_ius))/2
    #print via.GetWidth()
    #print via.GetDrillValue()
    return annr

def f_mm(raw):
    return repr(raw/mm_ius)

def calculate_AR():
    global mm_ius, DRL_EXTRA, AR_SET, AR_SET_V, DRL_EXTRA_ius, MIN_AR_SIZE, MIN_AR_SIZE_V
    board = pcbnew.GetBoard()
    PassC=FailC=0
    PassCV=FailCV=0

    PassCN=FailCN=0
    PassCVN=FailCVN=0

    fileName = GetBoard().GetFileName()
    dirpath = os.path.abspath(os.path.expanduser(fileName))
    path, fname = os.path.split(dirpath)
    ext = os.path.splitext(os.path.basename(fileName))[1]
    name = os.path.splitext(os.path.basename(fileName))[0]
    #wx.LogMessage(dir)
    #lsep=os.linesep
    lsep='\n'
    proj_path = os.path.dirname(os.path.abspath(fileName))
    out_filename_AR_checking=proj_path+os.sep+name+"_AR-Check.txt"

    if len(fileName)==0:
        wx.LogMessage("a board needs to be saved/loaded!")
    else:
        found_violations=False
        pcbnew_window = find_pcbnew_w()
        # _pcbnew_frame = [x for x in wx.GetTopLevelWindows() if x.GetTitle().lower().startswith('pcbnew')][0]
        aResult = AnnularResult_Dlg(pcbnew_window)
        #import pcbnew;pcbnew.GetWizardsBackTrace()
        writeTxt= aResult.m_richTextResult.WriteText
        rt = aResult.m_richTextResult
        rt.BeginItalic()
        writeTxt("AR checking written on:\n'"+out_filename_AR_checking+"'\n")
        writeTxt("'action_menu_annular_check.py'\n")
        #frame.m_richText1.WriteText("'action_menu_annular_check.py'\n")
        LogMsg=""
        msg="AR checking written on:\n'"+out_filename_AR_checking+"'\n"
        msg+="'action_menu_annular_check.py'\n"
        msg+="version = "+___version___
        writeTxt("version = "+___version___)
        msg+="\nTesting PCB for Annular Rings\nTH Pads >= "+repr(AR_SET)+" Vias >= "+repr(AR_SET_V)+"\nPHD margin on PTH = "+ repr(DRL_EXTRA)
        writeTxt("\nTesting PCB for Annular Rings\nTH Pads >= "+repr(AR_SET)+" Vias >= "+repr(AR_SET_V)+"\nPHD margin on PTH = "+ repr(DRL_EXTRA))
        rt.EndItalic()
        writeTxt('\n\n')
        #print (msg)
        LogMsg+=msg+'\n\n'

        # print "LISTING VIAS:"
        for item in board.GetTracks():
            if  hasattr(pcbnew,'VIA'):
                via = pcbnew.VIA
                testing = type(item)
            else:
                via = pcbnew.PCB_VIA  #'PCB_VIA'
                item = pcbnew.Cast_to_PCB_VIA(item)
                testing = type(item) #item.GetClass()

            #writeTxt(str(testing))
            if testing is via: # or via in testing:
                #writeTxt(str(testing)+"here")
                pos = item.GetPosition()
                drill = item.GetDrillValue()
                width = item.GetWidth()
                ARv = vias_annring_size(item)
                if ARv  < MIN_AR_SIZE_V:
                    #            print("AR violation at %s." % (pad.GetPosition() / mm_ius ))  Raw units, needs fixing
                    XYpair =  item.GetPosition()
                    msg="AR Via violation of "+f_mm(ARv)+" at XY "+f_mm(XYpair[0])+","+f_mm(XYpair[1])
                    rt.BeginTextColour('red')
                    writeTxt("AR Via violation of "+f_mm(ARv)+" at XY "+f_mm(XYpair[0])+","+f_mm(XYpair[1])+'\n')
                    rt.EndTextColour()
                    #print (msg)
                    LogMsg+=msg+'\n'
                    FailCV = FailCV+1
                else:
                    PassCV = PassCV+1
            #print type(item)

        msg="VIAS that Pass = "+repr(PassCV)+"; Fails = "+repr(FailCV)
        if FailCV >0:
            rt.BeginBold()
        writeTxt("VIAS that Pass = "+repr(PassCV)+"; ")
        if FailCV >0:
            rt.BeginTextColour('red')
        writeTxt("Fails = "+repr(FailCV)+'\n\n')
        if FailCV >0:
            rt.EndTextColour()
            rt.EndBold()
        print(msg)
        LogMsg+=msg+'\n'

        if  hasattr(board,'GetModules'):
            footprints = board.GetModules()
            PAD_ATTR_STANDARD = PAD_ATTRIB_STANDARD
            PAD_ATTR_HOLE_NOT_PLATED = PAD_ATTRIB_HOLE_NOT_PLATED
        else:
            footprints = board.GetFootprints()
            PAD_ATTR_STANDARD = PAD_ATTRIB_PTH # HOLE_ATTRIBUTE_HOLE_PAD
            PAD_ATTR_HOLE_NOT_PLATED = HOLE_ATTRIBUTE_HOLE_MECHANICAL
        for module in footprints:
            try:
                module_Pads=module.PadsList()
            except:
                module_Pads=module.Pads()
            for pad in module_Pads:                    #print(pad.GetAttribute())
                if pad.GetAttribute() == PAD_ATTR_STANDARD: #TH pad
                    ARv = annring_size(pad)
                    #print(f_mm(ARv))
                    if ARv  < MIN_AR_SIZE:
        #                print("AR violation at %s." % (pad.GetPosition() / mm_ius ))  Raw units, needs fixing
                        XYpair =  pad.GetPosition()
                        msg="AR PTH violation of "+f_mm(ARv)+" at XY "+f_mm(XYpair[0])+","+f_mm(XYpair[1])
                        rt.BeginTextColour('red')
                        writeTxt("AR PTH violation of "+f_mm(ARv)+" at XY "+f_mm(XYpair[0])+","+f_mm(XYpair[1])+'\n')
                        rt.EndTextColour()
                        #print (msg)
                        LogMsg+=msg+'\n'
                        FailC = FailC+1
                    else:
                        PassC = PassC+1
                if pad.GetAttribute() == PAD_ATTR_HOLE_NOT_PLATED:
                    ARvX, ARvY = annringNP_size(pad)
                    #print(f_mm(ARvX));print(f_mm(ARvY))
                    if (ARvX) != 0 or ARvY != 0:
                        ARv = min(ARvX, ARvY)
                        if ARv < MIN_AR_SIZE:
        #                    print("AR violation at %s." % (pad.GetPosition() / mm_ius ))  Raw units, needs fixing
                            XYpair =  pad.GetPosition()
                            msg="AR NPTH warning of "+f_mm(ARv)+" at XY "+f_mm(XYpair[0])+","+f_mm(XYpair[1])
                            rt.BeginTextColour('red')
                            writeTxt("AR NPTH warning of "+f_mm(ARv)+" at XY "+f_mm(XYpair[0])+","+f_mm(XYpair[1])+'\n')
                            rt.EndTextColour()
                            #print (msg)
                            LogMsg+=msg+'\n'
                            FailCN = FailCN+1
                        else:
                            PassCN = PassCN+1
                    else:
                        PassCN = PassCN+1

        #if FailCV >0:
        #writeTxt('\n')
        msg = "TH PADS that Pass = "+repr(PassC)+"; Fails = "+repr(FailC)
        if FailC >0:
            rt.BeginBold()
        writeTxt("TH PADS that Pass = "+repr(PassC)+"; ")
        if FailC >0:
            rt.BeginTextColour('red')
        writeTxt("Fails = "+repr(FailC)+'\n')
        if FailC >0:
            rt.EndTextColour()
            rt.EndBold()
        print(msg)
        LogMsg+=msg+'\n'

        msg="NPTH PADS that Pass = "+repr(PassCN)+"; Fails = "+repr(FailCN)
        #writeTxt('\n')
        if FailCN >0:
            rt.BeginBold()
        writeTxt("NPTH PADS that Pass = "+repr(PassCN)+"; ")
        if FailCN >0:
            rt.BeginTextColour('red')
        writeTxt("Fails = "+repr(FailCN)+'\n')
        if FailC >0:
            rt.EndTextColour()
            rt.EndBold()
        print(msg)
        LogMsg+=msg+'\n'

        pcbName = (os.path.splitext(GetBoard().GetFileName())[0]) #filename no ext
        #wx.LogMessage(pcbName)#LogMsg)
        ##wx.LogMessage(LogMsg)
        FC=r"C:\FreeCAD\bin\freecad.exe"
        kSU=r"C:\Cad\Progetti_K\3D-FreeCad-tools\kicad-StepUp-tools.FCMacro"
        #subprocess.check_call([FC, kSU, pcbName])
        ##p = subprocess.Popen([FC, kSU, pcbName])

        #found_violations=False
        if (FailC+FailCN+FailCV)>0:
            found_violations=True

        if found_violations:
            #frame.m_staticTitle = wx.StaticText(frame, label=" Check result: (Violations found)")
            aResult.m_staticTitle.SetLabel(" Check result: (Violations found)")
            #self.title.SetForegroundColour('#FF0000')
            aResult.m_staticTitle.SetBackgroundColour('#FF0000')
            font = wx.Font(wx.DEFAULT, wx.DECORATIVE, wx.ITALIC, wx.BOLD)
            aResult.m_staticTitle.SetFont(font)
        else:
            #frame.m_staticTitle = wx.StaticText(frame, label=" Annular Check result: OK")
            aResult.m_staticTitle.SetLabel(" Annular Check result: OK")
            aResult.m_staticTitle.SetBackgroundColour('#00FF00')
            font = wx.Font(wx.DEFAULT, wx.DECORATIVE, wx.ITALIC, wx.BOLD)
            aResult.m_staticTitle.SetFont(font)

        with open(out_filename_AR_checking,'w') as f_out:
            f_out.write(LogMsg)

        aResult.Show()
        #modal_result = aResult.ShowModal()
        #if modal_result == wx.ID_OK:
        #    aResult.Destroy()
        #if modal_result == wx.ID_OK:
        #    aResult.Destroy()

# annular_check().register()
if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(
            description='KiCad PCB Annular Checker')
    parser.add_argument('file', type=str, help="KiCad PCB file")
    args = parser.parse_args()
    print("Loading %s" % args.file)
    main(pcbnew.LoadBoard(args.file))

else:
    annular_check().register()

#  execfile("annular.py")
# annular.py Testing PCB for Annular Ring >= 0.15
# AR violation of 0.1 at XY 172.974,110.744
# VIAS that Pass = 100 Fails = 1
# AR violation of 0.1 at XY 172.212,110.744
# AR violation of 0.0 at XY 154.813,96.52
# PADS that Pass = 49 Fails = 2


# dict = {}
# keys = dict.keys()
# val = dict.values()
# pairs = dict.items()

input_file = open('input.txt', 'r')
lines = input_file.readlines()
pure_lines = []
for i in range(len(lines)):
    pure_lines+=lines[i].split()

d = {}
for i in range(len(pure_lines)):
    word = pure_lines[i]
    if word not in d:
        d[word] = 0

    print(d[word], end=" ")
    d[word]+=1


import numpy as np
from scipy.fft import fft, ifft

BDF1 = 0
BDF2 = 1
TR = 2
TTR = 3

def dlt_lms(z,lms):
    if lms == BDF1:
        return 1-z
    elif lms == BDF2:
        return 1-z+(1/2)*(1-z)**2
    elif lms == TR:
        return 2*(1-z)/(1+z)
    elif lms == TTR:
        cs = [0.893817850529318,   0.684154908023834,   0.629642997466429]
        return 1-z+(1/2)*(1-z)**2+(1/4)*cs[0]*(1-z)**3+(1/8)*cs[1]*(1-z)**4+(1/16)*cs[2]*(1-z)**5
    else:
        return "No such LMS method"

def evalCQ(g, Ks, N, T, lms = TR):
    dt = T/N
    lam = 10**(-14/(2*N+1))
    lams = lam**np.arange(N+1)
    ts = np.linspace(0,T,N+1)
    zs = lam*np.exp(-2*np.pi*1j*np.arange(N+1)/(N+1))
    u = (1/lams)*ifft(Ks(dlt_lms(zs,lms)/dt)*fft(lams*g(ts)))

    return u


import numpy as np
import glob
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as qda
from scipy.signal import savgol_filter
import random

class_names = [0,8,16,24,72,80,88,96]
feature_count=16 #no of features(must be a factor of 96)
X =X_test= np.empty((0,feature_count))
y=y_test=np.empty(0,int)

for class_name in class_names:
	x_k=np.empty((0,feature_count),int)
	y_k=np.empty(0,int)
	# print('class_name:{}'.format(class_name))
	for filename in glob.glob('/home/sreekar/Work/swarm-robotics-project/swarm-rescue/rssi data/data/intervals of 8 pwm data/{0:d}data.txt'.format(class_name)):
		# print('opening..{}'.format(filename))
		with open(filename) as datafile:
			# print(filename)
			data=np.loadtxt(datafile)
			# print('data:\n{}\nshape:{}'.format(data[:,1],data[:,1].shape))
			while data.shape[0]>=96:
				raw_data = data[:96,1]#choosing first sequence of values(right to left data)
				scaled_data = (raw_data-np.mean(raw_data))/np.std(raw_data) #feature scaling to limit the magnitudes
				smooth_tuple = savgol_filter(scaled_data, 11, 3) # window size 11, polynomial order 3
				reduced_feature_tuple = np.mean(smooth_tuple.reshape(feature_count,int(smooth_tuple.size/feature_count)),axis=1)
				x_k = np.append(x_k, np.array(reduced_feature_tuple,ndmin=2),axis=0)	#taking right to left data for now
				data = data[96:,:]
				y_k=np.append(y_k,np.array(class_name))

	train_count=22
	select_index=random.sample(range(x_k.shape[0]),train_count)#randomly select 20 indices
	# print(select_index)
	X=np.append(X,x_k[select_index,:],axis=0)
	y=np.append(y,y_k[select_index],axis=0)
	test_index=list(set(range(x_k.shape[0])).difference(set(select_index)))#selecting the complement indices
	X_test=np.append(X_test,x_k[test_index,:],axis=0)
	y_test=np.append(y_test,y_k[test_index],axis=0)

print('X:\n{}\nshape:{}'.format(X,X.shape))
print('y:\n{}\nshape:{}'.format(y,y.shape))

print('X_test:\n{}\nshape:{}'.format(X_test,X_test.shape))
print('y_test:\n{}\nshape:{}'.format(y_test,y_test.shape))

clf=qda()

clf.fit(X,y)

#on TRAINING data
train_prediction=clf.predict(X)
train_score = clf.score(X,y)

print('prediction on training set:\n{}'.format(train_prediction))
train_prediction = np.expand_dims(train_prediction,axis=1)
y= np.expand_dims(y,axis=1)
print('prediction - truth  array for TEST data: \n{}'.format(np.hstack((train_prediction,y))))
print('score on training set: {}'.format(train_score))

#on TEST data
test_prediction=clf.predict(X_test)
test_score=clf.score(X_test,y_test)

print('prediction on test set:\n{}'.format(test_prediction))
test_prediction = np.expand_dims(test_prediction,axis=1)
y_test= np.expand_dims(y_test,axis=1)
print('prediction - truth  array for TEST data: \n{}'.format(np.hstack((test_prediction,y_test))))
print('score on test set: {}'.format(test_score))


from lxml import html
import requests
import csv
import os

class FreeEbookCrawler:
    categories = []

    def __init__(self, depth):
        self.depth = depth
        self.home_page = 'https://www.free-ebooks.net'
        self.csv_file_path = os.path.join(
            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
            'output',
            'full_books.csv'
        )
        self.writer = None

    def read_categories_from_csv_file(self):
        category_file_path = os.path.join(
            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
            'output',
            'categories_pages.csv'
        )

        with open(category_file_path) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',')
            for row in csv_reader:
                self.categories.append([row[0], row[1]])
        print(self.categories)
        print(len(self.categories))
        self.crawl()

    def crawl(self):
        count = 1
        csv_titles = []
        csv_titles.append([
                        'book_links',
                        'book_images',
                        'book_sub_categories',
                        'book_ids',
                        'book_titles',
                        'book_authors',
                        'book_public_dates',
                        'book_ratings',
                        'book_rated_times',
                        'book_downloads',
                        'book_pages',
                        'book_desciptions'
                        ])
        with open(self.csv_file_path, 'a+') as csvFile:
            print("open file")
            self.writer = csv.writer(csvFile)
            self.writer.writerows(csv_titles)
            for category in self.categories:
                category_url = category[0]
                page_number = int(category[1])
                print("starting crawling data from ", category_url, ' ', page_number, '     ', count)
                self.crawl_book_from_category(category_url, page_number)
                count += 1
            print("write done")
        csvFile.close()
        return

    def crawl_book_from_category(self, category_url, page_number):
        if page_number <= 0:
            print('page <= 0')
            return
        num = 1
        while num <= page_number:
            url = f'{category_url}/{num}'
            self.get_book_detail_from_single_page(url)
            num += 1

    def get_book_detail_from_single_page(self, url):
        print(url)
        start_page = requests.get(url)
        tree = html.fromstring(start_page.text)

        book_links = tree.xpath('//div[@class="row laText"]/div/a[@class="img"]/@href')
        book_ids = tree.xpath('//div[@class="row laText"]/@data-id')
        book_titles = tree.xpath('//div[@class="row laText"]/div/h3/a[@class="title"]/text()')
        book_descriptions = tree.xpath('//p[@class="book-description"]/text()')
        ratings = (tree.xpath('//div[@class="col-sm-12 padIt"]/span/@title'))
        rated_times = (tree.xpath('//div[@class="col-sm-12 padIt"]/b/text()'))
        author_category = tree.xpath('//div[@class="col-sm-12 padIt"]/a/text()')
        date_download_page = tree.xpath('//div[@class="col-sm-12 hidden-xs padIt"]/b/text()')
        book_images = tree.xpath('//a[@class="img"]/@href')

        book_ratings = []
        book_rated_times = []
        book_authors = []
        book_sub_categories = []
        book_public_dates = []
        book_downloads = []
        book_pages = []

        for index, item in enumerate(ratings):
            if index % 5 == 0:
                book_ratings.append(item)
                continue

        for index, item in enumerate(rated_times):
            if index % 2 == 0:
                book_rated_times.append(item)
                continue

        for index, item in enumerate(author_category):
            if index % 2 == 0:
                book_authors.append(item)
                continue

            book_sub_categories.append(item)

        for index, item in enumerate(date_download_page):
            num = index % 3
            if num == 0:
                book_public_dates.append(item)
                continue

            if num == 1:
                book_downloads.append(item)
                continue

            book_pages.append(item)

        print(len(book_links), len(book_ids), len(book_titles), len(book_descriptions),
                 len(book_ratings), len(book_rated_times), len(book_authors), len(book_sub_categories), len(book_public_dates),
              len(book_downloads), len(book_pages), len(book_images))

        if( len(book_links)<= 0):
            print("no data, return")
            return

        if (len(book_links) == len(book_ids) == len(book_titles) == len(book_descriptions)
                == len(book_ratings) == len(book_rated_times) == len(book_authors)
                == len(book_sub_categories) == len(book_public_dates) == len(book_downloads) == len(book_pages) == len(book_images)):
            print('data available')
            csv_data = []
            i = 0
            while i < len(book_links):
                csv_data.append([self.home_page+book_links[i], self.home_page+book_images[i], book_sub_categories[i],
                                 book_ids[i], book_titles[i], book_authors[i],  book_public_dates[i], book_ratings[i],
                                 book_rated_times[i], book_downloads[i], book_pages[i], book_descriptions[i]
                                 ])

                i += 1
                print(f'{i} books was appended')
            print(csv_data)
            self.writer.writerows(csv_data)
            print(f'{len(csv_data)} books was written')
            return
        print(f'can not crawl data from {url}')

crawler = FreeEbookCrawler(0)
crawler.read_categories_from_csv_file()


#! /usr/bin/env python

import os, sys

# Script to search for cds headers and create a file of matching gene sequences from different species

#File containing headers for a specific gene
gene = sys.argv[1]

search_ids = []
with open(gene, 'r') as f:
    for line in f:
        search_ids.append(line.strip())

output_dir = "./output/"  # Output directory to save results
if not os.path.exists(output_dir):
    os.mkdir(output_dir)

# Loop through all files in the current directory and subdirectories
for root, dirs, files in os.walk("."):
    for file in files:
        if file.endswith(".fasta"):
            filepath = os.path.join(root, file)
            with open(filepath, "r") as f:
                # Search through the file for matching IDs
                header = ""
                sequence = ""
                for line in f:
                    if line.startswith(">"):
                        # Check if the header contains a matching ID
                        if any(search_id in line for search_id in search_ids):
                            header = line.strip()
                    else:
                        # Append sequence lines to the current sequence
                        sequence += line.strip()

                # If a matching sequence was found, write it to the output file
                if header and sequence:
                    output_file = os.path.join(output_dir, file)
                    with open(output_file, "a") as out:
                        out.write(header + "\n" + sequence + "\n")


import json
from flask import Flask
from flask import request
import sqlite3

# Setting the connection within a global variable
conn = sqlite3.connect('notable.db')

# Creating dummy data within SQLLite DB for interaction through APIs
def setup():
    conn.execute('''drop table APPOINTMENTS''')
    conn.execute('''CREATE TABLE APPOINTMENTS
     (PHYSICIAN TEXT,
     TIME TEXT,
     DATE TEXT,
     TYPE TEXT,
     PATIENT TEXT);''')

    conn.execute("DELETE FROM APPOINTMENTS;")

    conn.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
  VALUES ('DR.SMITH', '08:15 AM', '2022-01-05', 'FOLLOW-UP', 'HARRY STYLES');")

    conn.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
          VALUES ('DR.BRETT', '10:15 AM', '2022-06-02', 'NEW', 'ROBERTO CARLOS');")

    conn.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
          VALUES ('DR.MATHUR', '12:15 PM', '2022-07-15', 'FOLLOW-UP', 'TOM HANKS');")

    conn.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
          VALUES ('DR.JOSH', '1:15 AM', '2021-12-15', 'NEW', 'OLIVIA NELSON');")

    conn.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
          VALUES ('DR.PALANKAR', '11:15 AM', '2021-05-15', 'FOLLOW-UP', 'JOHN DENVER');")

    conn.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
          VALUES ('DR.PALANKAR', '11:15 AM', '2021-05-15', 'NEW', 'JOHN OLIVER');")

    conn.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
          VALUES ('DR.PALANKAR', '11:15 AM', '2021-05-15', 'FOLLOW-UP', 'OLGA RICHARDSON');")

    conn.commit()
    print('Records created successfully')

app = Flask(__name__)
setup()

# Get all appointments
@app.route('/getAllAppointments/', methods=['GET'])
def getAllAppointments():
    conn_ = sqlite3.connect('notable.db')
    doctor = request.args.get('doctor_name')
    date = request.args.get('date')
    records = conn_.execute("SELECT * FROM APPOINTMENTS where PHYSICIAN = '" + doctor + "' and DATE = '" + date + "'")
    field_name_1 = "Doctor\'s Name:"
    field_name_2 = "Appointment Time:"
    field_name_3 = "Date:"
    field_name_4 = "Type:"
    field_name_5 = "Patient\'s Name:"
    out = {}
    res = ''
    for row in records:
        out[field_name_1] = row[0]
        out[field_name_2] = row[1]
        out[field_name_3] = row[2]
        out[field_name_4] = row[3]
        out[field_name_5] = row[4]

        res += ''.join(str(json.dumps(out, indent=4)))
    conn_.close()
    return res

# Create newer appointment
@app.route('/createAppointment/', methods=['POST'])
def createAppointment():
    conn_ = sqlite3.connect('notable.db')
    doctor = request.args.get('doctor_name')
    date = request.args.get('date')
    time = request.args.get('time')

    minutes = int(time.split(' ')[0].split(':')[1]) % 15

    # Validation for valid appointments
    if minutes != 0:
        return "Invalid appointment time selected. Please choose 15min intervals example : 8:15 AM "

    patient = request.args.get('patient')
    records = conn_.execute("SELECT count(*) FROM APPOINTMENTS where PHYSICIAN = '" + doctor + "' and DATE = '" + date + "' and TIME = '" + time + "'")
    c = 0

    # Check if doctor is available, max 3 appointments
    for p in records:
        if p[0] == 3:
            conn_.close()
            return "No appointments are available at the given time"

    # Check if patient already has an appointment during the same time
    records = conn_.execute("SELECT count(*) FROM APPOINTMENTS where PHYSICIAN = '" + doctor + "' and DATE = '" + date + "' and TIME = '" + time + "' and PATIENT = '" + patient + "'")
    for p in records:
        if p[0] >= 1:
            conn_.close()
            return "Patients appointment is already scheduled"

    # Check if existing patient
    patient_list = conn_.execute("SELECT DISTINCT PATIENT FROM APPOINTMENTS where PHYSICIAN = '" + doctor + "'")

    msg = ''
    for p in patient_list:
        if p[0] == patient:
            conn_.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
      VALUES ('" + doctor + "', '" + time + "', '" + date + "', 'FOLLOW-UP', '" + patient + "');")
            msg = 'Follow up appointment scheduled for existing patient: ' + patient
            break
        else:
            conn_.execute("INSERT INTO APPOINTMENTS (PHYSICIAN, TIME, DATE, TYPE, PATIENT) \
      VALUES ('" + doctor + "', '" + time + "', '" + date + "', 'NEW', '" + patient + "');")
            msg = 'New appointment created for ' + patient
            break

    conn_.commit()
    conn_.close()
    return msg

# Delete an existing appointment
@app.route('/deleteAppointment/', methods=['DELETE'])
def deleteAppointment():
    conn_ = sqlite3.connect('notable.db')
    doctor = request.args.get('doctor_name')
    date = request.args.get('date')
    time = request.args.get('time')
    patient = request.args.get('patient')

    # Check if appointment exists which needs to be deleted
    records = conn_.execute("select count(*) from APPOINTMENTS where PHYSICIAN = '" + doctor + "' and DATE = '" + date + "' and TIME = '" + time + "' and PATIENT = '" + patient + "'")
    for p in records:
        if p[0] == 0:
            conn_.close()
            return patient + ' does not have an appointment to cancel'

    # Delete appointment
    conn_.execute("DELETE FROM APPOINTMENTS where PHYSICIAN = '" + doctor + "' and DATE = '" + date + "' and TIME = '" + time + "' and PATIENT = '" + patient + "'")
    conn_.commit()
    conn_.close()
    return patient + '\'s appointment with ' + doctor + ' on ' + date + ' at ' + time + ' has been cancelled.'

# Get list of all doctors/physicians
@app.route('/getAllPhysicians/', methods=['GET'])
def getAllPhysicians():
    conn_ = sqlite3.connect('notable.db')
    records = conn_.execute('SELECT distinct PHYSICIAN FROM APPOINTMENTS')
    field_name_1 = "Physician\'s Name:"
    out = {}
    res = ''
    for row in records:
        out[field_name_1] = row[0]

        res += ''.join(str(json.dumps(out, indent=4)))

    conn_.close()
    return res

# Home page
@app.route("/")
def homePage():
    return "Welcome to the Notable Hospital Web UI"

# Running the Flask Application
app.run()


import json

class Human:
    def __init__(self, name):
        self.name = name

class HumanSerializer(json.JSONEncoder):
    def default(self, o):
        return o.__dict__
        # return {
        #     'name': o.name
        # }

human = {
    "name": 'roma',
    "city": "almaty"
}

thuman = """{
    "name": 'roma',
    "city": "almaty"
}"""
# f = open('/Users/roman/Desktop/pp2-spring2023/plan.md')
# dictionary = json.loads(f.read())

# json.loads(thuman)
text_human = json.dumps(human)
obj = Human(name='roma')

print(type(human), type(text_human))
print(human)
print(text_human)
print(json.dumps(obj, cls=HumanSerializer))  # => json.dumps({'name': 'roma})


import json
import os
import random
from nltk.tokenize import word_tokenize
from keras.preprocessing.sequence import pad_sequences
from collections import Counter
import pandas as pd
import _pickle as pickle
import numpy as np
from nltk.corpus import stopwords
from string import punctuation
from lemmatization import lemmatize
from tqdm import tqdm
from valence import assign_valence
from dependency_parsing import shift_valence, dependency_parse

def load_embeddings(file_name, vocabulary):
    """
    Loads word embeddings from the file with the given name.
    :param file_name: name of the file containing word embeddings
    :type file_name: str
    :param vocabulary: captions vocabulary
    :type vocabulary: numpy.array
    :return: word embeddings
    :rtype: dict
    """
    embeddings = dict()
    with open(file_name, 'r', encoding='utf-8') as doc:
        line = doc.readline()
        while line != '':
            line = line.rstrip('\n').lower()
            parts = line.split(' ')
            vals = np.array(parts[1:], dtype=np.float)
            if parts[0] in vocabulary:
                embeddings[parts[0]] = vals
            line = doc.readline()
    return embeddings

def load_embedding_weights(vocabulary, embedding_size, embedding_source, r1, r2):
    """
    Creates and loads embedding weights.
    :param vocabulary: captions vocabulary
    :type vocabulary: numpy.array
    :param embedding_size: embedding size
    :type embedding_size: int
    :param embedding_source: source of the pre-trained embeddings
    :type embedding_source: string
    :param r1: lower range boundary of reviews
    :type r1: int
    :param r2: upper range boundary of reviews
    :type r2: int
    :return: embedding weights
    :rtype: numpy.array
    """
    assert embedding_source in ['wikipedia', 'twitter']
    if os.path.exists(f'data/embedding_matrix_{embedding_source}_{embedding_size}_{r1}_{r2}.pkl'):
        with open(f'data/embedding_matrix_{embedding_source}_{embedding_size}_{r1}_{r2}.pkl', 'rb') as f:
            embedding_matrix = pickle.load(f)
    else:
        print('Creating embedding weights...')
        if embedding_source == 'wikipedia':
            embeddings = load_embeddings(f'data/glove.6B.{embedding_size}d.txt', vocabulary)
        else:
            embeddings = load_embeddings(f'data/glove.twitter.27B.{embedding_size}d.txt', vocabulary)
        embedding_matrix = np.zeros((len(vocabulary), embedding_size))
        for i in range(len(vocabulary)):
            if vocabulary[i] in embeddings.keys():
                embedding_matrix[i] = embeddings[vocabulary[i]]
            else:
                embedding_matrix[i] = np.random.standard_normal(embedding_size)
        with open(f'data/embedding_matrix_{embedding_source}_{embedding_size}_{r1}_{r2}.pkl', 'wb') as f:
            pickle.dump(embedding_matrix, f)
    return embedding_matrix

def load_word_mappings(vocabulary, r1, r2):
    """
    Loads word_to_id and id_to_word according to the given vocabulary. They are created if they do not exist.
    :param vocabulary: reviews vocabulary
    :type vocabulary: numpy.array
    :param r1: lower range boundary of reviews
    :type r1: int
    :param r2: upper range boundary of reviews
    :type r2: int
    :return: word mappings
    :rtype: dict, dict
    """
    if os.path.exists(f'data/word_to_id_{r1}_{r2}.pkl') and os.path.exists(f'data/id_to_word_{r1}_{r2}.pkl'):
        with open(f'data/id_to_word_{r1}_{r2}.pkl', 'rb') as f:
            id_to_word = pickle.load(f)
        with open(f'data/word_to_id_{r1}_{r2}.pkl', 'rb') as f:
            word_to_id = pickle.load(f)
    else:
        id_to_word = dict()
        word_to_id = dict()
        for i, word in zip(range(len(vocabulary)), vocabulary):
            id_to_word[i + 1] = word
            word_to_id[word] = i + 1
        with open(f'data/id_to_word_{r1}_{r2}.pkl', 'wb') as f:
            pickle.dump(id_to_word, f)
        with open(f'data/word_to_id_{r1}_{r2}.pkl', 'wb') as f:
            pickle.dump(word_to_id, f)
    return word_to_id, id_to_word

def load_train_val_test_subsets(r1, r2):
    if not os.path.exists(f'data/yelp_reviews_train_ids_{r1}_{r2}.csv'):
        review_ids = list()
        with open(f'data/yelp_reviews_filtered_{r1}_{r2}.json', 'r', encoding='utf-8') as doc_r:
            line = doc_r.readline()
            while line != '':
                review_id = json.loads(line)['review_id']
                review_ids.append(review_id)
                line = doc_r.readline()
        random.shuffle(review_ids)
        train_ids = review_ids[:int(0.7 * len(review_ids))]
        val_ids = review_ids[int(0.7 * len(review_ids)):int(0.85 * len(review_ids))]
        test_ids = review_ids[int(0.85 * len(review_ids)):]
        pd.DataFrame(train_ids).to_csv(f'data/yelp_reviews_train_ids_{r1}_{r2}.csv', index=None, header=None)
        pd.DataFrame(val_ids).to_csv(f'data/yelp_reviews_val_ids_{r1}_{r2}.csv', index=None, header=None)
        pd.DataFrame(test_ids).to_csv(f'data/yelp_reviews_test_ids_{r1}_{r2}.csv', index=None, header=None)
    else:
        train_ids = pd.read_csv(f'data/yelp_reviews_train_ids_{r1}_{r2}.csv').get_values().flatten()
        val_ids = pd.read_csv(f'data/yelp_reviews_val_ids_{r1}_{r2}.csv').get_values().flatten()
        test_ids = pd.read_csv(f'data/yelp_reviews_test_ids_{r1}_{r2}.csv').get_values().flatten()
    return train_ids, val_ids, test_ids

def load_sequences(r1, r2, review_ids, word_to_id, pad_size, pad=True):
    with open(f'data/yelp_reviews_lemmas_{r1}_{r2}.pkl', 'rb') as doc_r:
        data = pickle.load(doc_r)
    stars = pd.read_csv(f'data/yelp_reviews_stars_{r1}_{r2}.csv', index_col=[0]).to_dict(orient='index')
    sequences = list()
    classes = list()
    for review_id in review_ids:
        sequences.append(np.array([word_to_id[d] for d in data[review_id] if d in word_to_id.keys()]))
        classes.append(stars[review_id]['0'])
    if pad:
        return pad_sequences(sequences, pad_size), classes
    else:
        return sequences, classes

def tokenize_review(review):
    tokens = word_tokenize(review.lower())
    return tokens

def parse_json_review(review):
    review_json = json.loads(review)
    review_id = review_json['review_id']
    review_text = tokenize_review(review_json['text'])
    # return {'Review_ID': review_id, 'Text': review_text}
    return review_id, review_text

def filter_reviews(file_name):
    frequencies = pd.read_csv('data/user_review_frequencies.csv', sep=',', index_col=0)
    lengths = pd.read_csv('data/reviews_length.csv', sep=',', index_col=0)
    p_25 = np.percentile(lengths.get_values(), 25)
    p_75 = np.percentile(lengths.get_values(), 75)
    total = 0
    filtered_50_500 = 0
    filtered_10_500 = 0
    with open(file_name, 'r', encoding='utf-8') as doc_r, \
            open('data/yelp_reviews_filtered_50_500.json', 'w+', encoding='utf-8') as doc_w1, \
            open('data/yelp_reviews_filtered_10_500.json', 'w+', encoding='utf-8') as doc_w2:
        line = doc_r.readline()
        i = 0
        while line != '':
            i += 1
            if i % 100000 == 0:
                print(str(i))
            review = json.loads(line)
            total += 1
            f = frequencies.loc[review['user_id']].values[0]
            l = lengths.loc[review['review_id']].values[0]
            if f in range(50, 500) and l in range(int(p_25), int(p_75)):
                filtered_50_500 += 1
                doc_w1.write(line)
            if f in range(10, 500) and l in range(int(p_25), int(p_75)):
                filtered_10_500 += 1
                doc_w2.write(line)
            line = doc_r.readline()
    print('Total: ' + str(total))
    print('Filtered 50-500: ' + str(filtered_50_500))
    print('Percentage 50-500: ' + str(filtered_50_500 * 100.0 / total))
    print('Filtered 10-500: ' + str(filtered_10_500))
    print('Percentage 10-500: ' + str(filtered_10_500 * 100.0 / total))

def tokenize_reviews(r1, r2):
    reviews = dict()
    with open(f'data/yelp_reviews_filtered_{r1}_{r2}.json', 'r', encoding='utf-8') as doc_r:
        line = doc_r.readline()
        i = 0
        while line != '':
            i += 1
            if i % 100000 == 0:
                print(str(i))
            review_id, review_text = parse_json_review(line)
            reviews[review_id] = review_text
            line = doc_r.readline()
    with open(f'data/yelp_reviews_tokens_{r1}_{r2}.pkl', 'wb') as doc_w:
        pickle.dump(reviews, doc_w)

def lemmatize_reviews(r1, r2):
    with open(f'data/yelp_reviews_tokens_{r1}_{r2}.pkl', 'rb') as doc_r:
        data = pickle.load(doc_r)
    reviews = dict()
    for _, review_id in zip(tqdm(list(range(len(list(data.keys()))))), list(data.keys())):
        review_tokens = data[review_id]
        review_lemmas = lemmatize(review_tokens)
        reviews[review_id] = review_lemmas
    with open(f'data/yelp_reviews_lemmas_{r1}_{r2}.pkl', 'wb') as doc_w:
        pickle.dump(reviews, doc_w)

def create_reviews_subset(file_name):
    reviews = list()
    s = [0, 0, 0, 0, 0]
    with open(file_name, 'r', encoding='utf-8') as doc:
        line = doc.readline()
        while line != '':
            review = json.loads(line)
            stars = review['stars']
            if s[int(stars) - 1] < 500:
                s[int(stars) - 1] += 1
                review_id = review['review_id']
                review_text = review['text']
                reviews.append({'Review_ID': review_id, 'Text': review_text, 'Stars': stars})
            line = doc.readline()
    with open('data/yelp_reviews_subset.json', 'w+', encoding='utf-8') as doc:
        json.dump(reviews, doc)

def load_vocabulary(r1, r2):
    return pd.read_csv(f'data/vocabulary_{r1}_{r2}.csv', index_col=[0]).get_values().flatten()

def filter_vocabulary(vocab, frequencies):
    allowed_tokens = [f[0] for f in frequencies if f[1] > 15]
    return [w for w in vocab if w in allowed_tokens and w not in stopwords.words('english') and w not in punctuation]

def create_vocabulary(r1, r2):
    with open(f'data/yelp_reviews_lemmas_{r1}_{r2}.pkl', 'rb') as doc_r:
        data = pickle.load(doc_r)
    vocabulary = list()
    for _, review_id in zip(tqdm(list(range(len(list(data.keys()))))), list(data.keys())):
        review_lemmas = data[review_id]
        vocabulary.extend(review_lemmas)
    pd.DataFrame(list(set(vocabulary))).to_csv(f'data/vocabulary_full_{str(r1)}_{str(r2)}.csv')
    frequencies = sorted(Counter(list(vocabulary)).items(), key=lambda x: x[1], reverse=True)
    with open(f'data/vocabulary_full_frequencies_{str(r1)}_{str(r2)}.pkl', 'wb') as f:
        pickle.dump(frequencies, f)
    filtered_vocabulary = filter_vocabulary(list(set(vocabulary)), frequencies)
    pd.DataFrame(list(set(filtered_vocabulary))).to_csv(f'data/vocabulary_{str(r1)}_{str(r2)}.csv')

def assign_valence_vocabulary(r1, r2):
    vocabulary = pd.read_csv(f'data/vocabulary_{str(r1)}_{str(r2)}.csv', index_col=[0]).get_values().flatten()
    valences_nrc = assign_valence(vocabulary, ['afinn', 'nrc-hashtag', 'nrc-vad'])
    pd.DataFrame().from_dict(valences_nrc, orient='index').to_csv(f'data/vocabulary_{str(r1)}_{str(r2)}_val_nrc.csv')
    valences_yelp = assign_valence(vocabulary, ['yelp-sentiment'])
    pd.DataFrame().from_dict(valences_yelp, orient='index').to_csv(f'data/vocabulary_{str(r1)}_{str(r2)}_val_yelp.csv')

def assign_valence_reviews(r1, r2):
    with open(f'data/yelp_reviews_lemmas_{r1}_{r2}.pkl', 'rb') as doc_r:
        data = pickle.load(doc_r)
    val_nrc = pd.read_csv(f'data/vocabulary_{str(r1)}_{str(r2)}_val_nrc.csv', index_col=[0])
    val_y = pd.read_csv(f'data/vocabulary_{str(r1)}_{str(r2)}_val_yelp.csv', index_col=[0])
    lemmas_nrc = dict()
    values_nrc = dict()
    valence_nrc = dict()
    lemmas_yelp = dict()
    values_yelp = dict()
    valence_yelp = dict()
    for _, review_id in zip(tqdm(list(range(len(list(data.keys()))))), list(data.keys())):
        lemmas = list(set(data[review_id]))
        lemmas_list_nrc = sorted([(lemma, round(val_nrc.loc[lemma][0], 5)) for lemma in lemmas
                                  if lemma in val_nrc.index], key=lambda x: x[1], reverse=True)
        valence_nrc[review_id] = lemmas_list_nrc
        lemmas_nrc[review_id] = [x[0] for x in lemmas_list_nrc]
        values_nrc[review_id] = [x[1] for x in lemmas_list_nrc]
        lemmas_list_yelp = sorted([(lemma, round(val_y.loc[lemma][0], 5)) for lemma in lemmas
                                   if lemma in val_y.index], key=lambda x: x[1], reverse=True)
        valence_yelp[review_id] = lemmas_list_yelp
        lemmas_yelp[review_id] = [x[0] for x in lemmas_list_yelp]
        values_yelp[review_id] = [x[1] for x in lemmas_list_yelp]
    with open(f'data/yelp_reviews_lemmas_valence_nrc_{r1}_{r2}.pkl', 'wb') as doc_w:
        pickle.dump(valence_nrc, doc_w)
    pd.DataFrame().from_dict(lemmas_nrc, orient='index').to_csv(
        f'data/yelp_reviews_lemmas_val_nrc_words_{r1}_{r2}.csv')
    pd.DataFrame().from_dict(values_nrc, orient='index').to_csv(
        f'data/yelp_reviews_lemmas_val_nrc_values_{r1}_{r2}.csv')
    with open(f'data/yelp_reviews_lemmas_valence_yelp_{r1}_{r2}.pkl', 'wb') as doc_w:
        pickle.dump(valence_yelp, doc_w)
    pd.DataFrame().from_dict(lemmas_yelp, orient='index').to_csv(
        f'data/yelp_reviews_lemmas_val_yelp_words_{r1}_{r2}.csv')
    pd.DataFrame().from_dict(values_yelp, orient='index').to_csv(
        f'data/yelp_reviews_lemmas_val_yelp_values_{r1}_{r2}.csv')

def parse_dependencies():
    if os.path.exists('data/yelp_reviews_dependencies.pkl'):
        with open('data/yelp_reviews_dependencies.pkl', 'rb') as doc_r:
            dependencies = pickle.load(doc_r)
    else:
        dependencies = dict()
    with open('data/yelp_reviews_filtered_50_500.json', 'r', encoding='utf-8') as doc_r:
        line = doc_r.readline()
        i = 0
        while line != '':
            i += 1
            if i % 10 == 0:
                print(str(i))
            review = json.loads(line)
            review_id = review['review_id']
            if review_id not in dependencies.keys():
                review_text = review['text']
                try:
                    rel = dependency_parse(review_text.replace('/', ' '))
                except AssertionError:
                    rel = []
                    print(review_text)
                    with open('data/yelp_reviews_dependencies.pkl', 'wb') as doc_w:
                        pickle.dump(dependencies, doc_w)
                dependencies[review_id] = rel
                if i % 2000 == 0:
                    with open(f'data/yelp_reviews_dependencies_{i}.pkl', 'wb') as doc_w:
                        pickle.dump(dependencies, doc_w)
            line = doc_r.readline()
    with open('data/yelp_reviews_filtered_10_500.json', 'r', encoding='utf-8') as doc_r:
        line = doc_r.readline()
        i = 0
        while line != '':
            i += 1
            if i % 100 == 0:
                print(str(i))
            review = json.loads(line)
            review_id = review['review_id']
            if review_id not in dependencies.keys():
                review_text = review['text']
                try:
                    rel = dependency_parse(review_text.replace('/', ' '))
                except AssertionError:
                    rel = []
                    print(review_text)
                    with open('data/yelp_reviews_dependencies.pkl', 'wb') as doc_w:
                        pickle.dump(dependencies, doc_w)
                dependencies[review_id] = rel
                if i % 3000 == 0:
                    with open(f'data/yelp_reviews_dependencies_{i}.pkl', 'wb') as doc_w:
                        pickle.dump(dependencies, doc_w)
            line = doc_r.readline()
    with open('data/yelp_reviews_dependencies.pkl', 'wb') as doc_w:
        pickle.dump(dependencies, doc_w)

def context_shift(r1, r2):
    with open(f'data//yelp_reviews_lemmas_valence_nrc_{r1}_{r2}.pkl', 'rb') as doc_r:
        valence_nrc = pickle.load(doc_r)
    with open(f'data//yelp_reviews_lemmas_valence_yelp_{r1}_{r2}.pkl', 'rb') as doc_r:
        valence_yelp = pickle.load(doc_r)
    with open('data/yelp_reviews_dependencies_8000.pkl', 'rb') as doc_r:
        dependencies = pickle.load(doc_r)
    valence_nrc_shifted = dict()
    valence_yelp_shifted = dict()
    with open(f'data/yelp_reviews_filtered_{r1}_{r2}.json', 'r', encoding='utf-8') as doc_r:
        line = doc_r.readline()
        i = 0
        while line != '':
            i += 1
            if i % 50000 == 0:
                print(str(i))
            review = json.loads(line)
            review_id = review['review_id']
            if review_id in valence_nrc.keys() and review_id in dependencies.keys() and len(dependencies[review_id]) > 0:
                review_text = review['text']
                lemmas = [x[0] for x in valence_nrc[review_id]]
                values = [float(x[1]) for x in valence_nrc[review_id]]
                rel = dependencies[review_id]
                val_shift, rel = shift_valence(review_text.replace('/', ' '), lemmas, values, rel)
                valence_nrc_shifted[review_id] = [(l, v) for l, v in zip(lemmas, val_shift)]
                lemmas = [x[0] for x in valence_yelp[review_id]]
                values = [float(x[1]) for x in valence_yelp[review_id]]
                val_shift, rel = shift_valence(review_text.replace('/', ' '), lemmas, values, rel)
                valence_yelp_shifted[review_id] = [(l, v) for l, v in zip(lemmas, val_shift)]
            line = doc_r.readline()
    with open(f'data/yelp_reviews_lemmas_valence_shifted_nrc_{r1}_{r2}.pkl', 'wb') as doc_w:
        pickle.dump(valence_nrc_shifted, doc_w)
    with open(f'data/yelp_reviews_lemmas_valence_shifted_yelp_{r1}_{r2}.pkl', 'wb') as doc_w:
        pickle.dump(valence_yelp_shifted, doc_w)

def create_user_review_frequencies(file_name):
    frequencies = dict()
    with open(file_name, 'r', encoding='utf-8') as doc:
        line = doc.readline()
        i = 0
        while line != '':
            i += 1
            if i % 100000 == 0:
                print(str(i))
            review = json.loads(line)
            user_id = review['user_id']
            if user_id in frequencies.keys():
                frequencies[user_id] += 1
            else:
                frequencies[user_id] = 1
            line = doc.readline()
    pd.DataFrame().from_dict(frequencies, orient='index').to_csv('data/user_review_frequencies.csv')

def calculate_reviews_length(file_name):
    lengths = dict()
    with open(file_name, 'r', encoding='utf-8') as doc:
        line = doc.readline()
        i = 0
        while line != '':
            i += 1
            if i % 100000 == 0:
                print(str(i))
            review = json.loads(line)
            lengths[review['review_id']] = len(tokenize_review(review['text']))
            line = doc.readline()
    pd.DataFrame().from_dict(lengths, orient='index').to_csv('data/reviews_length.csv')

def calculate_review_length_distribution(r1, r2):
    with open(f'data/yelp_reviews_lemmas_{r1}_{r2}.pkl', 'rb') as doc_r:
        data = pickle.load(doc_r)
    vocab = load_vocabulary(r1, r2)
    lengths = dict()
    lengths_filtered = dict()
    for _, k in zip(tqdm(list(range(len(list(data.keys()))))), list(data.keys())):
        lengths[k] = len(data[k])
        lengths_filtered[k] = len([d for d in data[k] if d in vocab])
    pd.DataFrame().from_dict(lengths, orient='index').to_csv(f'data/reviews_length_{r1}_{r2}.csv')
    pd.DataFrame().from_dict(lengths_filtered, orient='index').to_csv(f'data/reviews_length_filtered_{r1}_{r2}.csv')

def create_label_data(r1, r2):
    stars = dict()
    with open(f'data/yelp_reviews_filtered_{r1}_{r2}.json', 'r', encoding='utf-8') as doc_r:
        line = doc_r.readline()
        while line != '':
            review = json.loads(line)
            review_id = review['review_id']
            review_stars = review['stars']
            stars[review_id] = review_stars
            line = doc_r.readline()
    pd.DataFrame().from_dict(stars, orient='index').to_csv(f'data/yelp_reviews_stars_{r1}_{r2}.csv')

if __name__ == '__main__':
    # create_reviews_subset('data/yelp_reviews.json')
    # create_user_review_frequencies('data/yelp_reviews.json')
    # calculate_reviews_length('data/yelp_reviews.json')
    # filter_reviews('data/yelp_reviews.json')
    # create_label_data(50, 500)
    # create_label_data(10, 500)
    # tokenize_reviews(50, 500)
    # tokenize_reviews(10, 500)
    # lemmatize_reviews(50, 500)
    # lemmatize_reviews(10, 500)
    # calculate_review_length_distribution(50, 500)
    # calculate_review_length_distribution(10, 500)
    # create_vocabulary(50, 500)
    # create_vocabulary(10, 500)
    # assign_valence_vocabulary(50, 500)
    # assign_valence_vocabulary(10, 500)
    # assign_valence_reviews(50, 500)
    # assign_valence_reviews(10, 500)
    # parse_dependencies()
    context_shift(50, 500)
    context_shift(10, 500)


from collections import Counter
import math

def get_nums(fn):
    nums = [int(line) for line in open(fn).read().split('\n')]
    return nums

def get_deltas(nums):
    j = 0
    deltas = []
    for a in sorted(nums):
        if a < j + 4:
            deltas.append(a - j)
            j = a
        else:
            return deltas
    return deltas

def count_sequential(deltas):
    seqs = {}
    count = 1
    for ind in range(len(deltas) - 1):
        if deltas[ind] == deltas[ind + 1]:
            count += 1
        else:
            if deltas[ind] in seqs:
                seqs[deltas[ind]].append(count)
            else:
                seqs[deltas[ind]] = [count]
            count = 1
    seqs[deltas[-1]].append(count)
    return seqs

def all_comb(n):
    # all combinations nCr where r is [len(n)...0]
    ac = 0
    for x in range(n + 1):
        ac += math.comb(n, x)
    return ac

def ways_from_seq(seqs):
    ans = 1
    for n in seqs[1]:
        m = all_comb(n - 1)
        if n > 3:
            m = m - 3**(n - 4)
        ans *= m
    return ans

nums = get_nums('10.txt')
deltas = get_deltas(nums)

# Part A
ans1 = (Counter(deltas)[1] * (Counter(deltas)[3] + 1))
print(f"Part 1:\t{ans1}")

# Part B
seqs = count_sequential(deltas)
ans2 = ways_from_seq(seqs)
print(f"Part 2:\t{ans2}")


import sys, os

sys.path.append(os.path.join(os.path.dirname(__file__)))

from datetime import datetime, date
import pandas_market_calendars as mcal
import pytz
import airflow_env_variables
import traceback

sys.path.append(airflow_env_variables.DWH_SCRIPT)
from common.utils import send_line_message

def send_notification(dag_id, tags, type, optional_message=None):
    jst = pytz.timezone("Asia/Tokyo")
    ts_now = datetime.now(jst).strftime("%Y-%m-%d %H:%M:%S")
    message = "{} [{}]{}\nAirflow Dags: {}".format(ts_now, ",".join(tags), type, dag_id)
    if optional_message:
        message += "\n\n" + optional_message
    send_line_message(message)

def _unix_time_millisecond_to_second(unix_time):
    return int((unix_time) / 1000.0)

def _get_ts_now(_timezone):
    tz = pytz.timezone(_timezone)
    return datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S")

def get_dt_from_unix_time(dt_unix_time):
    ts = datetime.utcfromtimestamp(int(dt_unix_time))
    dt = date(ts.year, ts.month, ts.day).strftime("%Y-%m-%d")
    return dt

def process_candle_data_from_poloniex(data):
    timezone = "UTC"
    batch_data = []
    for asset_name, asset_data in data.items():
        for d in asset_data:
            ts_create_utc = datetime.utcfromtimestamp(int(d[13]) / 1000.0)
            dt_create_utc = date(ts_create_utc.year, ts_create_utc.month, ts_create_utc.day).strftime("%Y-%m-%d")
            try:
                batch_data.append(
                    [
                        asset_name,  # id
                        float(d[0]),  # low
                        float(d[1]),  # high
                        float(d[2]),  # open
                        float(d[3]),  # close
                        float(d[4]),  # amount
                        float(d[5]),  # quantity
                        float(d[6]),  # buyTakerAmount
                        float(d[7]),  # buyTakerQuantity
                        int(d[8]),  # tradeCount
                        _unix_time_millisecond_to_second(d[9]),  # ts
                        float(d[10]),  # weightedAverage
                        d[11],  # interval
                        _unix_time_millisecond_to_second(d[12]),  # startTime
                        _unix_time_millisecond_to_second(d[13]),  # closeTime
                        dt_create_utc,  # dt_create_utc
                        str(ts_create_utc),  # ts_create_utc
                        _get_ts_now(timezone),  # ts_insert_utc
                    ]
                )
            except Exception as error:
                print("Error:".format(error))
                print(traceback.format_exc())
                print("asset_data ==>> \n", asset_data)
                sys.exit(1)
    return batch_data

def process_yahoofinancials_data(data):
    timezone = "UTC"
    batch_data = []
    for symbol_name, data in data.items():
        currency = data["currency"]
        prices = data["prices"]
        tz_gmtoffset = data["timeZone"]["gmtOffset"]
        for p in prices:
            try:
                if None not in list(p.values()):
                    batch_data.append(
                        [
                            symbol_name,  # id
                            float(p["low"]),  # low
                            float(p["high"]),  # high
                            float(p["open"]),  # open
                            float(p["close"]),  # close
                            float(p["volume"]),  # volume
                            float(p["adjclose"]),  # adjclose
                            currency,  # currency
                            int(p["date"]),  # dt_unix
                            p["formatted_date"],  # dt
                            int(tz_gmtoffset),  # tz_gmtoffset
                            _get_ts_now(timezone),  # ts_insert_utc
                        ]
                    )
                else:
                    print("WARN: None data is containing: ", p)
            except Exception as error:
                print("Error:".format(error))
                print(traceback.format_exc())
                print("data ==>> \n", p)
                sys.exit(1)
    return batch_data

def is_market_open(date, market):
    calendar = mcal.get_calendar(market)
    if len(calendar.schedule(start_date=date, end_date=date).index) > 0:
        return True
    else:
        return False


import numpy as np
import torch
from sentence_transformers import SentenceTransformer
import nltk.data
import nltk

nltk.download('punkt')

model = SentenceTransformer('all-mpnet-base-v2')
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')

def cos_sim(a, b):
    sims = a @ b.T
    a_norm = np.linalg.norm(a, axis=-1)
    b_norm = np.linalg.norm(b, axis=-1)
    a_normalized = (sims.T / a_norm.T).T
    sims = a_normalized / b_norm
    return sims

def degree_power(A, k):
    degrees = np.power(np.array(A.sum(1)), k).ravel()
    D = np.diag(degrees)
    return D

def normalized_adjacency(A):
    normalized_D = degree_power(A, -0.5)
    return torch.from_numpy(normalized_D.dot(A).dot(normalized_D))

def get_sentences(source_text):
    sentence_ranges = list(sent_detector.span_tokenize(source_text))
    sentences = [source_text[start:end] for start, end in sentence_ranges]
    return sentences, sentence_ranges

def text_rank(sentences):
    vectors = model.encode(sentences)
    adjacency = torch.tensor(cos_sim(vectors, vectors)).fill_diagonal_(0.)
    adjacency[adjacency < 0] = 0
    return normalized_adjacency(adjacency)

def terminal_distr(adjacency, initial=None):
    sample = initial if initial is not None else torch.full((adjacency.shape[0],), 1.)
    scores = sample.matmul(torch.matrix_power(adjacency, 10)).numpy().tolist()
    return scores

def extract(source_text):
    sentences, sentence_ranges = get_sentences(source_text)
    adjacency = text_rank(sentences)
    return sentence_ranges, adjacency

def get_results(sentences, adjacency):
    scores = terminal_distr(adjacency)
    for score, sentence in sorted(zip(scores, sentences), key=lambda xs: xs[0]):
        if score > 1.1:
            print('{:0.2f}: {}'.format(score, sentence))


"""
Singleton:
The Singleton pattern ensures that a class has only one instance and provides a global point of access to that instance.
It is named after the singleton set, which is defined to be a set containing one element. The office of the
President of the United States is a Singleton. The United States Constitution specifies the means by which a president
is elected, limits the term of office, and defines the order of succession. As a result, there can be at most one active
president at any given time. Regardless of the personal identity of the active president, the title, "The President of the
United States" is a global point of access that identifies the person in the office.

Problem:
Application needs one, and only one, instance of an object. Additionally, lazy initialization and global access are necessary.

*References:
https://www.tutorialspoint.com/python_design_patterns/python_design_patterns_singleton.htm
https://sourcemaking.com/design_patterns/singleton
"""

class Singleton:
   instance = None
   test = "Atma"
   @staticmethod
   def getInstance():
      """ Static access method. """
      if Singleton.instance == None:
         Singleton()
      return Singleton.instance
   def __init__(self):
      """ Virtually private constructor. """
      if Singleton.instance != None:
         raise Exception("This class is a singleton!")
      else:
         Singleton.instance = self
   def setTest(self, test):
   	self.test = test
   def getTest(self):
   	return self.test

firstInstance = Singleton()
secondInstance = Singleton.getInstance()

firstInstance.setTest("Ultima")
print("SecondInstance - variable test: ", secondInstance.getTest())

print("Trying to create another instance:")
x = Singleton()
"""
OUTPUT:
SecondInstance - variable test: Ultima

Trying to create another instance:
	raise Exception("This class is a singleton!")
"""


"""
Tests for Statistic
"""
import numpy as np
import os
from statistic import Statistic, ModelStatistic, \
    ReactionStatistic, \
    ComplexTransformationReactionStatistic
from sbml_shim import SBMLShim
#from util import createSBML, createReaction
import unittest

IGNORE_TEST = False
DIRECTORY = os.path.dirname(os.path.realpath(__file__))
TEST_FILE = os.path.join(DIRECTORY, "chemotaxis.xml")

class DummyReactionStatistic(ReactionStatistic):

  def _addValues(self, value_dict, idx):
    if "Dummy" not in value_dict:
      value_dict["Dummy"] = [1]
    else:
      value_dict["Dummy"].append(1)
    return value_dict

class TestClass(object):
  pass
class TestSubclassA(TestClass):
  pass
class TestSubclassB(TestClass):
  pass
class TestSubclassC(TestSubclassA):
  pass
LEAF_SUBCLASSES = set([TestSubclassB, TestSubclassC])

#############################
# Statistic
#############################
class TestStatistic(unittest.TestCase):

  def setUp(self):
    self.shim = SBMLShim(filepath=TEST_FILE)

  def testConstructor(self):
    if IGNORE_TEST:
      return
    statistic = Statistic(self.shim)
    self.assertIsNotNone(statistic._shim)

  def _testJointSubstring(self, substrings, string, expected):
    result = Statistic._jointSubstring(substrings, string)
    self.assertEqual(result, expected)
    substrings.reverse()
    result = Statistic._jointSubstring(substrings, string)
    self.assertEqual(result, expected)

  def testJointSubstring(self):
    if IGNORE_TEST:
      return
    self._testJointSubstring(["xx", "yy"], "xx_yy", 2)
    self._testJointSubstring(["xxy", "y"], "xx_yy", 1)
    self._testJointSubstring(["xx", "yy", "zz"], "zzz_xx_yy", 3)

  def testFindLeafSubclasses(self):
    leaves = Statistic._findLeafSubclasses(TestClass)
    self.assertEqual(leaves, LEAF_SUBCLASSES)

  def testGetAllStatistics(self):
    statistics = Statistic.getAllStatistics(self.shim)
    self.assertTrue(len(statistics.values()) > 0)
    self.assertTrue("Complex_Disassociation_mean" in statistics)

  def testStatisticError(self):
    shim = SBMLShim.getShimForBiomodel("BIOMD0000000020")
    statistics = Statistic.getAllStatistics(self.shim)
    self.assertTrue(len(statistics.keys()) > 10)

#############################
# Reaction Statistics
#############################
class TestReactionStatistic(unittest.TestCase):

  def setUp(self):
    self.shim = SBMLShim(filepath=TEST_FILE)

  def testgetStatistic(self):
    if IGNORE_TEST:
      return
    reaction_statistic = DummyReactionStatistic(self.shim)
    result = reaction_statistic.getStatistic()
    self.assertEqual(result["Dummy_mean"], 1.0)
    self.assertEqual(result["Dummy_std"], 0.0)

  def _testTransformStatistic(self,
        reactants, products, key, value):
    """
    :param list-of-str reactants:
    :param list-of-str products:
    :param str key: key in the result
    :param float value: value in the result
    """
    sbmlstr = SBMLShim.createSBMLReaction(reactants, products)
    shim = SBMLShim(sbmlstr=sbmlstr)
    complex = ComplexTransformationReactionStatistic(shim)
    self.assertEqual(complex._addValues({}, 0)[key], [value])

  def testComplexDisassociationReactionStatistic(self):
    if IGNORE_TEST:
      return
    key = "Complex_Disassociation"
    self._testTransformStatistic(["AB"], ["B", "A"], key, 1)
    self._testTransformStatistic(["AB"], ["A", "B"], key, 1)
    self._testTransformStatistic(["AB"], ["A", "C"], key, 0)
    self._testTransformStatistic(["AB"], ["A", "C", "B"], key, 1)
    self._testTransformStatistic(["A_B"], ["A", "B"], key, 1)
    self._testTransformStatistic(["A_B"], ["B", "A"], key, 1)
    key = "Complex_Formation"
    self._testTransformStatistic(["A", "B"], ["AB"], key, 1)
    self._testTransformStatistic(["B", "A"], ["AB"], key, 1)
    self._testTransformStatistic(["A", "C"], ["AB"], key, 0)
    self._testTransformStatistic(["A", "C", "B"], ["AB"], key, 1)
    self._testTransformStatistic(["A", "B"], ["A_B"], key, 1)
    self._testTransformStatistic(["B", "A"], ["A_B"], key, 1)

  def testReactionStatisticError(self):
    #if IGNORE_TEST:
    #  return
    shim = SBMLShim.getShimForBiomodel("BIOMD0000000020")
    statistics = Statistic.getAllStatistics(shim)
    self.assertEqual(statistics["Num_Reactions"], 0)

  def testGetDoc(self):
    doc_dict = Statistic.getDoc()
    self.assertTrue("Num_Parameters" in doc_dict)

if __name__ == '__main__':
  unittest.main()


"""
In this file you'll find functions used to load initial data in a safe and django way,
plus some tools to check and repair your DB.

See doc/DESIGN.txt for caveats about database
"""
import os
import shutil
import re

from django.utils.importlib import import_module
from django.conf import settings

import twistranet
from twistranet.twistapp.lib.log import log

def install_theme():
    """
    Copy theme in a new twistranet instance.
    Change settings of this instance
    """

    # Copy theme-defined static files into the static directory.
    # We start by importing the theme app
    theme_app = import_module(settings.TWISTRANET_THEME_APP)
    theme_app_dir = os.path.split(theme_app.__file__)[0]
    dest_root = os.path.abspath(os.path.join(settings.HERE, 'www', 'static'))
    source_root = os.path.abspath(os.path.join(theme_app_dir, 'static'))
    if not os.path.isdir(dest_root):
        os.makedirs(dest_root)
    for root, dirs, files in os.walk(source_root):
        relative_root = root[len(source_root) + 1:]
        for d in dirs:
            dest_dir = os.path.join(dest_root, relative_root, d)
            if not os.path.isdir(dest_dir):
                os.mkdir(dest_dir)
        for fname in files:
            dest_file = os.path.join(dest_root, relative_root, fname)
            shutil.copy(
                os.path.join(source_root, root, fname),
                dest_file,
            )

    log.info("The twistranet theme has been installed in your project.")


import numpy as np

# to plot the data
import matplotlib.pyplot as plt

# for neural network
import torch
import torch.nn as nn
import torch.nn.functional as F
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# to save parameters
import copy

# for evaluating results
from sklearn.metrics import classification_report

# to track progress
from tqdm.notebook import tqdm

# train network
def trainNet(net,criterion,optimizer,scheduler,train_loader,val_loader,epochs,print_every=None,earlyStopping=None,approach="defailt"):

  print("training network")
  # early stopping parameter indicates how long to wait after last time validation loss improved.

  if not print_every:
      print_every = int(epochs / 10)

  stopCounter = 0
  avg_trainLosses = []
  avg_valLosses = []

  for epoch in tqdm(range(epochs)):  # loop over the dataset multiple times

    train_loss = []
    val_loss = []

    net.train()
    for i, (inputBatch,labelBatch) in enumerate(train_loader):

        if approach == "fusion":
          inputBatch = [ip.to(device).float() for ip in inputBatch]
        else:
          inputBatch = inputBatch.to(device).float()

        labelBatch = labelBatch.to(device)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputBatch = net(inputBatch)
        loss = criterion(outputBatch, labelBatch)
        loss.backward()
        optimizer.step()

        # print statistics
        train_loss.append(loss.item())

    net.eval()
    for i, (inputBatch,labelBatch) in enumerate(val_loader):
      with torch.no_grad():

        if approach == "fusion":
          inputBatch = [ip.to(device).float() for ip in inputBatch]
        else:
          inputBatch = inputBatch.to(device).float()

        labelBatch = labelBatch.to(device)

        # forward + backward + optimize
        outputBatch = net(inputBatch)
        loss = criterion(outputBatch, labelBatch)
        val_loss.append(loss.item())

    avg_trainLoss = sum(train_loss) / len(train_loss)
    avg_valLoss = sum(val_loss) / len(val_loss)
    avg_trainLosses.append(avg_trainLoss)

    if (epoch > 0) and (avg_valLoss < min(avg_valLosses)):
        best_params = copy.deepcopy(net.state_dict())
        best_epoch, best_loss = epoch, avg_valLoss
        stopCounter = 0
    else:
      stopCounter += 1
      if stopCounter == earlyStopping:
        break
    avg_valLosses.append(avg_valLoss)

    # print statistics
    if epoch % print_every == print_every - 1:
      print('epoch: %d, train loss: %.3f, val loss: %.3f' % (epoch + 1, avg_trainLoss, avg_valLoss))

    scheduler.step(avg_valLoss)

  print('Finished Training')
  plt.plot(avg_trainLosses, label='train loss')
  plt.plot(avg_valLosses, label='val loss')
  plt.plot([best_loss]*epoch, linestyle='dashed')
  plt.plot(best_epoch, best_loss, 'o')
  plt.legend()
  plt.show()

  return best_params

# evaluate and print
def evaluate(net,input1,output_true,classes=None, approach="default"):

  output_pred = netOutput(net,input1,outType="class", approach=approach)

  if classes is not None:
    print(classification_report(output_true, output_pred, target_names=classes, labels=range(len(classes)) ,digits=4))
  else:
    print(classification_report(output_true, output_pred, digits=4))

def netOutput(net, inputs, outType="default", approach="default"):
  net.eval()
  outputs = []

  with torch.no_grad():
    for input1 in tqdm(inputs):
      if outType == "latent":
        output = net.latent(torch.from_numpy(input1).unsqueeze(0).float().to(device)).cpu().numpy()

      elif outType == "class":
        if approach == "fusion":
          input1 = [torch.from_numpy(ip).unsqueeze(0).float() for ip in input1]
          output = net(input1).cpu().numpy()
        else:
          output = net(torch.from_numpy(input1).unsqueeze(0).float().to(device)).cpu().numpy()
        output = np.argmax(output)

      else:
        output = net(torch.from_numpy(input1).unsqueeze(0).float().to(device)).cpu().numpy()

      outputs.append(output)

  return np.array(outputs)


from git import *
from date import *
from numpy import *
from bug_finder import get_bug_dict
import datetime
import pickle

def get_dico(repo_path, branch, last_release_commit_id):
    """
    arg: repo_path -> path to the git repository
         branch -> name of the branch to be analysed
         last_release_commit_id -> last commit of the release to be alaysed
    return: dictionary where the key is a filename and the value is another dictionary of the type:
                {"size":size, "churn": churn,"minor":minor, "major":major, "total":total , "ownership":ownership}
    """
    repo = Repo(repo_path)
    commits  = repo.iter_commits()
    dico = {}
    dico_sizechurn = {}

    for commit in repo.iter_commits(branch):

        if not (between_beginning_2016_RL(commit)): continue

        contributor_name = commit.author.name
        changes = commit.stats.files

        for file in changes:

            file_name = str(file)
            churn = changes[file]['deletions']+changes[file]['insertions']

            if file_name in dico:
                if contributor_name in dico[file_name]:
                    dico[file_name][contributor_name] += 1
                else:
                    dico[file_name][contributor_name] = 1
            else:
                dico[file_name] = {contributor_name: 1}

            if file_name in dico_sizechurn:
                dico_sizechurn[file_name]["churn"] += churn
            else :
                dico_sizechurn[file_name] = {"churn" : churn, "size" : 0}

    deletedFiles = []

    for filename in dico_sizechurn:

        try:
            size = 0
            for commit, lines in repo.blame(last_release_commit_id, filename):
                size += len(lines)
            dico_sizechurn[filename]["size"]=size

        except :
            #the file named filename doesn't exist in the last release of 2016
            deletedFiles.append(filename)
            pass

    for file in deletedFiles:
            del dico[file]
            del dico_sizechurn[file]

    return(dico, dico_sizechurn)

def metrics(dico):
    """
    arg: dico ->dictionary where the keys are file names and the values are dictionaries with authors as keys and number of contributions as values

    return: dictionary that contains the metrics major, minor, total, ownership for each file
    """
    output= {}
    for file in dico:
        contributions=0
        major=0
        minor=0
        ownership=0
        for user in dico[file]:
            contributions += dico[file][ user]
        for user in dico[file]:
            prop = dico[file][ user]/float(contributions)
            if prop>=0.05:
                major+=1
            else:
                minor+=1
            if prop > ownership:
                ownership = prop
        dico_file = {'total' : minor+major, 'major':major, 'minor':minor, 'ownership':ownership}
        output[file]= dico_file
    return(output)

def write_results(X,y,dict):

    file  = open("results.tsv", "w")
    file.write("filename\tsize\tchurn\tminor\tmajor\ttotal\townership\tbugs\n")
    i = 0

    for filename in dict:
        file.write(filename+"\t"+str(X[i,0])+"\t"+str(X[i,1])+"\t"+str(X[i,2])+"\t"+str(X[i,3])+"\t"+str(X[i,4])+"\t"+str(X[i,5])+"\t"+str(y[i])+"\n")
        i+=1

    file.close()

def vectorization(dict, dict_sizechurn, dict_bugs):
    X = []
    y = []

    for filename in dict:
        d = dict[filename]
        d_sizechurn = dict_sizechurn[filename]

        if filename in dict_bugs :
            bugs = dict_bugs[filename]
        else :
            bugs = 0

        X.append( [d_sizechurn["size"],d_sizechurn["churn"],d["minor"],d["major"],d["total"],d["ownership"]])

        y.append(bugs)

    return (array(X), array(y))

def save_data(X, y, path):
    pickle.dump([X,y], open(path, "wb"))

def load_data(path):
    data = pickle.load(open(path, "rb"))
    return data[0], data[1]

def main():
    path = "/Users/vahagn/bootstrap"
    dict_contributors, dict_sizechurn = get_dico(path, 'v4-dev', 'b5890e0608ad2262cde4a38e90afa19f1cb5d852')
    dict_metrics = metrics(dict_contributors)
    dict_bugs = get_bug_dict(Repo(path), 'v4-dev')

    (X, y) = vectorization(dict_metrics, dict_sizechurn, dict_bugs)
    write_results(X,y, dict_contributors)

    save_data(X,y,"/home/guillaume/Documents/Athens TUD18/save_bootstrap.pkl")

if __name__ == "__main__":
    main()


import pytest

def test_library_add_book():
    """Test adding books works"""
    from library import Library
    library = Library()
    library.add_book('My First Book', 'Alice')
    library.add_book('My Second Book', 'Alice')
    library.add_book('A Different Book', 'Bob')

    assert len(library.books) == 3

def test_library_len():
    """Test 'len` works"""
    from library import Library
    library = Library()
    library.add_book('My First Book', 'Alice')
    library.add_book('My Second Book', 'Alice')
    library.add_book('A Different Book', 'Bob')

    assert len(library) == 3

def test_library_get_book():
    """Test getting books by author works"""
    from library import Library
    library = Library()
    library.add_book('My First Book', 'Alice')
    library.add_book('My Second Book', 'Alice')
    library.add_book('A Different Book', 'Bob')

    book = library[1]
    assert book.title == 'My Second Book'

def test_library_by_author():
    """Test getting books by author works"""
    from library import Library
    library = Library()
    library.add_book('My First Book', 'Alice')
    library.add_book('My Second Book', 'Alice')
    library.add_book('A Different Book', 'Bob')

    books = library.by_author('Alice')
    assert len(books) == 2
    assert [b.author for b in books] == ['Alice']*2

def test_library_by_author_missing():
    """Test getting books by author works when author is missing"""
    from library import Library
    library = Library()
    library.add_book('My First Book', 'Alice')
    library.add_book('My Second Book', 'Alice')
    library.add_book('A Different Book', 'Bob')

    with pytest.raises(KeyError):
        library.by_author('Carol')

def test_library_authors():
    """Test getting all authors in the library"""
    from library import Library
    library = Library()
    library.add_book('My First Book', 'Alice')
    library.add_book('My Second Book', 'Alice')
    library.add_book('A Different Book', 'Bob')

    authors = library.authors
    assert set(authors)=={'Alice','Bob'}

def test_library_titles():
    """Test getting all titles in the library"""
    from library import Library
    library = Library()
    library.add_book('My First Book', 'Alice')
    library.add_book('My Second Book', 'Alice')
    library.add_book('A Different Book', 'Bob')

    titles = library.titles
    assert set(titles)=={'My First Book','My Second Book', 'A Different Book'}

def test_library_merge():
    """Test merging two libraries"""
    from library import Library
    library1 = Library()
    library1.add_book('My First Book', 'Alice')
    library1.add_book('A Different Book', 'Bob')

    library2 = Library()
    library2.add_book('My First Book', 'Alice')
    library2.add_book('My Second Book', 'Alice')
    library2.add_book('A Different Book', 'Bob')
    library2.add_book('Yet another Book', 'Carol')

    new_library = library1.union(library2)
    assert len(new_library)==4


#!/usr/bin/env python

import os.path as osp
import yaml

from typing import List, Dict, Any
from difflib import get_close_matches

import torch.nn as nn

import timm.models as tmodels
from detectron2.config import CfgNode

_PREFIX: str = 'build_'
_DETECTRON2: str = 'detectron2'
_SUFFIX: str = '_backbone'

def get_models(default_cfgs: dict, name: str = None) -> Dict[str, Any]:
    if name is not None:
        try:
            assert name in default_cfgs
            return getattr(tmodels, name)
        except AttributeError:
            raise ValueError(f'Model name {name} not found')

    return {
        model_name: getattr(tmodels, model_name)
        for model_name in default_cfgs.keys()
        if getattr(tmodels, model_name, None)
    }

def remove_named_children(cfg, original_model) -> nn.Module:
    removals = cfg.MODEL.BACKBONE.CONFIG.REMOVE_LAYERS
    # model = nn.Sequential()
    for name, module in original_model.named_children():
        if name in removals:
            setattr(original_model, name, nn.Identity())
            # continue
        # model.add_module(name, module)
    # return model
    return original_model

def replace_layers(cfg, original_model):

    from detectron2_timm.layers import LAYER_REGISTRY

    for replace in cfg.MODEL.BACKBONE.CONFIG.REPLACE_LAYERS:
        assert isinstance(replace, (list, tuple))
        if len(replace) == 0:
            continue
        layer = LAYER_REGISTRY.get(replace[1])
        original_model.add_module(replace[0], layer(cfg))

    return original_model

def find_model_config(model_cfgs: dict, model_name: str) -> dict:
    match = get_close_matches(model_name, model_cfgs.keys())[0]
    return model_cfgs[match]

def get_func_name(func, model_name: str) -> str:
    assert len(model_name) > 0 and callable(func)
    return func.__name__.replace(_DETECTRON2, model_name)

def get_model_name(func_name: str, replace_with: str = '') -> str:
    assert len(func_name) > 0
    name = func_name.replace(_PREFIX, replace_with)
    name = name.replace(_SUFFIX, replace_with)
    return name.replace('_fpn', replace_with)

def load_yaml(path: str) -> dict:
    assert osp.isfile(path)
    with open(path, 'r') as stream:
        data = yaml.safe_load(stream)
    return data

def get_attr(name: str):
    return getattr(tmodels, name)

def patch_size_mod2(cfg: CfgNode) -> List[int]:
    stride = max(cfg.MODEL.BACKBONE.CONFIG.STRIDES)
    max_size = cfg.INPUT.MAX_SIZE_TRAIN
    min_size = max(cfg.INPUT.MIN_SIZE_TRAIN)
    assert stride > 0
    return (min_size - (min_size % stride), max_size - (max_size % stride))


"""
python3.4 construct_matrices_sdss.py -0.1 0.05 X_AGN
"""
import sys
import numpy as n
from scipy.interpolate import interp1d
import os
import astropy.io.fits as fits
import pickle
import glob

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as p

version = 'v5'

# parameters of the run
zmin = 1. # float(sys.argv[1])
zmax = 2. # float(sys.argv[2])
gal_type = 'cosmosagn' # sys.argv[3]
# ELG, LRG, X_AGN, QSO

Nspec_max = 3000

name = "deimos_"+gal_type+"_zmin_"+str(int(10*zmin)).zfill(2)+"_zmax_"+str(int(10*zmax)).zfill(2)+"_Nlt_"+str(int(Nspec_max))

SN_threshold = 0.1#0.5
wlmin = 6000.
wlmax = 9500.
wlmin_rf = int(wlmin/(1+zmax))
wlmax_rf = int(wlmax/(1+zmin))
d_lambda = 0.2
masterwave = n.arange(wlmin_rf, wlmax_rf, d_lambda) #0.5)

cat = fits.open(os.path.join(os.environ['OBS_REPO'], 'COSMOS', 'DEIMOS_catalog_27MAR2018.fits'))[1].data

ok=(cat['z']>zmin)&(cat['z']<=zmax)
print(len(cat['z'][ok]))
print(n.histogram(cat['Z'][ok],bins=n.arange(0,3.1,0.5)))
nGal = len(ok.nonzero()[0])

#cat['Mask']
#cat['Slt']
#cat['Name']

get_path_to_spectrum = lambda slit, name : glob.glob(os.path.join( os.environ['OBS_REPO'], 'COSMOS', 'spec1d.*-*.' +str(slit).zfill(3) +"."+name+"_acal.fits" ))[0]

def get_spec(path_to_spectrum):
	hdulist = fits.open(path_to_spectrum)
	wavelength = hdulist[1].data['LAMBDA'][0]
	flux = hdulist[1].data['FLUX'][0]
	ivar = hdulist[1].data['IVAR'][0]
	sel = (flux>0)&(ivar>0)
	return wavelength[sel], flux[sel], ivar[sel] #, selection

allflux = n.zeros( ( nGal, len(masterwave)) )
allivar = n.zeros( ( nGal, len(masterwave)) )

print("gets the spectra to construct the matrix")
for index in range(nGal) :# = 0
	try :
		print(index, cat['Mask'][ok][index], cat['Slt'][ok][index], cat['Name'][ok][index])
		path = get_path_to_spectrum(cat['Slt'][ok][index], cat['Name'][ok][index])
		print( path )
		redshift = cat['z'][ok][index]
		wl, fl, iv = get_spec(path)
		p.figure(0,(10,5))
		p.plot(wl,fl)
		p.savefig(path+".png")
		p.clf()
		w_new = masterwave*(1+redshift)
		itp_fl = interp1d( n.hstack((100.,wl[0]-1.,wl,wl[-1]+1,80000.)), n.hstack((0.,0.,wl*fl,0.,0.)) )
		itp_iv = interp1d( n.hstack((100.,wl[0]-1.,wl,wl[-1]+1,80000.)), n.hstack((0.,0.,iv,0.,0.)) )
		f_new = itp_fl(w_new)/w_new
		iv_new = itp_iv(w_new)
		allflux[index] = f_new
		allivar[index] = iv_new
	except(ValueError, IndexError):
		print('error', index, cat['Slt'][ok][index], cat['Name'][ok][index])

out_dir = os.path.join( os.environ['OBS_REPO'], 'archetypes', version, gal_type )
if os.path.isdir(out_dir)==False:
  os.system('mkdir -p '+out_dir)

n.savetxt(os.path.join(out_dir, "allflux_"+name+".txt")   , allflux)
n.savetxt(os.path.join(out_dir, "allivar_"+name+".txt")   , allivar)
n.savetxt(os.path.join(out_dir, "masterwave_"+name+".txt"), masterwave)

#class ObjIds:
    #def __init__(self, plate, mjd, fiberid):
        #self.plate = plate
        #self.mjd = mjd
        #self.fiberid = fiberid

#f = open(os.path.join(out_dir, "specObj_"+name+".pkl"), 'wb')
#obj = ObjIds(cat['PLATE'][ok], cat['MJD'][ok], cat['FIBERID'][ok])
#pickle.dump(obj, f)
#f.close()