import sys
import re
from csv import DictWriter
from PyQt5.QtWidgets import QWidget, QLabel, QApplication
from PyQt5 import uic
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWidgets import QLabel
from PyQt5.QtWidgets import QVBoxLayout
from PyQt5.QtWidgets import QWidget
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import QCoreApplication, Qt
from PyQt5.QtGui import QIcon, QColor, QPixmap, QFont, QIntValidator
from PyQt5.QtWidgets import QApplication, QWidget, QMainWindow, QPushButton, QAction, QMessageBox, QLabel, QVBoxLayout
from PyQt5.QtWidgets import QCalendarWidget, QFontDialog, QColorDialog, QTextEdit, QFileDialog
from PyQt5.QtWidgets import QCheckBox, QProgressBar, QComboBox, QLabel, QStyleFactory, QLineEdit, QInputDialog

import data_load_proxy,missing_values,features,model

# screen variable for the screen widget
screens = ''

#### VARIABLES THAT ARE NEEDED FOR FRAMEWORK ####

## STATION_ID

station_id_dict = {'R.K. Puram':33,'Mandir Marg':21,'Anand Vihar':2,'Punjabi Bagh':40 }
STATION_ID = 33 ## THIS VALUE WILL GET SET IN def station_select() in Screen1

## TIME INTERVAL ( 15 minutes,30 minutes,1 hour,4 hour,8 hour,12 hour,24 hour)

INTERVAL = '30 minutes'

## MISSING VALUES:

#1. TEMPERATURE
TEMP_NA = 0
temp_na_dict = {'Wunderground data':0,'3 Nearest Stations':1, "Don't fill (Drop NA)":2}

#2. PM10 & PM2.5
PM_NA = 0
pm_na_dict = {'Previous Avg Values':0,'3 Nearest Stations':1, "Don't fill (Drop NA)":2}

#3. OTHER POLLUTANTS
POLL_NA = 0
poll_na_dict = {'Previous Avg Values':0,'3 Nearest Stations':1, "Don't fill (Drop NA)":2}

#4. METERIOLOGICAL PARAMETERS
MET_NA = 0
met_na_dict = {'Use Wunderground Values':0,'Previous Avg Values':1,'3 Nearest Stations':2, "Don't fill (Drop NA)":3}

## FEATURE ENGINEERING:

features_dict = {'PM10_yest':0,'PM10_week':0,'PM10_month':0,'PM2_yest':0,'PM2_week':0,'PM2_month':0, 'PM10_last_yest':0,'PM10_last_week':0,'PM10_last_month':0,'PM2_last_yest':0,'PM2_last_week':0,'PM2_last_month':0,'seasons':0,'weekends':0,'all_pollutants':0,'all_met':0}

## LAG:
LAG_TARGET = 0
LAG_POLL = 0

## MODEL:
target_dict = {'PM 2.5':0,'PM 10':1}
TARGET = 0

standard_dict = {'No Standardization':0, 'Standard Scaling':1,'Min Max Scaling':2}
STANDARDIZATION = 0

SPLIT_PERC = 20

model_dict = {'Linear Regression':0,'Ridge Regression':1,'Lasso Regression':2,
     'Elastic Net Regression':3,'SGD Regression':4,
      'Decision Tree':5,'Random Forest':6,'Ada Boost':7,
     'Bagging':8,'Gradient Boost':9,'XGBoost':10,'Extra Trees':11,'SVR':12,
     'HistGradientBoosting':13,'LGBMRegressor':14,'CatBoostRegressor':15}
MODEL = 0

metrics_dict = {'RMSE':1,'R2 score':1,'MAE':1,'MAPE':1}

df = []

result = ''

## Screen1: Station select and Interval setting
class Screen1(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        helloMsg = QLabel('
Selection 1: Choose Station and Interval
', self)
        helloMsg.setGeometry(320,50,600,100)

        self.station = QLabel('
Choose Station
', self)
        station_choice = QComboBox(self)
        station_choice.addItem('R.K. Puram')
        station_choice.addItem('Mandir Marg')
        station_choice.addItem('Anand Vihar')
        station_choice.addItem('Punjabi Bagh')
        station_choice.setGeometry(460,290,200,50)
        self.station.setGeometry(485,250,200,50)
        station_choice.activated[str].connect(self.station_select)

        show_process = QLabel('(Pressing on Continue will start the processing)', self)
        show_process.setGeometry(410,530,500,100)

        self.interval = QLabel('
Choose Time Interval
', self)
        interval_choice = QComboBox(self)
        interval_choice.addItem('15 minutes')
        interval_choice.addItem('30 minutes')
        interval_choice.addItem('1 hour')
        interval_choice.addItem('4 hours')
        interval_choice.addItem('8 hours')
        interval_choice.addItem('12 hours')
        interval_choice.addItem('24 hours')
        interval_choice.setGeometry(460,390,200,50)
        self.interval.setGeometry(460,350,200,50)
        interval_choice.activated[str].connect(self.interval_select)

        btn_next_screen = QPushButton('Continue', self)
        btn_next_screen.resize(500, 80)
        btn_next_screen.move(300, 500)
        btn_next_screen.clicked.connect(self.nextScreen)
        #self.show()

    def nextScreen(self):
        ## Go to next screen on clicking Continue Button

        s2 = Screen2()
        screens.addWidget(s2)
        screens.setCurrentIndex(screens.currentIndex()+1)

    def station_select(self,text):
        global  STATION_ID
        STATION_ID = station_id_dict[text]

    def interval_select(self,text):
        global INTERVAL
        INTERVAL = text
        #print(INTERVAL)

## SCREEN 1 is done here

## SCREEN 2 has display labels for showing what all processing has been done
class Screen2(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        print('In Screen 2')
        processMsg = QLabel('
File Processed
', self)
        processMsg.setGeometry(400,50,300,100)

        stats = data_load_proxy.start_load(STATION_ID,INTERVAL)
        #stats = {'ok':True, 'process':'Done a lot!'}
        statsMsg = QLabel(stats['process'],self)
        statsMsg.setGeometry(200,150,600,300)

        if stats['ok'] == True:

            btn_next_screen = QPushButton('Continue', self)
            btn_next_screen.resize(500, 80)
            btn_next_screen.move(300, 500)
            btn_next_screen.clicked.connect(self.nextScreen)

        else:
            errMsg = QLabel('
Some error occured. Please try another configuration
',self)
            errMsg.setGeometry(400,500,300,100)

        btn_back = QPushButton('Back', self)
        btn_back.resize(80, 50)
        btn_back.move(50, 20)
        btn_back.clicked.connect(self.backScreen)

    def backScreen(self):

        screens.setCurrentIndex(screens.currentIndex()-1)

    def nextScreen(self):

        s3 = Screen3()
        screens.addWidget(s3)
        screens.setCurrentIndex(screens.currentIndex()+1)

## Filling Missing Values
class Screen3(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        print('In Screen 3')
        helloMsg = QLabel('
Selection 2: How to fill missing values?
', self)
        helloMsg.setGeometry(340,50,500,100)

        self.temp = QLabel('
Fill Temperature column using
', self)
        temp_choice = QComboBox(self)
        temp_choice.addItem('Wunderground data')
        temp_choice.addItem('3 Nearest Stations')
        temp_choice.addItem("Don't fill (Drop NA)")
        temp_choice.setGeometry(140,240,200,50)
        self.temp.setGeometry(100,200,300,50)
        temp_choice.activated[str].connect(self.temp_select)

        self.pm = QLabel('
Fill PM2.5 & PM10 column using
', self)
        pm_choice = QComboBox(self)
        pm_choice.addItem('Previous Avg Values')
        pm_choice.addItem('3 Nearest Stations')
        pm_choice.addItem("Don't fill (Drop NA)")
        pm_choice.setGeometry(690,240,200,50)
        self.pm.setGeometry(650,200,300,50)
        pm_choice.activated[str].connect(self.pm_select)

        self.pollutants = QLabel('
Fill other Pollutants columns using
', self)
        pollutants_choice = QComboBox(self)
        pollutants_choice.addItem('Previous Avg Values')
        pollutants_choice.addItem('3 Nearest Stations')
        pollutants_choice.addItem("Don't fill (Drop NA)")
        pollutants_choice.setGeometry(140,410,200,50)
        self.pollutants.setGeometry(80,370,400,50)
        pollutants_choice.activated[str].connect(self.pollutants_select)

        self.met = QLabel('
Fill other Meteriological columns using
', self)
        met_choice = QComboBox(self)
        met_choice.addItem('Use Wunderground Values')
        met_choice.addItem('Previous Avg Values')
        met_choice.addItem('3 Nearest Stations')
        met_choice.addItem("Don't fill (Drop NA)")
        met_choice.setGeometry(690,410,200,50)
        self.met.setGeometry(650,370,400,50)
        met_choice.activated[str].connect(self.met_select)

        btn_ref_file = QPushButton('CONTINUE', self)
        btn_ref_file.resize(500, 80)
        btn_ref_file.move(300, 500)
        btn_ref_file.clicked.connect(self.nextScreen)
        #self.show()

        btn_back = QPushButton('Back', self)
        btn_back.resize(80, 50)
        btn_back.move(50, 20)
        btn_back.clicked.connect(self.backScreen)

    def backScreen(self):

        screens.setCurrentIndex(screens.currentIndex()-1)

    def nextScreen(self):

        s4 = Screen4()
        screens.addWidget(s4)
        screens.setCurrentIndex(screens.currentIndex()+1)

    def temp_select(self,text):
        global TEMP_NA
        TEMP_NA = temp_na_dict[text]

    def pm_select(self,text):
        global PM_NA
        PM_NA = pm_na_dict[text]

    def pollutants_select(self,text):
        global POLL_NA
        POLL_NA = poll_na_dict[text]

    def met_select(self,text):
        global MET_NA
        MET_NA = met_na_dict[text]

## Feature Engineering: What all features are needed in Dataframe
class Screen4(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        print('In Screen 4')
        processMsg = QLabel('
Selection 3: Feature Engineering
', self)
        processMsg.setGeometry(350,50,450,100)

        ## CHECKBOXES
## 1st column

        checkBox_pm_10_yest = QCheckBox(self)
        checkBox_pm_10_yest.setText('Average PM10 yesterday')
        checkBox_pm_10_yest.setGeometry(120, 180, 200, 20)
        checkBox_pm_10_yest.stateChanged.connect(self.checked_pm_10_yest)

        checkBox_pm_10_week = QCheckBox(self)
        checkBox_pm_10_week.setText('Average PM10 this week')
        checkBox_pm_10_week.setGeometry(120, 230, 200, 20)
        checkBox_pm_10_week.stateChanged.connect(self.checked_pm_10_week)

        checkBox_pm_10_month = QCheckBox(self)
        checkBox_pm_10_month.setText('Average PM10 this month')
        checkBox_pm_10_month.setGeometry(120, 280, 200, 20)
        checkBox_pm_10_month.stateChanged.connect(self.checked_pm_10_month)

        checkBox_pm_2_yest = QCheckBox(self)
        checkBox_pm_2_yest.setText('Average PM2.5 yesterday')
        checkBox_pm_2_yest.setGeometry(120, 330, 200, 20)
        checkBox_pm_2_yest.stateChanged.connect(self.checked_pm_2_yest)

        checkBox_pm_2_week = QCheckBox(self)
        checkBox_pm_2_week.setText('Average PM2.5 this week')
        checkBox_pm_2_week.setGeometry(120, 380, 200, 20)
        checkBox_pm_2_week.stateChanged.connect(self.checked_pm_2_week)

        checkBox_pm_2_month = QCheckBox(self)
        checkBox_pm_2_month.setText('Average PM2.5 this month')
        checkBox_pm_2_month.setGeometry(120, 430, 200, 20)
        checkBox_pm_2_month.stateChanged.connect(self.checked_pm_2_month)

## 2nd column
        checkBox_pm_10_last_yest = QCheckBox(self)
        checkBox_pm_10_last_yest.setText('Average PM10 last year this day')
        checkBox_pm_10_last_yest.setGeometry(420, 180, 300, 20)
        checkBox_pm_10_last_yest.stateChanged.connect(self.checked_pm_10_last_yest)

        checkBox_pm_10_last_week = QCheckBox(self)
        checkBox_pm_10_last_week.setText('Average PM10 last year this week')
        checkBox_pm_10_last_week.setGeometry(420, 230, 300, 20)
        checkBox_pm_10_last_week.stateChanged.connect(self.checked_pm_10_last_week)

        checkBox_pm_10_last_month = QCheckBox(self)
        checkBox_pm_10_last_month.setText('Average PM10 last year this month')
        checkBox_pm_10_last_month.setGeometry(420, 280, 300, 20)
        checkBox_pm_10_last_month.stateChanged.connect(self.checked_pm_10_last_month)

        checkBox_pm_2_last_yest = QCheckBox(self)
        checkBox_pm_2_last_yest.setText('Average PM2.5 last year this day')
        checkBox_pm_2_last_yest.setGeometry(420, 330, 300, 20)
        checkBox_pm_2_last_yest.stateChanged.connect(self.checked_pm_2_last_yest)

        checkBox_pm_2_last_week = QCheckBox(self)
        checkBox_pm_2_last_week.setText('Average PM2.5 last year this week')
        checkBox_pm_2_last_week.setGeometry(420, 380, 300, 20)
        checkBox_pm_2_last_week.stateChanged.connect(self.checked_pm_2_last_week)

        checkBox_pm_2_last_month = QCheckBox(self)
        checkBox_pm_2_last_month.setText('Average PM2.5 last year this month')
        checkBox_pm_2_last_month.setGeometry(420, 430, 300, 20)
        checkBox_pm_2_last_month.stateChanged.connect(self.checked_pm_2_last_month)

## 3rd column

        checkBox_poll = QCheckBox(self)
        checkBox_poll.setText('Include Pollutants')
        checkBox_poll.setGeometry(720, 180, 200, 20)
        checkBox_poll.stateChanged.connect(self.checked_poll)

        checkBox_met = QCheckBox(self)
        checkBox_met.setText('Include Meteriological Factors')
        checkBox_met.setGeometry(720, 265, 250, 20)
        checkBox_met.stateChanged.connect(self.checked_met)

        checkBox_seasons = QCheckBox(self)
        checkBox_seasons.setText('Seasons(Winter/Summer/etc)')
        checkBox_seasons.setGeometry(720, 350, 250, 20)
        checkBox_seasons.stateChanged.connect(self.checked_seasons)

        checkBox_weekend = QCheckBox(self)
        checkBox_weekend.setText('Weekends(Weekend/Weekday)')
        checkBox_weekend.setGeometry(720, 430, 250, 20)
        checkBox_weekend.stateChanged.connect(self.checked_weekend)

        btn_next_screen = QPushButton('Continue', self)
        btn_next_screen.resize(500, 80)
        btn_next_screen.move(300, 500)
        btn_next_screen.clicked.connect(self.nextScreen)

        btn_back = QPushButton('Back', self)
        btn_back.resize(80, 50)
        btn_back.move(50, 20)
        btn_back.clicked.connect(self.backScreen)

    def backScreen(self):

        screens.setCurrentIndex(screens.currentIndex()-1)

# 1st column's functions
    def checked_pm_10_yest(self, checked):
        global features_dict
        if checked:
            features_dict['PM10_yest']= 1
    def checked_pm_10_week(self, checked):
        global features_dict
        if checked:
            features_dict['PM10_week']= 1

    def checked_pm_10_month(self, checked):
        global features_dict
        if checked:
            features_dict['PM10_month']= 1

    def checked_pm_2_yest(self, checked):
        global features_dict
        if checked:
            features_dict['PM2_yest']= 1
    def checked_pm_2_week(self, checked):
        global features_dict
        if checked:
            features_dict['PM2_week']= 1

    def checked_pm_2_month(self, checked):
        global features_dict
        if checked:
            features_dict['PM2_month']= 1

# 2nd column's functions:
    def checked_pm_10_last_yest(self, checked):
        global features_dict
        if checked:
            features_dict['PM10_last_yest']= 1
    def checked_pm_10_last_week(self, checked):
        global features_dict
        if checked:
            features_dict['PM10_last_week']= 1

    def checked_pm_10_last_month(self, checked):
        global features_dict
        if checked:
            features_dict['PM10_last_month']= 1

    def checked_pm_2_last_yest(self, checked):
        global features_dict
        if checked:
            features_dict['PM2_last_yest']= 1
    def checked_pm_2_last_week(self, checked):
        global features_dict
        if checked:
            features_dict['PM2_last_week']= 1

    def checked_pm_2_last_month(self, checked):
        global features_dict
        if checked:
            features_dict['PM2_last_month']= 1

# 3rd column's function
    def checked_poll(self, checked):
        global features_dict
        if checked:
            features_dict['all_pollutants']= 1

    def checked_met(self, checked):
        global features_dict
        if checked:
            features_dict['all_met']= 1
    def checked_seasons(self, checked):
        global features_dict
        if checked:
            features_dict['seasons']= 1

    def checked_weekend(self, checked):
        global features_dict
        if checked:
            features_dict['weekends']= 1

    def nextScreen(self):

        s5 = Screen5()
        screens.addWidget(s5)
        screens.setCurrentIndex(screens.currentIndex()+1)

## This screen is again just for displaying status of Filling Missing values
class Screen5(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        print('In Screen 5')
        processMsg = QLabel('
File Processed
', self)
        processMsg.setGeometry(400,50,350,100)

        global df

        df, exists = data_load_proxy.check_file(STATION_ID,INTERVAL,TEMP_NA,PM_NA,POLL_NA,MET_NA,features_dict)

        if exists:
            statsMsg = QLabel('Fetching already available file',self)
            statsMsg.setGeometry(400,70,500,500)
            btn_next_screen = QPushButton('Continue', self)
            btn_next_screen.resize(500, 80)
            btn_next_screen.move(300, 500)
            btn_next_screen.clicked.connect(self.nextScreen)
        else:
            stats1 = missing_values.fill_values(TEMP_NA,PM_NA,POLL_NA,MET_NA)
            #stats = {'ok':True, 'process':'Done a lot!'}
            statsMsg = QLabel(stats1['process'],self)
            statsMsg.setGeometry(400,70,500,500)

            df, stats2 = features.get_data(STATION_ID, features_dict)
            stats2Msg = QLabel(stats2['process'],self)
            stats2Msg.setGeometry(400,180,500,500)

            if stats1['ok'] == True and stats2['ok']==True:

                btn_next_screen = QPushButton('Continue', self)
                btn_next_screen.resize(500, 80)
                btn_next_screen.move(300, 500)
                btn_next_screen.clicked.connect(self.nextScreen)

            else:
                errMsg = QLabel('
Some error occured. Please try another configuration
',self)
                errMsg.setGeometry(400,500,500,100)

        btn_back = QPushButton('Back', self)
        btn_back.resize(80, 50)
        btn_back.move(50, 20)
        btn_back.clicked.connect(self.backScreen)

    def backScreen(self):

        screens.setCurrentIndex(screens.currentIndex()-1)

    def nextScreen(self):

        s6 = Screen6()
        screens.addWidget(s6)
        screens.setCurrentIndex(screens.currentIndex()+1)

## Lag values for Pollutants(if chosen for PARAMS) and for TARGET variable
class Screen6(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        print('In Screen 6')
        LagMsg = QLabel('
Selection 4:Choose Lag Value
', self)
        LagMsg.setGeometry(350,50,450,100)

        if features_dict['all_pollutants'] == 1:
            polLagMsg = QLabel('
Lag Value for Pollution Parameters
', self)
            polLagMsg.setGeometry(400,250,300,50)
            e1 = QLineEdit(self)
            e1.setValidator(QIntValidator())
            e1.setMaxLength(2)
            e1.setAlignment(Qt.AlignRight)
            e1.setFont(QFont("Arial",18))
            e1.setGeometry(700,260,50,30)
            e1.textChanged.connect(self.polLag)

        tarLagMsg = QLabel('
Lag Value for Target Parameter
', self)
        tarLagMsg.setGeometry(400,400,300,50)
        e2 = QLineEdit(self)
        e2.setValidator(QIntValidator())
        e2.setMaxLength(2)
        e2.setAlignment(Qt.AlignRight)
        e2.setFont(QFont("Arial",18))
        e2.setGeometry(700,410,50,30)
        e2.textChanged.connect(self.tarLag)

        btn_next_screen = QPushButton('Continue', self)
        btn_next_screen.resize(500, 80)
        btn_next_screen.move(300, 500)
        btn_next_screen.clicked.connect(self.nextScreen)

        btn_back = QPushButton('Back', self)
        btn_back.resize(80, 50)
        btn_back.move(50, 20)
        btn_back.clicked.connect(self.backScreen)

    def backScreen(self):

        screens.setCurrentIndex(screens.currentIndex()-1)

    def polLag(self,text):
        global LAG_POLL
        try:
            LAG_POLL = int(text)
        except:
            LAG_POLL = 0

    def tarLag(self,text):
        global LAG_TARGET

        try:
            LAG_TARGET = int(text)
        except:
            LAG_TARGET = 0

    def nextScreen(self):

        s7 = Screen7()
        screens.addWidget(s7)
        screens.setCurrentIndex(screens.currentIndex()+1)

## Model selection
class Screen7(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        print('In Screen 7')
        LagMsg = QLabel('
Selection 5:Model Training
', self)
        LagMsg.setGeometry(350,50,350,100)

        self.target = QLabel('
Target Variable
', self)
        target_choice = QComboBox(self)
        target_choice.addItem('PM 2.5')
        target_choice.addItem('PM 10')
        target_choice.setGeometry(140,240,200,50)
        self.target.setGeometry(150,200,300,50)
        target_choice.activated[str].connect(self.target_select)

        self.standard = QLabel('
Standardization Method
', self)
        standard_choice = QComboBox(self)
        standard_choice.addItem('No Standardization')
        standard_choice.addItem('Standard Scaling')
        standard_choice.addItem("Min Max Scaling")
        standard_choice.setGeometry(690,240,200,50)
        self.standard.setGeometry(675,200,300,50)
        standard_choice.activated[str].connect(self.standard_select)

        self.split = QLabel('
Test-Train Split
', self)
        split_choice = QComboBox(self)
        split_choice.addItem('20%')
        split_choice.addItem('25%')
        split_choice.addItem("33%")
        split_choice.setGeometry(140,330,200,50)
        self.split.setGeometry(140,300,400,50)
        split_choice.activated[str].connect(self.split_select)

        self.model = QLabel('
ML Model
', self)
        model_choice = QComboBox(self)
        model_choice.addItem('Linear Regression')
        model_choice.addItem('Ridge Regression')
        model_choice.addItem('Lasso Regression')
        model_choice.addItem('Elastic Net Regression')
        model_choice.addItem('SGD Regression')
        model_choice.addItem("Decision Tree")
        model_choice.addItem('Random Forest')
        model_choice.addItem("Ada Boost")
        model_choice.addItem('Bagging')
        model_choice.addItem('Gradient Boost')
        model_choice.addItem('XGBoost')
        model_choice.addItem('Extra Trees')
        model_choice.addItem('SVR')
        model_choice.addItem('HistGradientBoosting')
        model_choice.addItem('LGBMRegressor')
        model_choice.addItem('CatBoostRegressor')

        model_choice.setGeometry(690,330,200,50)
        self.model.setGeometry(720,300,400,50)
        model_choice.activated[str].connect(self.model_select)

        metricMsg = QLabel('Metrics:', self)
        metricMsg.setGeometry(120, 400, 200, 20)

        checkBox_metrics_a = QCheckBox(self)
        checkBox_metrics_a.setText('RMSE')
        checkBox_metrics_a.setGeometry(200, 400, 200, 20)
        checkBox_metrics_a.stateChanged.connect(self.metrics_a)
        checkBox_metrics_a.setChecked(True)

        checkBox_metrics_b = QCheckBox(self)
        checkBox_metrics_b.setText('R2 Score')
        checkBox_metrics_b.setGeometry(350, 400, 200, 20)
        checkBox_metrics_b.stateChanged.connect(self.metrics_b)
        checkBox_metrics_b.setChecked(True)

        checkBox_metrics_c = QCheckBox(self)
        checkBox_metrics_c.setText('MAE')
        checkBox_metrics_c.setGeometry(500, 400, 200, 20)
        checkBox_metrics_c.stateChanged.connect(self.metrics_c)
        checkBox_metrics_c.setChecked(True)

        checkBox_metrics_d = QCheckBox(self)
        checkBox_metrics_d.setText('MAPE')
        checkBox_metrics_d.setGeometry(650, 400, 200, 20)
        checkBox_metrics_d.stateChanged.connect(self.metrics_d)
        checkBox_metrics_d.setChecked(True)

        btn_ref_file = QPushButton('START PROCESS', self)
        btn_ref_file.resize(500, 80)
        btn_ref_file.move(300, 500)
        btn_ref_file.clicked.connect(self.nextScreen)
        #self.show()

        btn_back = QPushButton('Back', self)
        btn_back.resize(80, 50)
        btn_back.move(50, 20)
        btn_back.clicked.connect(self.backScreen)

    def backScreen(self):
        screens.setCurrentIndex(screens.currentIndex()-1)

    def nextScreen(self):
        s8 = Screen8()
        screens.addWidget(s8)
        screens.setCurrentIndex(screens.currentIndex()+1)

    def target_select(self,text):
        global TARGET
        TARGET = target_dict[text]

    def standard_select(self,text):
        global STANDARDIZATION
        STANDARDIZATION = standard_dict[text]

    def split_select(self,text):
        global SPLIT_PERC
        SPLIT_PERC = int(text.strip('%'))

    def model_select(self,text):
        global MODEL
        MODEL = model_dict[text]

    def metrics_a(self, checked):
        global metrics_dict
        if not checked:
            metrics_dict['RMSE']= 0

    def metrics_b(self, checked):
        global metrics_dict
        if not checked:
            metrics_dict['R2 Score']= 0

    def metrics_c(self, checked):
        global metrics_dict
        if not checked:
            metrics_dict['MAE']= 0

    def metrics_d(self, checked):
        global metrics_dict
        if not checked:
            metrics_dict['MAPE']= 0

## Displaying Output
class Screen8(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):
        print('In Screen 8')
        OutMsg = QLabel('
Working on the data...
', self)
        OutMsg.setGeometry(350,50,450,100)
        global result

        ## Until here we have called data_load
        ## and missing_values

        ## Now we start from features

        try:
            features.final_features(df,features_dict,LAG_POLL,TARGET,LAG_TARGET)
            stats = model.get_data(SPLIT_PERC,STANDARDIZATION,MODEL)
            result = stats['result']
        except Exception as e:
            print(e)
            stats = {'ok':False, 'result':'Error!'}

        ## Features will give X,Y. We create train & test data from it
        #model.get_data(SPLIT_PERC,STANDARDIZATION,MODEL)

        ## Once we have model and X_test & Y_test, we can test it
        #stats = metrics.test(metrics_dict)

        ## Display the ouput
        resultsMsg = QLabel(stats['result'],self)
        resultsMsg.setGeometry(220,100,900,700)

        if stats['ok']==True:
            btn_ref_file = QPushButton('LOG RESULTS', self)
            btn_ref_file.resize(500, 40)
            btn_ref_file.move(300, 550)
            btn_ref_file.clicked.connect(self.log_results)

        btn1_ref_file = QPushButton('QUIT', self)
        btn1_ref_file.resize(500, 40)
        btn1_ref_file.move(300, 600)
        btn1_ref_file.clicked.connect(QtWidgets.qApp.quit)

    def log_results(self):
        field_names = ['STATION_ID','INTERVAL','MISSING TEMPERATURE FILLED USING',
               'MISSING PM FILLED USING','MISSING POLLUTANTS FILLED USING','MISSING MET PARAMS FILLED USING',
               'EXTRA FEATURES ADDED','LAG FOR POLLUTANTS','LAG FOR TARGET','TARGET','STANDARDIZATION','SPLIT_PERC','MODEL',
               'MSE','MAE','RMSE','MAPE','R2']

        li = []
        for key, val in features_dict.items():
                 if val == 1:
                     li.append(key)

        res = re.findall(r'[0-9]*[.,][0-9]*',result)

        # Dictionary
        dict_items={'STATION_ID':STATION_ID,'INTERVAL':INTERVAL,'MISSING TEMPERATURE FILLED USING': list(temp_na_dict.keys())[list(temp_na_dict.values()).index(TEMP_NA)],
               'MISSING PM FILLED USING': list(pm_na_dict.keys())[list(pm_na_dict.values()).index(PM_NA)],'MISSING POLLUTANTS FILLED USING':list(poll_na_dict.keys())[list(poll_na_dict.values()).index(POLL_NA)],
               'MISSING MET PARAMS FILLED USING': list(met_na_dict.keys())[list(met_na_dict.values()).index(MET_NA)],
               'EXTRA FEATURES ADDED':li,'LAG FOR POLLUTANTS':LAG_POLL,'LAG FOR TARGET':LAG_TARGET,'TARGET':list(target_dict.keys())[list(target_dict.values()).index(TARGET)],
               'STANDARDIZATION':list(standard_dict.keys())[list(standard_dict.values()).index(STANDARDIZATION)],'SPLIT_PERC':SPLIT_PERC,'MODEL':list(model_dict.keys())[list(model_dict.values()).index(MODEL)],
               'MSE':res[0],'MAE':res[1],'RMSE':res[2],'MAPE':res[3],'R2':res[4]}

        with open('results.csv', 'a') as f_object:

            # Pass the file object and a list
            # of column names to DictWriter()
            # You will get a object of DictWriter
            dictwriter_object = DictWriter(f_object, fieldnames=field_names)

            #Pass the dictionary as an argument to the Writerow()
            dictwriter_object.writerow(dict_items)

            #Close the file object
            f_object.close()

            QtWidgets.qApp.quit()

    # def quit(self):

    #     self.close()

## Run function for running the app
## This initialized the 1st screen.
## There onwards, every screen is initialized by a screen before it
def run():
    app = QApplication(sys.argv)

    global screens
    screens = QtWidgets.QStackedWidget()
    s1 = Screen1()
    screens.addWidget(s1)

    screens.setGeometry(50, 50, 1080, 800)

    screens.show()

    sys.exit(app.exec_())

if __name__ == "__main__":
    run()


import numpy as np
import pandas as pd
import random
import math
from scipy.stats import pearsonr
import statsmodels.api as sm

# matrix adjustment

def nothing(matrix):

    pass

    return matrix

# exponent weighed adjustment
# f(i,j) = std(i) * std(j) * r(i,j)
# when calculating std and r different half-life are setted
# fr = [factor returns, moments] and sorted by moments
def exponentWeight(fr, halfLifeStd = 252, halfLifeR = 84):

    w = 0.5**(1 / halfLifeStd) * np.matrix((np.linspace(1,np,shape(fr)[1],np.shape(fr)[1])))
    w = np.diag(w)
    frStd = np.dot(fr, w)
    std = np.std(frStd, axis = 1)

    w = 0.5**(1 / halfLifeR) * np.matrix((np.linspace(1,np,shape(fr)[1],np.shape(fr)[1])))
    w = np.diag(w)
    frR = np.dot(fr, w)
    r = np.corrcoef(frR)

    newMatrix = np.dot(std, std.T) * r

    return newMatrix

# fr = [factor returns, moments] and sorted by moments
# cov0 = Sigma(fri*fri.T)/t (1~t)
# cov0 can be inplaced by matrix that has already been adjusted by exponentWeight()
# cov = cov0 + Sigma(wi*(covi + covi.T)) (1~q) , wi = 1 - i/(1+q), covi = Sigma(frj*frj+i.T)/t (1~t-i)
# factor return frt  of moment 't' is influenced by moments of 't-1','t-2','t-3' ...... 't-q'
def neweyWestRaw(fr, q, halfLife = None):

    cov0 = np.zeros((np.shape(fr)[0],np.shape(fr)[0]))
    t = np.shape(fr)[1]
    for i in range (t):
        cov0 = cov0 + np.dot(fr[:,i],fr[:,i].T)
    cov0 = cov0/t

    # if use exponent weights or not
    if halfLife != None:
        w = 0.5**(1 / halfLife) * np.matrix((np.linspace(1,np,shape(fr)[1],np.shape(fr)[1])))
        w = np.diag(w)
        fr = np.dot(fr, w)

    sigmaCov = np.zeros((np.shape(fr)[0],np.shape(fr)[0]))
    t = np.shape(fr)[1]
    for i in range (1,q+1):
        covi = np.zeros((np.shape(fr)[0],np.shape(fr)[0]))
        for j in range (t-i):
            covi = covi + np.dot(fr[:,j], fr[:,i+j].T)
        covi = covi/t
        wi = 1 - i / (1+q)
        sigmaCov = wi * (covi + covi.T) + sigmaCov
    cov = cov0 + cov

    # change cov of daily data into monthly data
    # 22 trade days per month. so monthly return is supposed to be 22 times of daily return
    newMatrix = 22*cov

    return newMatrix

# this newey west adjustment can adjust std matrix and r matrix respectively
def neweyWest(exponentMat, fr, qStd = 5, qR = 2, halfLifeStd = 252, halfLifeR = 84):

    factorNum = np.shape(fr)[0]
    t = np.shape(fr)[1]
    w = 0.5**(1 / halfLifeR) * np.matrix((np.linspace(1,np,shape(fr)[1],np.shape(fr)[1])))
    w = np.diag(w)
    frR = np.dot(fr, w)
    r0 = np.corrcoef(frR)
    sigmaR = np.zeros((factorNum,factorNum))
    for i in range (1,qR+1):
        fr1 = frR[:,0:t-i]
        fr2 = frR[:,i:t]
        ri = np.zeros((factorNum,factorNum))
        for j in range(factorNum):
            for k in range(j,factorNum):
                ri[j,k] = pearsonr(fr1[j,:], fr2[k,:])[0]
                ri[k,j] = pearsonr(fr1[k,:], fr2[j,:])[0]
        wi = 1 - i / (1+qR)
        sigmaR = wi * (ri + ri.T) + sigmaR
    r = r0 + r

    covRaw = neweyWestRaw(fr, qStd, halfLifeStd)
    cov = covRaw.diagonal().T
    std = np.sqrt(cov)

    '''
    w = 0.5**(1 / halfLifeStd) * np.matrix((np.linspace(1,np,shape(fr)[1],np.shape(fr)[1])))
    w = np.diag(w)
    frCov = np.dot(fr, w)
    sigmaCov = np.zeros((factorNum,factorNum))
    for i in range (1,qCov+1):
        fr1 = frCov[:,0:t-i]
        fr2 = frCov[:,i:t]
        covi = np.zeros((factorNum,factorNum))
        for j in range(factorNum):
            for k in range(j,factorNum):
                covi[j,k] = np.std(fr1[j,:])*np.std(fr2[k,:])*r[j,k]
                covi[k,j] = np.std(fr1[k,:])*np.std(fr2[j,:])*r[k,j]
        covi = covi*(t - i)/t
        wi = 1 - i / (1+qCov)
        sigmaCov = wi * (covi + covi.T) + sigmaCov

    newMatrix = exponent + sigmaCov
    '''
    newMatrix = np.dot(std * std.T) * r

    return newMatrix

# adjustment based on eigen values is based on neweyWest adjustment
# d0 = u0.T*neweyWestMat*u0  u0 = eigVector
# rm = u0 * bm   fm = cov(rm, rm) dm = um.T*fm*um dmReal = um.T*neweyWest*um
# lamda = sqrt(Sigma(dmReal/dm)/m) (m = 1,2,3 ...... M)
# gama = alpha*(lamda-1)+1 d0Real = gama**2 * d0
# eigenMat = u0*d0Real*u0.T
def eigen(neweyWestMat, fr, M = 1000, alpha = 1.2):

    eigValues, eigVectors = np.linalg.eig(neweyWestMat)
    u0 = eigVectors
    d0 = u0.T * neweyWestMat * u0

    # try M times
    sigma = np.zeros((M,1))
    for m in range(M):
        bm = np.zeros((np.shape(d0)[0],1))
        for i in range(np.shape(d0)[0]):
            bm[i][0] = random.normalvariate(0, sqrt(d0[i][i]))
        rm = u0 * bm
        fm = np.cov(rm)
        em, um = np.linalg.eig(fm)
        dm = um.T * fm * um
        dmReal = um.T * neweyWest * um
        sigma = sigma + dmReal / dm

    lamda = np.sqrt(sigma/M)
    gama = alpha * (lamda - 1) + 1
    d0Real = gama**2 * d0
    newMatrix = u0 * d0Real * u0.T

    return newMatrix

# adjustment based on fluctuation ratio is based on eigen adjustment
# the number of factors is K
# bft = sqrt((frkt/flrkt)**2 /K)  fr is factor return is. flr is fluctuation.
# lamda = sqrt(sigma(bft**2, wt))
# fvar = lamda**2 * feigen
# when it comes to heterogeneity return of stocks, adjustment is based on the output of bayes adjustment
def fluctuation(eigenMat, fr, flr):

    sigma = np.zeros((np.shape(fr)[1],1))
    for i in range(np.shape(sigma)[0]):
        sigma[i] = (fr[:,i]/flr[:,i])**2
    bf = np.sqrt(sigma/np.shape(fr)[0])
    w = np.zeros((np.shape(fr)[0],))
    for i in range(np.shape(w)[0]):
        w[i] = 0.5**((np.shape(w)[0]-i-1)/np.shape(w)[0])
    w = w / np.sum(w)
    i = 0
    lamda = 0
    for bft in bf:
        lamda = lamda + bft**2*w[i]
        i += 1
    newMatrix = lamda * eigenMat

    return newMatrix

# structure model adjustment
# sigma = gama * sigmaNeweyWest + (1-gama) * sigmaStruct    sigma is the result,sigmaNeweyWest is std after neweyWest adjustment,
# gama = (min(1, max(0, h/120-0.5))) * (min(1, max(0, exp(1-zu))))    gama is a parameter for judging the quanlity of a resid, the bigger the better  h: h = 252
# sigmaStruct(gama != 1) = E0 * exp(Sigma(Xnk * bk))     Xnk: exposure value  bk: weight of the No.k factor E0: a const
# use ln(sigmaStruct(gama == 1)) = Sigma(Xnk*bk) + en doing linear regression then work out bk and sigmaStruct(gama == 1) == sigmaNeweyWest(gama == 1)
# zu = abs(sigma / sigmaReference -1)    zu is a paramter which has the same function as what gama has the less the better  sigma: covariance of the returns without any adjustment
# sigmaReference = 1/1.35 * (Q3 - Q1)
# Q3 = 3/4*rn Q1 = 1/4 * rn    rn: the return of No.n stock
# all inputs should in form of matrix
def structure(neweyWestMat, r, exposure, omiga, h = 252, E0 = 1.05):

    n = len(r)

    Q3 = 3/4 * r
    Q1 = 1/4 * r
    sigmaReference = 1/1.35 * (Q3 -Q1)
    zu = np.abs(np.diag(omiga).reshape(n,1)/sigmaReference - 1)
    gama = np.zeros((n,1))
    for i in range(len(zu)):
        gama[i,0] = (min(1, max(0, h/120-0.5))) * (min(1, max(0, np.exp(1-zu[i,0]))))

    locations = np.where(gama == 1)[0]
    locations2 = np.where(gama != 0)[0]
    sigmaStruct = np.zeros((n,n))
    lnSigma = list()
    for location in locations:
        sigmaStruct[location,location] = neweyWestMat[location,location]
        lnSigma.append(neweyWestMat[location,location])
    lnSigma = np.log(np.mat(lnSigma))
    # use generalized linear model
    model = sm.GLM(lnSigma, exposure)
    b = model.fit().params
    SigmaXb = np.dot(exposure, b.T)
    for location in locations2:
        sigmaStruct[location,location] = E0 * np.exp(SigmaXb[location,0])

    newMatrix = gama * neweyWestMat + (1 - gama) * sigmaStruct

    return newMatrix

# bayes adjustment
# sigmaSHn = vn * sigmaSn + (1-vn)*sigman  n = 1,2,3,....,N
# sigmaSn = Sigma(wn*sigman)  n = 1,2,3,......, N
# vn = 1 /(deltaSn/(q*abs(sigman - sigmaSn)) + 1)
# deltaSn = sqrt(Sigma((sigman - sigmaSn)^2)/NSn)
# weight is read from barra_factor_exposure also weight should be normalized beforehand
def bayes(structureMat, weight, groups = 10, q = 1):

    sigma = np.mat(np.diag(structureMat)).T

    dfw = pd.DataFrame({'sigma':np.diag(structureMat),'weight':weight})
    dfw['location'] = dfw.index
    dfw.sort_values(by = 'weight', ascending = True, inplace = True)
    dfw.reset_index(drop = True, inplace = True)
    v = np.zeros((len(dfw)//groups,1))
    sigmaS = np.zeros((len(dfw)//groups,1))
    for i in range(groups-1):
        df = dfw.iloc[i*(len(dfw)//groups),(i+1)*(len(dfw)//groups)]
        locations = list(df['location'])
        sigmaTmp = sigma[locations]
        sigmaSTmp = np.ones((len(locations),1))*((df['weight']*df['sigma']).mean())
        deltaS = df['sigma'].std()
        sigmaS[locations] = sigmaSTmp
        v[locations] = 1 /(deltaS/(q*abs(sigmaTmp - sigmaSTmp)) + 1)

    df = dfw.iloc[(groups-1)*(len(dfw)//groups),len(dfw)]
    locations = list(df['location'])
    sigmaTmp = sigma[locations]
    sigmaSTmp = np.ones((len(locations),1))*((df['weight']*df['sigma']).mean())
    deltaS = df['sigma'].std()
    sigmaS[locations] = sigmaSTmp
    v[locations] = 1 /(deltaS/(q*abs(sigmaTmp - sigmaSTmp)) + 1)

    newArray = v*sigmaS + (1-v)*sigma
    newMatrix = np.eye(len(sigma))
    for i in range(len(sigma)):
        newMatrix[i,i] = newArray[i,0]

    return newMatrix


import os, operator, json, copy
from abc import ABC, abstractmethod
from functools import reduce  # forward compatibility for Python 3
from tqdm import tqdm
from . import pprinter
from . import settings
from . import wview
from .. import util

__all__ = ['ProviderManager']

class url_ctx:
    def __init__(self, provider):
        self._provider = provider
        self._debug    = provider._debug
        self._url      = []
        self._num_urls = 0
        self._count    = 0
        self._ticker   = ""

    def add_url(self, url, parser_callback):
        exists = False
        for eurl in self._url:
            if url == eurl[0]:
                exists = True

        if not exists:
            self._url.append( (url, parser_callback) )
            self._num_urls += 1

    def __call__(self, ticker):
        self._ticker = ticker
        return self

    def __iter__(self):
        return self

    def __next__(self):
        if not self._ticker:
            print("You need to call the iterator using a ticker symbol!")
            raise StopIteration
        if self._count == self._num_urls:
            self._count = 0
            raise StopIteration

        url    = self._url[self._count][0].format(ticker=self._ticker)
        parser = self._url[self._count][1]
        self._count += 1

        return url, parser

class provider_data_ctx:
    def __init__(self, debug=False):
        self._debug    = debug
        self._data     = {
            "num_records": 0,
            "active_record": 0,
        }

    def add_entry(self, path, value):
        util.misc.DictUtil.add_by_path(self._data, path, value)

    def get_data(self, path):
        return util.misc.DictUtil.get_by_path(self._data, path)

    def set_data(self, path, new_value):
        dict_value = util.misc.DictUtil.get_by_path(self._data, path)
        if dict_value is not None:
            if type(dict_value) is list:
                dict_value.append(new_value)
            elif type(dict_value) is dict:
                raise Exception("Cannot deal with dictionary values yet!")
            else:
                util.misc.DictUtil.set_by_path(self._data, path, new_value)
        else:
            raise Exception("Key '{}' does not exist!". format(path))

    def data_ref(self):
        return self._data

    def data_copy(self):
        return json.loads( json.dumps(self._data) )

    def load_data(self, data):
        # Thread safe alternative to copy.deepcopy
        self._data = json.loads( json.dumps(data) )

class ProviderBase(ABC):
    def __init__(self, settings_manager, cache_manager, webview_manager, debug=False):
        # Managers
        self._smng = settings_manager
        self._cmng = cache_manager
        self._wmng = webview_manager

        # Internal variables
        self._provider_id = self.provider_id()
        self._ticker_data = {}
        self._cmd = {}
        self._pos_x = 0
        self._pos_y = 1
        self._debug = debug

        # Context data structures
        self._uctx = url_ctx(provider=self)
        self._data = provider_data_ctx(debug=debug)
        self._opts = settings.settings_ctx(provider=self)
        self._dctx = pprinter.draw_ctx(provider=self, dims=(None, None))
        self._wctx = wview.webview_ctx(provider=self)

        # Populate context data structures
        self.generate_URL_ctx(self._uctx, self)
        self.generate_data(self._data)
        self.generate_settings(self._opts)
        self.generate_draw_ctx(self._dctx)
        self.generate_webview_ctx(self._wctx)

        # Validate panels
        self._validate_panels()

        # Validate views
        self._validate_views()

        # Create webviews for this provider
        self._create_webviews()

    def _validate_panels(self):
        for panel_name, panel in self._dctx._panels.items():
            pprinter.validate_panel(panel_name, panel, debug=self._debug)
            self._cmd[panel_name] = pprinter.generate_draw_commands(self.provider_id(), panel, self._opts,  debug=self._debug)

    def _validate_views(self):
        for view_name, view in self._dctx._views.items():
            pprinter.validate_view(view_name, view, debug=self._debug)

    def update_draw_ctx(self, smng, setting_path, new_value, old_value, user_data=None):
        self._opts.update_settings()
        self._validate_panels()
        self._validate_views()
        return new_value

    def _create_webviews(self):
        for webview_data in self._wctx("create"):
            self._wmng.process(webview_data)

    def debug(self, debug=True):
        self._debug       = debug
        self._uctx._debug = debug
        self._data._debug = debug
        self._opts._debug = debug
        self._dctx._debug = debug
        self._wctx._debug = debug

    @staticmethod
    @abstractmethod
    def provider_id():
        return ""

    @staticmethod
    @abstractmethod
    def generate_URL_ctx(uctx, provider):
        pass

    @staticmethod
    @abstractmethod
    def generate_data():
        return {}

    @staticmethod
    @abstractmethod
    def generate_settings(settings):
        pass

    @staticmethod
    @abstractmethod
    def generate_draw_ctx(ctx):
        pass

    @staticmethod
    @abstractmethod
    def generate_webview_ctx(ctx):
        pass

    def stock_data_ref(self):
        return self._data.data_ref()

    def stock_data_copy(self):
        return self._data.data_copy()

    def stock_data_dict_ref(self, ticker):
        return self._ticker_data[ticker].data_ref()

    def stock_data_dict_copy(self, ticker):
        return self._ticker_data[ticker].data_copy()

    def get_panel_data(self, item):
        return util.misc.DictUtil.get_by_path(self._dctx._panels, item)

    def set_panel_data(self, item, value):
        return util.misc.DictUtil.set_by_path(self._dctx._panels, item, value)

    def get_view_data(self, item):
        return util.misc.DictUtil.get_by_path(self._dctx._views, item)

    def set_view_data(self, item, value):
        return util.misc.DictUtil.set_by_path(self._dctx._views, item, value)

    def get_terminal_size(self):
        x, y = os.get_terminal_size()
        return x, y

    def reset_pos(self):
        self._pos_x = 0
        self._pos_y = 0

    def draw_view(self, view_name=None, pos_x=None, pos_y=None):
        # Get first view in dictionary if no view_name is provided
        if not view_name:
            view_name = next( iter( self._dctx._views.keys() ) )

        # Check to see if view exists
        if not self._dctx._views.get(view_name, None):
            raise Exception("View {} does not exist!".format(view_name))

        # Get terminal size
        term_x, term_y = os.get_terminal_size()

        # Get view
        view = self._dctx._views[view_name]

        # Get current x,y position
        cpos_x = pos_x if pos_x else self._pos_x
        cpos_y = pos_y if pos_y else self._pos_y

        # If next view draw exceeds the terminal height
        # then make some space by scrolling down, before drawing
        if cpos_y + view["height"]  > term_y:
            util.screen.move_cursor( term_y - 4 )
            print(util.screen.Style.RESET_ALL + util.screen.pos(0, 0))
            cpos_y = 1

        # Draw each panel in the view
        for panel_name, panel in view["panels"].items():
            width  = panel["width"]
            height = panel["height"]
            ppos_x  = panel["pos_x"]
            ppos_y  = panel["pos_y"]
            self.draw_panel(panel_name, width, height, cpos_x + ppos_x, cpos_y + ppos_y)

        cpos_x = 0
        cpos_y += view["height"]

        self._pos_x = cpos_x
        self._pos_y = cpos_y

        return cpos_x, cpos_y

    def draw_panel(self, panel_name, width, height, pos_x, pos_y):
        # Check to see if panel exists
        if not self._dctx._panels.get(panel_name, None):
            raise Exception("Panel {} does not exist!".format(panel_name))

        # Get drawing commands for this panel
        draw_cmds = self._cmd[panel_name]

        # Draw panel
        for cmd in draw_cmds:
            cmd(pos_x, pos_y, self._data._data)

    def _webview_process_impl(self, wctx, ticker_list):
        pass

    def webview_process(self, command="update", ticker="AMD"):
        ticker_list = []
        return_data = True

        if type(ticker) is str:
            ticker_list.append(ticker)
        elif type(ticker) is list:
            ticker_list = ticker
        elif type(ticker) is None:
            return_data = False
        else:
            raise Exception("'ticker' input must be either of type 'str' or 'list'. Type {} was given".format(type(ticker)))

        # Update the html data dictionary as necessary
        self._webview_process_impl(self._wctx, ticker_list)

        # Update webviews for this provider
        for webview_data in self._wctx(command, data=return_data):
            self._wmng.process(webview_data)

    def process(self, ticker):
        # This method processes a list of ticker symbols in one go. We keep all
        # the extracted data in an internal dictionary, which is cleared next
        # time this method is processed.

        ticker_list = []

        if type(ticker) is str:
            ticker_list.append(ticker)
        elif type(ticker) is list:
            ticker_list = ticker
        else:
            raise Exception("'ticker' input must be either of type 'str' or 'list'. Type {} was given".format(type(ticker)))

        # # Check to see if internal cache is large. If that is the case,
        # # clear it before continuing.
            # if get_size(self._ticker_data) > self._smng._get_setting("global.cache_max_size"):
            #     self._ticker_data.clear()

        uptick_symbol       = self._smng.get_setting("global.progressbar.uptick_symbol")
        ascii_              = self._smng.get_setting("global.progressbar.ascii")
        dynamic_ncols       = self._smng.get_setting("global.progressbar.dynamic_ncols")
        bar_style           = pprinter.Style.RESET_ALL + self._smng.get_setting("global.progressbar.style.bar")
        desc_style          = pprinter.Style.RESET_ALL + self._smng.get_setting("global.progressbar.style.desc")
        uptick_symbol_style = pprinter.Style.RESET_ALL + self._smng.get_setting("global.progressbar.style.uptick")
        ticker_style        = pprinter.Style.RESET_ALL + self._smng.get_setting("global.progressbar.style.ticker")

        if len(uptick_symbol) > 1 :
            uptick_symbol = uptick_symbol[0]

        #================================================================
        # Deal with webviews
        #================================================================
        # Update the html data dictionary as necessary
        self._webview_process_impl(self._wctx, ticker_list)

        # # Update webviews for this provider
        for webview_data in self._wctx("load_html"):
            self._wmng.process(webview_data)

        # Deal with CLI views
        util.screen.hide_cursor()
        for ticker_symbol in tqdm(
            ticker_list,
            # desc=tick_symbol_style + "\u2713 " + desc_style + "Downloaded data from '{}' provider".format(self._provider_id) + bar_style,
            desc=uptick_symbol_style + uptick_symbol + " " + desc_style + "Downloaded data from '{}' provider".format(self._provider_id) + bar_style,
            ascii=ascii_,
            dynamic_ncols=dynamic_ncols,
            ):
            if ticker_symbol not in self._ticker_data:
                # Load stock data for this provider and ticker_symbol
                cached_data = self._cmng.load_stock_data( self, ticker_symbol)

                # Copy cached data to internal data dictionary
                # self._ticker_data[ticker_symbol] = copy.deepcopy(self._data)
                self._ticker_data[ticker_symbol] = self._data
                self._ticker_data[ticker_symbol].load_data(cached_data)

            # Get website data, parse it and store it in internal data dictionary
            print("\r" + desc_style + "Downloading" + ticker_style + "{:^7s}".format(ticker_symbol) + desc_style, end="")
            res = None
            for URL, parser in self._uctx(ticker_symbol):
                website_data = util.web.get_URL_data(URL)
                res = parser(URL, website_data, self._ticker_data[ticker_symbol])

            # Increase number of records and active record entries in data
            if res is not None:
                self._ticker_data[ticker_symbol]._data["num_records"] += 1
                self._ticker_data[ticker_symbol]._data["active_record"] = self._ticker_data[ticker_symbol]._data["num_records"] - 1

            # Save new extracted data
            self._cmng.save_stock_data_dict( self, ticker_symbol)

        print(pprinter.Style.RESET_ALL, end="")
        util.screen.show_cursor()

class ProviderManager:
    def __init__(self, settings_manager, debug=False):
        self._smng  = settings_manager
        self._providers = {}
        self._debug = debug

        # Check external providers directory
        self._check_external_providers()

        # Add available providers to internal dictionary
        self.instantiate_all_providers()

    def instantiate_all_providers(self):
        from .. import get_providers
        providers_temp = get_providers( self._smng.get_setting("global.external_providers_directory") )
        for provider_name, provider in providers_temp.items():
            if not self._providers.get(provider_name):
                if self._debug:
                    print("Registering provider {}".format(provider_name))
                self._providers[provider_name] = provider[1]( settings_manager=self._smng )
                self.register_with_all_style_settings( self._providers[provider_name] )

    def register_with_all_style_settings(self, provider):
        for key in self._smng._settings["gfx"]["style"].keys():
            setting_path = ".".join( ["gfx", "style", key] )
            self._smng.add_set_listener_for_setting(setting_path, provider.update_draw_ctx, user_data=None)

    def _check_external_providers(self):
        # First check to see if the setting is available already. If not, create it and
        # save the new settings state.
        if not self._smng.get_setting("global.external_providers_directory"):
            # Add the default directory as new setting in settings manager.
            self._smng.add_setting(
                "global.external_providers_directory",
                os.path.join(self._smng.get_config_path(), "providers")
            )

            # Save settings state
            self._smng.save_settings()

        # At this stage the external providers directory exists as a setting, so now
        # we need to make sure that the physical directory exists. If not we need to
        # create it here.
        directory = self._smng.get_setting("global.external_providers_directory")
        access_rights = 0o755
        try:
            if not os.path.exists(directory):
                res = create_directory(directory, access_rights)
        except:
            print("There was an error while creating '{}'".format(directory))
            directory = None

        # Add listener for 'global.external_providers_directory' setting
        if directory:
            self._smng.add_set_listener_for_setting("global.external_providers_directory", self._update_external_providers_directory, self)
        else:
            # If there was an error creating the directory in the previous step,
            # then set the value of the setting to an empty string (i.e None)
            self._smng.set_setting("global.external_providers_directory", "")

    @staticmethod
    def _update_external_providers_directory(smng, setting_path, new_value, old_value, user_data):
        if smng._debug:
            print("User updated the external providers path")

        # Create the new directory
        access_rights = 0o755
        directory = new_value
        try:
            if not os.path.exists(directory):
                res = create_directory(directory, access_rights)
        except:
            raise Exception("There was an error while creating '{}'".format(directory))

        try:
            # Copy any existing data from the old location to the new one.
            print("Copying data from '{}' to '{}' ".format(old_value, new_value))
            smng.copytree(old_value, new_value)

            # Since this is a big change, we can save the settings here
            # which will also update the manager's status to not dirty
            print("Saving new settings state...")
            smng.save_settings()
        except:
            raise Exception("There was an error while copying from '{}' to '{}'".format(old_value, directory))

    def display_providers(self):
        print("=================================================================")
        print("Number of available providers: {}\n".format(len(self._providers)))
        for idx, provider in enumerate(self._providers.values()):
            print("\tProvider {:<3}: {}".format(idx+1, provider[0]) )
        print("\n=================================================================\n")

def data_parser(args):
    import ipdb; ipdb.set_trace()
    #=========================================
    # Get list of data providers
    #=========================================
    # At this stage the user has requested to get information for, potentally,
    # several companies and from several data providers.

    # First of all, we need to get a list of all the data providers that the user
    # has requested.
    data_providers = get_list_of_data_providers(args)

    #=========================================
    # For each ticker symbol
    #=========================================
    # Then ,for each ticker symbol we need to do the following:
    cur_i = 0
    cur_y = 1
    for tik in args.ticker_symbols:

        #=========================================
        # For each data provider
        #=========================================
        for provider in data_providers:
            # First check if the information requested by a particular provider is in the cache.
            #
            # 1.) If the information is in the cache, then do the following:
            #     * check to see if the cache report is outdated:
            #       - If the cached report is outdated, retrieve a new report from the provider
            #       - If the cached report is not outdated, then show that to the user.
            # If the information requested is not in the cache, retrieve a new report from the provider.

            #=========================================
            # Check if report is in cache
            #=========================================

            #=========================================
            # Retrieve report
            #=========================================
            # Get URL of ticker
            # URL = zacks.create_URL_stock(tik)
            URL = provider.create_URL_stock(tik)

            # Get data from URL
            web_data = util.web.get_URL_data(URL)

            # Parse website data
            # parsed_data = zacks.website_parser_stock(web_data)
            stock_data = provider.website_parser_stock(web_data)

            #=========================================
            # Save report into cache
            #=========================================

            #=========================================
            # Print report to screen
            #=========================================
            # Once a report has been generated (either by retrieving it from the provider or the cache)
            # we need to print it on screen.

            # Print main title using ticker symbol
            start_x  = util.font.find_starting_pos_for_ticker_title(tik.upper())
            _, cur_y = util.font.print_using_font(tik.upper(), start_x, cur_y, data_formatter.format_tik_font)

            # Print data and get height of row
            # row_y = zacks.format_stock(parsed_data, zacks.logo, 0, cur_y)
            row_y = provider.format_stock(stock_data, provider.logo, 0, cur_y)
            cur_y += row_y

            #=========================================
            # Update screen to make room if necessary
            #=========================================
            # Check to see if next row exceeds the terminals height. If that is the case,
            # we make sure to make some room before printing the next row
            if (cur_y + row_y) > term_y:
                util.screen.move_cursor(term_y)
                print(util.screen.pos(0, 0) + util.screen.Style.RESET_ALL)
                cur_y = 1


import re
import numpy as np
import pickle
import os

flatten=lambda i,d=-1:[a for b in i for a in(flatten(b,d-(d>0))if hasattr(b,'__iter__')and d else(b,))]

def parse2slice(snapshot):

    def parsesection(snap):
        try:
            section = int(snap)
        except ValueError:
            section = [int(s) if s else None for s in snap.split(':')]
            if len(section) > 3:
                raise ValueError('snapshots input incorrect')
            section = slice(*section)
        return section

    section = [s if s else None for s in snapshot.split(',')]
    return tuple([parsesection(section[i]) for i in range(len(section))])

def loadpickle(filename):
    f = open(filename)
    h = pickle.load(f)
    f.close()
    return h

def savepickle(filename, instance):
    f = open(filename, 'w')
    h = pickle.dump(instance, f)
    f.close()

def mkdir(DIR, rm=False):

    if rm:
        import shutil
        try:
            shutil.rmtree(DIR)
        except:
            os.makedirs(DIR)
    try:
        os.makedirs(DIR)
    except:
        return 0

def rm(DIR):
    import shutil
    shutil.rmtree(DIR)

def mkdir_suff(f, suff, base="", ex=""):
    """
    out suffix maker.

    Examples
    ----------------------------

    If argument f is not set, suffix is set to suff.

    >>> st.mkdir_suff("","suff",base="base",ex=".txt")
    base_suff.txt

    If argument f is set only directry, suffix is set to base and make dir automatically.

    >>> st.mkdir_suff("dir/","suff",base="base",ex=".txt")
    dir/base_suff.txt

    If argument f is set, f is interpreted as suffix.

    >>> st.mkdir_suff("test","suff",base="base",ex=".txt")
    base_test.txt

    If argument f is set with dir, after dir/ argument is interpreted as suffix and make dir automatically.

    >>> st.mkdir_suff("dir/test","suff",base="base",ex=".txt")
    dir/base_test.txt

    If argument f is set with extantion, return f.

    >>> st.mkdir_suff("test.dat","suff",base="base",ex=".txt")
    test.dat

    """
    try:
        if len(f.split(".")) > 1:
            file_out = f
            DIR = os.path.dirname(f) + "/"
            if not os.path.exists(DIR):
                os.mkdir(DIR)
            return file_out
    except:
        pass

    if os.path.basename(f) is "":
        SUFF = suff
    else:
        SUFF = os.path.basename(f)
    if os.path.dirname(f) is "":
        DIR = ""
    else:
        DIR = os.path.dirname(f) + "/"
        if not os.path.exists(DIR):
            os.mkdir(DIR)

    BASE=os.path.splitext(os.path.basename(base))[0]
    # print os.path.splitext(base)[0]
    # file_out = re.sub(r'_Sub.*..*$', '', os.path.basename(base))
    file_out = DIR + BASE + ("_" if BASE !="" else "") + SUFF + ex
    return file_out

def convert2new_axis(x, y, new_x):
    from scipy import signal, interpolate
    from scipy.interpolate import splev, splrep
    y_ = np.copy(y)
    x_ = np.copy(x)

    # index =np.where(((np.isinf(y_))+(np.isnan(y_)))==True)[0]
    # x_=np.delete(x_,index)
    # y_=np.delete(y_,index)

    mn, mx = np.searchsorted(new_x, x_[0]), np.searchsorted(new_x, x_[-1]) - 1
    ret = [0. for i in new_x]

    # tck=splrep(x_,y_)
    tck = interpolate.interp1d(x_, y_)
    # ret[mn:mx] = splev(new_x[mn:mx],tck)
    ret[mn:mx] = tck(new_x[mn:mx])
    return ret

def arrange(y):
    # y=np.copy(y_)
    index = np.where(((np.isinf(y)) + (np.isnan(y))) == True)[0]
    y[index] = 0
    return y

def golay(*args, **kwargs):
    # from scipy.interpolate import splev, splrep
    from scipy import signal, interpolate

    """

    This function provide dim-order differential smoothness filter.

    Examples
    ====

    >>> import st
    >>> y_ = st.golay(y,2) #second derivatives
    >>> y_ = st.golay(x,y,2) #second derivatives (with any x)
    >>> y_ = st.golay(x,y,0) #remove invalid values. (nan,inf)
    >>> y_ = st.golay(x,y,1,m=5) #apply smoothness

    Parametors
    -----------------

    x : array-like, shape (n_samples,) , optional
        x-axis data. The pitch size of x dont have to set evenly.
    y : array-like, shape (n_samples,)
        y-axis data
    dim : int
        dim-order differential.
    m : int
        smoothness. In technical, the polynomial order m must be less than the frame size x.
    mode : {valid', 'same'}, optional
        'same':
          By default, mode is 'same'. Mode 'same' returns output of length ``n_samples``.  Boundary
          effects are still visible.
        'valid':
          Mode 'valid' returns output of length ``n_samples-2*m``. Values outside the signal boundary will be cut off.

    Returns
    -----------------

    y_ : array-like, shape (n_samples,)
        Golay returns the array of dim-order differencial with m somoothness

    Reference
    ====
    Golay filter is one of the method of Savitzky-Golay.In ordinaly, See also http://www.mathworks.com/help/signal/ref/sgolay.html?s_tid=gn_loc_drop
    http://www.empitsu.com/pdf/sgd.20081106.pdf
    """

    # Read argumants
    if len(args) == 3:
        x = np.copy(args[0])
        y = np.copy(args[1])
        dim = args[2]
    elif len(args) == 2:
        y = np.copy(args[0])
        dim = args[1]
        x = np.copy(range(len(y)))
    else:
        raise Exception(
            "golay() takes at least 2 arguments (%d given)" % len(args))

    if "m" in kwargs:
        m = kwargs["m"]
    else:
        m = (dim + 1) / 2
    if "mode" in kwargs:
        mode = kwargs["mode"]
    else:
        mode = "same"

    # Check data
    if len(x) != len(y):
        raise ValueError("Input sequence length is invalid %d %d" %
                         (len(x), len(y)))
    if len(x) - m * 2 <= 3:
        raise ValueError("Input sequence length is too small. %d" % len(x_))
    if m < (dim + 1) / 2:
        raise ValueError("Smoothness value :m takes at most %d" %
                         ((dim + 1) / 2))
    if m == 0 and dim == 0:
        return y
    # remove invalid value.
    y = arrange(y)

    # Leveling new_x interavl.
    tck = interpolate.interp1d(x, y)

    alpha = 1.
    x_temp = np.linspace(min(x), max(x), len(x) * alpha)
    while not(x_temp[m:len(x_temp) - m][0] < x[m] and x[len(x) - m - 1] < x_temp[m:len(x_temp) - m][-1]):
        x_temp = np.linspace(min(x), max(x), len(x) * alpha)
        alpha *= 1.5
    y = tck(x_temp)

    def diff(_x, _y, _dim, _m):
        dx = _x[1] - _x[0]
        array = []
        k = _dim  # the number of times of differential
        for ik in range(k + 1):
            a = []
            for im in range(-_m, _m + 1):
                a.append(pow(im, ik))
            array.append(a)
        X = np.array(array)
        B = np.dot(np.linalg.inv(np.dot(X, X.T)), X)

        func = lambda s: (lambda m: m(m)(s))(lambda proc: (
            lambda n: 1 if n == 0 else n * proc(proc)(n - 1)))
        seeknum = len(_y) - 2 * _m
        AA = [np.dot(B, _y[seek:seek + 1 + _m * 2]) *
              func(dim) / dx**_dim for seek in range(seeknum)]
        xx = [_x[seek + _m] for seek in range(seeknum)]
        return np.array(xx), np.array(AA).T[dim]

    new_x, new_y = diff(x_temp, y, dim, m)
    # tck=interpolate.InterpolatedUnivariateSpline(new_x,new_y,)
    tck = interpolate.interp1d(new_x, new_y,)
    # print (new_x[0]<x[m] , x[len(x)-m-1]<new_x[-1])
    y = tck(x[m:len(x) - m])

    if mode is "same":
        ret = np.array([0. for i in range(len(x))])
        ret[m:len(x) - m] = y
        return ret
    if mode is "valid":
        return np.array(y)

    # return thinouted(np.array(xx),tnum),thinouted(np.array(AA).T[dim],tnum)
    # return np.array(xx),np.array(AA).T[dim]


# packages
# pip install NeuroLab
#     or
# conda install -c labfabulous neurolab
#

#
# This program train a single perceptron to simulate a "and" function
# s198249 Lam Wai Yuen

import neurolab as nl
import matplotlib.pyplot as pl

##AND
#input = [[0, 0], [0, 1], [1, 0], [1, 1]]
#target = [[0],   [0],   [0],     [1]]

##OR
input = [[0, 0], [0, 1], [1, 0], [1, 1]]
target = [[0],   [1],   [1],     [1]]
##XOR
#input = [[0, 0], [0, 1], [1, 0], [1, 1]]
#target = [[0],   [1],   [1],     [0]]
#
#  Create a single layer perceptron
#

net = nl.net.newp([[0, 1],[0, 1]], 1)

#
#  Training
#

error_progress = net.train(input, target, epochs=100, show=10, lr=0.1)

#
#  Testing
#

y = net.sim(input)

#
#  Plot the result
#

pl.subplot(211)  # a subfigure

pl.plot(error_progress)
pl.xlabel('Number of epochs')
pl.ylabel('Training error')

pl.subplot(212)  # a subfigure
pl.plot(target, '+',  y, 'x')
pl.xlabel('Test case')
pl.ylabel('Output)')
pl.legend(['Target output', 'System output'])

pl.show()


from sikuli import *
import logging
import myTools

#---------------------------------------------------#
def fFilter_Invoices():
#---------------------------------------------------#

    # selection
    logging.debug('- filters')
    click("selection_button.png")
    time.sleep(1)

    # clear any existing filters
    myTools.clickRemoveAll()

    # add tran type filter
    click("trans_type.png")
    time.sleep(1)
    click("add_filter-1.png")
    time.sleep(1)

    # unselect all
    type(Key.DELETE)

    # select invoice
    myTools.pressDOWN(5)
    type(Key.F4)
    time.sleep(1)

    # OK
    myTools.pressTAB(1)
    type(Key.ENTER)
    time.sleep(1)

    # OK
    type(Key.ENTER)
    time.sleep(1)

    # Update
    myTools.pressTAB(1)
    type(Key.ENTER)
    myTools.waitForTransList()

#---------------------------------------------------#
def fClear_Filter():
#---------------------------------------------------#

    # selection
    click("selection_button.png")
    time.sleep(1)
    # clear
    myTools.clickRemoveAll()
    # OK
    type(Key.ENTER)
    time.sleep(1)
    # Update
    myTools.pressTAB(1)
    type(Key.ENTER)
    myTools.waitForTransList()

#---------------------------------------------------#
def fCreate_SpecCredits(pMonth):
#---------------------------------------------------#

    myTools.sectionStartTimeStamp("specialCredits" + str(pMonth))
    logging.debug('Create_SpecCredits: ' + str(pMonth))

    # skipping for now. not sure why

    if int(Settings.tsVersion) < 2018:
        logging.debug('- SKIP until 2018')
        return

    myTools.getFocus()

    # list of invoices; one for each month
    invList = ["12400","12600","13100","13330","13830","14210","14370","14850","15200","15460","15960","16240"]

    # open a/r tran list
    logging.debug('- open a/r list')
    type("t",KeyModifier.CTRL)
    myTools.waitForTransList()

    # go to invoice
    type('g',KeyModifier.CTRL + KeyModifier.SHIFT)
    time.sleep(1)
    myTools.pressTAB(2)
    invNum = invList[pMonth-1]
    logging.debug('- look for: ' + invNum)
    type(invNum)
    type(Key.ENTER)
    time.sleep(1)

    # open it
    type(Key.ENTER)
    myTools.waitForTransEntry()

    # apply a new special credit
    click("apply_new.png")
    myTools.pressDOWN(5)
    type(Key.ENTER)
    time.sleep(1)

    # type
    type(Key.TAB)

    # client
    type(Key.TAB)

    # date
    tranDate = str(pMonth) + "/28/" + Settings.dataYear
    type(tranDate)
    time.sleep(1)
    type(Key.TAB)

    # Description
    type(Key.END)
    type(" - " + tranDate)
    time.sleep(1)
    myTools.pressTAB(2)

    # Fees
    feeAmount = 5 + pMonth/float(100)
    type(str(feeAmount))
    time.sleep(1)
    type(Key.TAB)

    # Costs
    costAmount = 4 + pMonth/float(100)
    type(str(costAmount))
    time.sleep(1)
    type(Key.TAB)

    # Interest
    interestAmount = 3 + pMonth/float(100)
    type(str(interestAmount))
    time.sleep(1)

    #save
    type("s",KeyModifier.CTRL)
    myTools.checkForUnappliedAmount()
    myTools.waitForTransSave()

    # close
    type(Key.F4,KeyModifier.CTRL)
    time.sleep(1)
    type(Key.F4,KeyModifier.CTRL)

    myTools.sectionEndTimeStamp()
    myTools.checkProcesses()


from pygame.sprite import Sprite

class MysteryBox(Sprite):
    def __init__(self, screen, tag, image, left, bottom):
        super().__init__()
        self.screen = screen
        self.image = image
        self.tag = tag
        self.rect = self.image.get_rect()
        self.rect.x = left
        self.rect.bottom = bottom
        self.item = None

    def set_item(self, item):
        self.item = item

    def draw(self, camera):
        self.screen.blit(self.image, camera.apply(self))