#!/usr/bin/env python
import website.bigfoot.models as models
from website.bigfoot.utils import config, siteaction, livewire_client, dataplus, statix
from django.http import HttpResponse, HttpResponseRedirect
import datetime, os, sys, Image #PIL from Pythonware, thx guys

max_per_file_size = 7340032     #7 MB

archive_types = ['zip', 'tgz', 'tar', 'gz', 'z']
html_types = ['html', 'htm', 'xhtml']
static_image_types = ['tiff', 'tif', 'psd']
image_types = ['gif', 'jpg', 'jpeg', 'bmp', 'png']
music_types = ['mp3', 'mid', 'wav', 'wma', 'ram']
openoffice_types = ['odt', 'odp', 'ods']
video_types = ['mpg', 'wmv', 'avi', 'mpeg', 'mov', 'rv', 'swf']

archive_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/archive-icon.gif')
file_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/file-icon.gif')
html_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/html-icon.gif')
image_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/image-icon.gif')
music_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/music-icon.gif')
openoffice_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/oo-icon.gif')
pdf_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/pdf-icon.gif')
powerpoint_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/ppt-icon.gif')
video_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/video-icon.gif')
word_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/word-icon.gif')
excel_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/xl-icon.gif')
text_icon_url = dataplus.getStaticUrl('/apps/socialray/website/bigfoot/ui/site/images/chat/text-icon.gif')

def getFileTypeNThumbnail(ext):
    if ext in archive_types:
        return 'archive', archive_icon_url
    if ext in html_types:
        return 'html', html_icon_url
    if ext in static_image_types:
        return 'static_image', image_icon_url
    if ext in image_types:
        return 'image', image_icon_url
    if ext in music_types:
        return 'music', music_icon_url
    if ext in openoffice_types:
        return 'openoffice', openoffice_icon_url
    if ext in video_types:
        return 'video', video_icon_url
    if ext == 'doc':
        return 'word', word_icon_url
    if ext in ['ppt', 'pps']:
        return 'powerpoint', powerpoint_icon_url
    if ext == 'xls':
        return 'excel', excel_icon_url
    if ext == 'pdf':
        return 'pdf', pdf_icon_url
    if ext == 'txt':
        return 'text', text_icon_url
    return 'unknown', file_icon_url

def saveFile(request, chat_id, field_name):
    file = None
    ext = None
    size = None
    try:
        if not request.FILES:
            return False, 'invalid_upload', '', '', '', '', ''

        file = request.FILES[field_name]
        filename = file.name.lower()

        filename_parts = filename.split('.')
        if len(filename_parts) == 1:
            ext = ''
        else:
            ext = filename_parts[len(filename_parts) -1]

        if ext in ['exe', 'vbs', 'wmf']:
            return False, 'unsupported_file_type', '', '', '', '', ''

        filecontent = file.read()
        size = len(filecontent)
        if size > max_per_file_size:
            return False, 'large_upload', '', '', '', '', ''

        #save the main file
        file_dir = config.website_base_path + '/chat/data/' + chat_id
        if not os.path.exists(file_dir):
            os.mkdir(file_dir)

        filename = getNewFilename(file_dir, filename)
        file_path = file_dir + '/' + filename
        f = open(file_path, 'wb')
        f.write(filecontent)
        f.close()

        file_type, thumbnail_url = getFileTypeNThumbnail(ext)

        if file_type == 'image':
            try:
                im = Image.open(file_path).convert('RGB')
                im.thumbnail((128,128), Image.ANTIALIAS)
                tmp_path = file_dir + '/preview-' + filename
                im.save(tmp_path, 'JPEG')
                thumbnail_url = dataplus.getStaticUrl(tmp_path)
            except:
                pass

        return True, '', filename, dataplus.getStaticUrl(file_path), file_type, size, thumbnail_url
    except:
        logError('File upload error: ' + str(sys.exc_info()[0]) + ', ' + str(sys.exc_info()[1]))
        return False, 'unknown', '', '', '', '', ''

def getNewFilename(folderPath, filename):
    namepart = ''
    ext = ''
    idx = filename.find('.')
    if idx != -1:
        namepart = filename[:idx]
        ext  = filename[idx:]
    else:
        namepart = filename

    new_filename = filename
    ctr  = 0
    while(os.path.exists(folderPath + '/' + new_filename)):
        ctr += 1
        new_filename = namepart + '(' + str(ctr) + ')' + ext

    return new_filename

def handle(request):
    if request.method == 'POST':
        upload_id = dataplus.dictGetVal(request.REQUEST, 'uploadId')
        if not upload_id:
            return getRenderedResponse('', False, 'invalid_upload_id')

        chat_id = dataplus.dictGetVal(request.REQUEST, 'chatId')
        if not chat_id:
            return getRenderedResponse(upload_id, False, 'invalid_chat_id')

        success, error_code, filename, file_url, file_type, size, thumbnail_url = saveFile(request, chat_id, 'uploadFile')
        if success:
            session_id = request.COOKIES['session_id']
            result = livewire_client.query('addFile', [session_id, chat_id, filename, file_url, file_type, size, thumbnail_url, ''])
            if result.startswith('error:'):
                return getRenderedResponse(upload_id, False, result[6:])

            return getRenderedResponse(upload_id, True, '')
        else:
            return getRenderedResponse(upload_id, False, error_code)

def getRenderedResponse(upload_id, success, error_code):
    return siteaction.render_to_response('chat/fileupload.htm', {  'upload_id':upload_id,
                                                        'success':('false','true')[success],
                                                        'error_code':error_code})

def logError(err):
    try:
        file = open('/apps/socialray/var/chat_fileupload_errors.txt', 'a')
        file.write(str(datetime.datetime.utcnow()) + '\t' + err + '\n')
    except:
        pass
    finally:
        file.close()


# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# example use: python code/invoice.py September 2019
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------

# LIBRARIES
#python 3.6.8
#tkinter 8.6
#numpy 1.15.4
#pandas 0.24.0
#docxtpl 0.6.3
#docx 0.8.7
#docxcompose 1.0.2

import tkinter as tk
import numpy as np
import pandas as pd
import os

from docxtpl import DocxTemplate
from docx import Document
import datetime
from docxcompose.composer import Composer
from sys import argv

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# COMMAND LINE INPUTS

# check command line args for month and year of invoice
if len(argv) > 2:
    MONTH = argv[1] # first argument is the month
    YEAR = argv[2] # second argument is the year
else:
    exit('Missing MONTH and/or YEAR argument.')

# month set
month_set = set(['January', 'February', 'March', 'April', 'May', 'June', 'July',
              'August', 'September', 'October', 'November', 'December'])
# check for proper month
if MONTH not in month_set:
    exit('Provide a proper month.')

# try for proper dates
try:
    CURR_DATE = datetime.datetime.strptime(MONTH+' ' + YEAR, '%B %Y')
except ValueError: # else exit
    str_exit =  'Improper command line inputs.\n'
    str_exit += 'Proper input looks like:\n'
    str_exit += 'python code/invoice.py September 2019'
    exit(str_exit)

### needed template files
# docs
fpage = 'templates/fpage.docx' # first page
invoice_template = 'templates/invoice_template.docx' # base template
invoice_template_mult = 'templates/invoice_template_mult.docx' # goes with RPind
invoice_template_z = 'templates/invoice_template_z.docx' # goes with Zind
invoice_template_z_gg = 'templates/invoice_template_z_gg.docx' # goes with Zind/ gg
invoice_template_v = 'templates/invoice_template_v.docx' # goes with Vind

# data
invoice_data_template = 'templates/invoice_data_template.xlsx'

# check for needed files
if not os.path.exists(fpage): # fpage template
    exit('Missing fpage template.')
if not os.path.exists(invoice_template): # word template
    exit('Missing invoice word template.')
if not os.path.exists(invoice_template_mult): # word template mult
    exit('Missing invoice word template mult.')
if not os.path.exists(invoice_template_z): # word template z
    exit('Missing invoice word template z.')
if not os.path.exists(invoice_template_z_gg): # word template z_gg
    exit('Missing invoice word template z_gg.')
if not os.path.exists(invoice_template_v): # word template v
    exit('Missing invoice word template v.')
if not os.path.exists(invoice_data_template): # data template
    exit('Missing invoice data template.')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# IMPORT and AUGMENT DATA

# column list
col_list = ['OWNER', 'DIRECTION', 'FIRST', 'LAST',
            'STREET_ADDRESS', 'CITY_ADDRESS',
            'MY_NOTES',
            'MONTHLY_CHARGE',
            'EMAIL_ADDRESS', 'EMAIL_IND',
            'COND_CHARGE', 'FILT_CHARGE',
            'COND_MONTHS', 'FILT_MONTHS',
            'SEND_REMINDER', 'RP_INDICATOR',
            'Z_INDICATOR', 'V_INDICATOR']
# data type dictionary
types = {'OWNER':str, 'DIRECTION':str, 'FIRST':str, 'LAST':str,
        'STREET_ADDRESS':str, 'CITY_ADDRESS':str,
        'MY_NOTES':str,
        'MONTHLY_CHARGE':float,
        'EMAIL_ADDRESS':str, 'EMAIL_IND':int,
        'COND_CHARGE':float, 'FILT_CHARGE':float,
        'COND_MONTHS':str, 'FILT_MONTHS':str,
        'SEND_REMINDER':int, 'RP_INDICATOR':int,
        'Z_INDICATOR':int, 'V_INDICATOR':int}

# read in data
tabl = pd.read_excel(invoice_data_template,
                     usecols= col_list,
                     dtype=types,
                     skiprows=0)
# fix email improper reading in of NA string values in emails
tabl['EMAIL_ADDRESS'] = tabl['EMAIL_ADDRESS'].astype(str) #'NA' is converted to string 'nan'
# fill in any missing data for string variables
# fill in with a blank ''
na_list = ['FIRST', 'LAST', 'STREET_ADDRESS', 'CITY_ADDRESS', 'MY_NOTES']
for col in na_list:
    tabl[col].fillna(value='', inplace=True)

# number of rows
N = tabl.shape[0]

# add new columns
tabl['ADD_CHARGE'] = np.zeros(N, dtype=float)
tabl['ADD_CHARGE_NOTES'] = ['']*N
tabl['ADD_CHARGE_2'] = np.zeros(N, dtype=float)
tabl['ADD_CHARGE_NOTES_2'] = ['']*N
tabl['CUST_REMINDER'] = ['']*N

# indicator for whether to include rows or not for the outgoing files
included = np.ones(N, dtype=int)

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
## CREATE DIRECTORY
print("Creating directories...")

### month dict: month name -> numerical two digit code
month_dict = {'January':'01', 'February':'02', 'March':'03', 'April':'04',
                'May':'05', 'June':'06', 'July':'07', 'August':'08',
                'September':'09', 'October':'10', 'November':'11', 'December':'12'}
next_month_dict = {'January':'February', 'February':'March', 'March':'April',
                    'April':'May', 'May':'June', 'June':'July', 'July':'August',
                    'August':'September', 'September':'October', 'October':'November',
                    'November':'December', 'December':'January'}

# make directory
# example: invoices/2019/2019_09_September
# this allows invoices to first be sorted by years to avoid eventual clutter, and
# the to be properly sorted numerically
# -- e.g. 2019_08_August is on top of 2019_09_September
path = 'invoices/'+YEAR+'/'+YEAR+'_'+month_dict[MONTH]+'_'+MONTH
try:
    os.mkdir(path)
except OSError:
    exit("Creation of the directory %s failed" % path)

##### sub directories
##### example: invoices/2019/2019_09_September/PP and invoices/2019/2019_09_September/GG
## pp
sub_path_pp = path+'/PP'
try:
    os.mkdir(sub_path_pp)
except OSError:
    exit("Creation of the directory %s failed" % sub_path_pp)
## gg
sub_path_gg = path+'/GG'
try:
    os.mkdir(sub_path_gg)
except OSError:
    exit("Creation of the directory %s failed" % sub_path_gg)

##### make "emails" directories
## pp
emails_path_pp = sub_path_pp+'/emails'
try:
    os.mkdir(emails_path_pp)
except OSError:
    exit("Creation of the directory %s failed" % emails_path_pp)
## gg
emails_path_gg = sub_path_gg+'/emails'
try:
    os.mkdir(emails_path_gg)
except OSError:
    exit("Creation of the directory %s failed" % emails_path_gg)

print("Directories created.")

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# GLOBAL VARS and CONSTANTS

##### global vars
# current row
index = 0
# indicator for whether or not the 'back button' is on in the gui
back_on = False
# indicator for restoring the main gui format upon going back to the main...
# ...program from the exit screen gui format
last_back = False
# error string to attach if we receive improper input
error_str = ''
# back string to infrom user we went back a row
back_str = ''
# indicator for whether we have finished the main invoicing program
final_end = False
# dates error indicator
dates_err_ind = False
# reminder notice indicator
remind_ind = False
##### last row of the gui format
# main -> 0; month charge -> 1;
# add charge -> 2; notes add charge -> 3;
# add charge 2 -> 4; notes add charge 2 -> 5;
# notes -> 6; cust remind -> 7; check -> 8;
# button one and button two -> 9
last_row = 9

##### constants for tkinter display windows
font = 'Times' # font type
bold_ind = 'bold' # bold indicator
size = 18 # font size
col1 = '#14bcfe' #color-> blue-ish
col2 = '#fe9114' #color -> orange-ish
notes_color = 'black' # notes color
reminder_color = 'purple' ## '#cf2b29';'#e34240'; red-ish
amt_color = 'green' # amount color
add_color = '#0a4cbf' # #blue-ish
add_color_2 = '#0a98bf' #light blue-sh'
cust_remind_color = 'red' # cust reminder color
thick = 2 # highlighted thickness
pad = 6 # amount of padding
width = 1000 # width dim -> of the entire window
height = 800 # height dim -> of the entire window
entry_width = 80 # width dim of the user input sections
on_color = 'red' # color of checked button when on
off_color = 'grey' # color of checked button when off

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# WINDOW FUNCTIONS

##### center the window upon opening with a given width and height
def center_window(base, width=300, height=200):
    # get screen width and height
    screen_width = base.winfo_screenwidth()
    screen_height = base.winfo_screenheight()
    # calculate position x and y coordinates
    x = (screen_width/2) - (width/2)
    y = (screen_height/2) - (height/2)
    base.geometry('%dx%d+%d+%d' % (width, height, x, y))
    return

##### close root i.e. the main program
def close_root():
    # global vars
    global root
    global final_end

    # destroy root
    root.destroy()
    # set checkpoint
    final_end = True
    return

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# HELPER FUNCTIONS for navigating

# check value of checkbutton
# - if its clicked, then turn it on to appropriate color
# - else leave it deafault black color
def on_check():
    global var, chk
    if var.get() == 1:
        chk["fg"] = on_color
    else:
        chk["fg"] = off_color
    return

##### function to begin the main program
def begin():
    # global vars
    global btn_one, btn_two
    global monthly_charge_lbl, monthly_charge_entry
    global add_charge_lbl, add_charge_entry
    global add_charge_notes_lbl, add_charge_notes_entry
    global add_charge_lbl_2, add_charge_entry_2
    global add_charge_notes_lbl_2, add_charge_notes_entry_2
    global my_notes_lbl, my_notes_entry
    global cust_reminder_lbl, cust_reminder_entry
    global chk

    ##### main label
    main_lbl.grid_forget()

    ##### first button
    # change text: begin -> next
    # change command
    # if clicked, we move to the next row
    # for proper display, we forget it so that we can place it at the end later
    btn_one.grid_forget()
    btn_one.config(text= 'Next')
    btn_one.config(command= next)

    ##### second button
    # change text: close -> back
    # change command
    # if clicked, we move back to the previous row
    # for proper display, we forget it so that place it at the end later
    btn_two.grid_forget()
    btn_two.config(text= 'Back')
    btn_two.config(command= back)

    ##### pack the labels and entries
    row_inc = 0
    # main label
    main_lbl.grid(row= row_inc, column= 0, pady= pad, columnspan=2)
    row_inc += 1
    # monthly charge
    monthly_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
    monthly_charge_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # additional charge
    add_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
    add_charge_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # notes on additional charges
    add_charge_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
    add_charge_notes_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # additional charge 2
    add_charge_lbl_2.grid(row= row_inc, column= 0, pady= pad)
    add_charge_entry_2.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # notes on additional charges 2
    add_charge_notes_lbl_2.grid(row= row_inc, column= 0, pady= pad)
    add_charge_notes_entry_2.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # my notes
    my_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
    my_notes_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # customer reminders
    cust_reminder_lbl.grid(row= row_inc, column= 0, pady= pad)
    cust_reminder_entry.grid(row= row_inc, column= 1, padx= pad)
    row_inc += 1
    # check button
    chk.grid(row= row_inc, column= 0, pady= pad)
    row_inc += 1

    ##### pack back in the next button, but not the back button just yet
    # - turn the back button on at the second row
    btn_one.grid(row= row_inc, column= 0, pady= pad)

    ##### update the display
    update()
    return

##### update the gui display
def update():
    # global vars
    global index
    global error_str, error_lbl
    global back_on, last_back, back_str
    global btn_one,  btn_two
    global var, chk
    global main_lbl
    global monthly_charge_lbl, monthly_charge_entry
    global add_charge_lbl, add_charge_entry
    global add_charge_notes_lbl, add_charge_notes_entry
    global add_charge_lbl_2, add_charge_entry_2
    global add_charge_notes_lbl_2, add_charge_notes_entry_2
    global my_notes_lbl, my_notes_entry
    global cust_reminder_lbl, cust_reminder_entry
    global CURR_DATE, MONTH
    global dates_err_ind, remind_ind

    ##### clear the entries
    monthly_charge_entry.delete(0, 'end')
    add_charge_entry.delete(0, 'end')
    add_charge_notes_entry.delete(0, 'end')
    add_charge_entry_2.delete(0, 'end')
    add_charge_notes_entry_2.delete(0, 'end')
    my_notes_entry.delete(0, 'end')
    cust_reminder_entry.delete(0, 'end')
    ##### reset checkbox
    var.set(0) # value -> set it back to zero ie off
    chk["fg"] = off_color # color -> set back to off color, not clicked

    ##### error string and label logic
    if error_str != '': # add in the error label and message
        error_lbl.config(text= error_str)
        error_lbl.grid(row= last_row+1, column= 0, pady = 0, columnspan=2)
    else:
        # remove the error lbl if present before
        # if error was not present before, this does nothing
        error_lbl.grid_forget()

    ##### back button logic
    # --------------------------------------------------------------------------
    # add the 'back button' if index > 0; we don't want a back button for the
    # first row becuase there is nothing to go back to; close the window to quit
    if index == 1 and back_on == False:
        # pack the back button
        btn_two.grid(row= last_row, column= 1, padx= pad)
        # update conditional that the 'back button' is on
        back_on = True
    # --------------------------------------------------------------------------
    # remove back button if on index=0 i.e. we got back to the first row and
    # need to remove the back button because there is nothing to go back to
    if index == 0 and back_on == True:
        btn_two.grid_forget() # remove button
        back_on = False # update conditional
    # --------------------------------------------------------------------------
    # going back from the exit - last - screen
    if last_back == True:
        # we want to restore the main gui format
        # -- i.e. the labels and entries followed by the buttons

        ##### forget the buttons
        btn_one.grid_forget()
        btn_two.grid_forget()

        ##### bring back the labels and entries
        # increment row bc we do not forget the main label
        row_inc = 1
        # monthly charge
        monthly_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
        monthly_charge_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge
        add_charge_lbl.grid(row= row_inc, column= 0, pady= pad)
        add_charge_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge notes
        add_charge_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
        add_charge_notes_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge 2
        add_charge_lbl_2.grid(row= row_inc, column= 0, pady= pad)
        add_charge_entry_2.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # additional charge notes 2
        add_charge_notes_lbl_2.grid(row= row_inc, column= 0, pady= pad)
        add_charge_notes_entry_2.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # my notes
        my_notes_lbl.grid(row= row_inc, column= 0, pady= pad)
        my_notes_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # customer reminders
        cust_reminder_lbl.grid(row= row_inc, column= 0, pady= pad)
        cust_reminder_entry.grid(row= row_inc, column= 1, padx= pad)
        row_inc += 1
        # check button
        chk.grid(row= row_inc, column= 0, pady= pad)
        row_inc += 1

        ##### bring back the buttons
        btn_one.grid(row= row_inc, column= 0, pady= pad)
        btn_two.grid(row= row_inc, column= 1, padx= pad)
        # reconfig the first button i.e. the begin button at start, close at end
        btn_one.config(text= 'Next') # update text
        btn_one.config(command= next) # update command

        ###### update last_back conditional i.e. if we press the back button
        # we are no longer on the end screen, coming back from the exit screen
        last_back = False
    ##### END ----------------------------------------------- back button logic

    ##### update logic
    # check if we can update
    # if no more rows left, end
    # if we can update, replace entries with appropriate text
    if index > N-1:
        end() # end screen logic
    else:
        # grab current line
        curr_line = dict(tabl.iloc[index])

        ##### input entries
        ## main label text
        text = ''
        # inform user that we've come back to this row
        if back_str != '':
            text += back_str
            # revert back string
            back_str = ''
        # main info
        text += curr_line['FIRST']+' '+ curr_line['LAST'] + '\n\n' + \
                curr_line['STREET_ADDRESS'] + '\n' + \
                curr_line['CITY_ADDRESS']

        ### ADD TEXT - conditional
        ## cond
        if curr_line['COND_MONTHS'] != 'exclude':
            pot_cond_months = set("".join(curr_line['COND_MONTHS'].split(',')).split())
            ## - proper months
            if len(pot_cond_months - month_set) == 0:
                # check if current month is in potential cond... months
                if MONTH in pot_cond_months:
                    remind_ind = True
                    text += '\n\n' + \
                    "Remember, it's time for cond...." + \
                    ' - $' + str(curr_line['COND_CHARGE'])

                # check if next month is in potential cond... months
                if (curr_line['SEND_REMINDER'] == 1) and (next_month_dict[MONTH] in pot_cond_months):
                    remind_ind = True
                    text += '\n\n' + \
                    "Next month time for cond... send reminder this month."
            ## - improper months
            else:
                dates_err_ind = True
                text += '\n\n' + "Error, improper cond... month"
        ## filt
        if curr_line['FILT_MONTHS'] != 'exclude':
            pot_filt_months = set("".join(curr_line['FILT_MONTHS'].split(',')).split())
            ## - proper months
            if len(pot_filt_months - month_set) == 0:
                # check if current month is in potential filt... months
                if MONTH in pot_filt_months:
                    remind_ind = True
                    text += '\n\n' + \
                    "Remember, it's time for filt...." + \
                    ' - $' + str(curr_line['FILT_CHARGE'])

                # check if next month is in potential cond... months
                if (curr_line['SEND_REMINDER'] == 1) and (next_month_dict[MONTH] in pot_filt_months):
                    remind_ind = True
                    text += '\n\n' + \
                    "Next month time for filt... send reminder this month."
            ## - improper months
            else:
                dates_err_ind = True
                text += '\n\n' + "Error, improper filt... month"

        ##### update the main label
        # color
        if dates_err_ind == True: # first indicator, red if any error
            main_lbl.config(fg= 'red')
        elif remind_ind == True: # if no error, then can set reminder
            main_lbl.config(fg= reminder_color)
        else: # revert to default, black color
            main_lbl.config(fg= 'black')
        # text
        main_lbl.config(text=text)
        ##### revert indicators
        dates_err_ind = False
        remind_ind = False

        ##### update entries
        ## monthly charge entry
        monthly_charge_entry.insert(0, curr_line['MONTHLY_CHARGE'])
        ## my notes
        my_notes_entry.insert(0, curr_line['MY_NOTES'])

        return

##### ending the program
def end():
    # global vars
    global main_lbl
    global btn_one
    global monthly_charge_lbl, monthly_charge_entry
    global add_charge_lbl, add_charge_entry
    global add_charge_notes_lbl, add_charge_notes_entry
    global add_charge_lbl_2, add_charge_entry_2
    global add_charge_notes_lbl_2, add_charge_notes_entry_2
    global my_notes_lbl, my_notes_entry
    global cust_reminder_lbl, cust_reminder_entry
    global chk
    global last_back

    ##### updating labels and buttons
    ## update the text in the main label and its color
    main_lbl.config(text= "You're done.")
    main_lbl.config(fg= 'black')

    ## update button one - originally the begin button then the next button
    btn_one.config(text= 'Close') # update the text of the button
    btn_one.config(command= close_root) # update the button's command
    # no need to change affect of the back button

    ##### forget the other labels and entries
    # monthly charge
    monthly_charge_lbl.grid_forget()
    monthly_charge_entry.grid_forget()
    # additonal charge
    add_charge_lbl.grid_forget()
    add_charge_entry.grid_forget()
    # additional charge notes
    add_charge_notes_lbl.grid_forget()
    add_charge_notes_entry.grid_forget()
    # additonal charge 2
    add_charge_lbl_2.grid_forget()
    add_charge_entry_2.grid_forget()
    # additional charge notes 2
    add_charge_notes_lbl_2.grid_forget()
    add_charge_notes_entry_2.grid_forget()
    # my notes
    my_notes_lbl.grid_forget()
    my_notes_entry.grid_forget()
    # customer reminder
    cust_reminder_lbl.grid_forget()
    cust_reminder_entry.grid_forget()
    # check button
    chk.grid_forget()

    # going back from the end
    last_back = True # update the indicator to restore main gui format
    return

##### next - go to the next row
def next():
    # global vars
    global index, error_str, var
    global monthly_charge_entry
    global add_charge_entry, add_charge_notes_entry
    global add_charge_entry_2, add_charge_notes_entry_2
    global my_notes_entry, cust_reminder_entry

    # grab and try reading in input
    try:
        # monthly charge input
        monthly_in_amt_str = monthly_charge_entry.get().strip()
        if monthly_in_amt_str == '':
            monthly_in_amt_str = '0'
        monthly_in_amt = float(eval(monthly_in_amt_str))
        # additional charge input
        add_in_amt_str = add_charge_entry.get().strip()
        if add_in_amt_str == '':
            add_in_amt_str = '0'
        add_in_amt = float(eval(add_in_amt_str))
        # additional charge notes input
        in_add_charge_notes = str(add_charge_notes_entry.get().strip())

        # additional charge input 2
        add_in_amt_str_2 = add_charge_entry_2.get().strip()
        if add_in_amt_str_2 == '':
            add_in_amt_str_2 = '0'
        add_in_amt_2 = float(eval(add_in_amt_str_2))
        # additional charge notes input 2
        in_add_charge_notes_2 = str(add_charge_notes_entry_2.get().strip())

        # my_notes
        in_my_notes = str(my_notes_entry.get().strip())
        # cust_reminder
        in_cust_reminder = str(cust_reminder_entry.get().strip())

        # update data
        tabl.at[index, 'MONTHLY_CHARGE'] = monthly_in_amt
        tabl.at[index, 'ADD_CHARGE'] = add_in_amt
        tabl.at[index, 'ADD_CHARGE_NOTES'] = in_add_charge_notes
        tabl.at[index, 'ADD_CHARGE_2'] = add_in_amt_2
        tabl.at[index, 'ADD_CHARGE_NOTES_2'] = in_add_charge_notes_2
        tabl.at[index, 'MY_NOTES'] = in_my_notes
        tabl.at[index, 'CUST_REMINDER'] = in_cust_reminder

        ##### checkbox - included update
        if var.get() == 1:
            included[index] = 0

        ##### clear the error string
        # ie we have not run into an error at this point
        error_str = ''

        ##### update index -- bc we have no error
        index += 1

    # reached an error
    except NameError or SyntaxError:
        # we have an error, so update error string
        # and do not move forward with index
        error_str = '\n\n' + \
        'Previous error. Make sure amounts are valid real numbers.'

    ##### update
    # -- if we had no error, we update to the next row
    # -- if we had an error, we re-update the current row with an error message
    update()
    return

##### next - go to the next row
def back():
    # global vars
    global index, back_str, error_str

    # revert index
    index -= 1

    # make sure we revert the included ariable indicator just in case
    included[index] = 1

    # update back string
    back_str = '- WENT BACK - \n\n'

    # update error string - in case we were at an error
    error_str = ''

    # we went back, so now update the main gui display
    update()
    return

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# SET-UP the gui

##### root for gui
root = tk.Tk()
root.title('Invoice: '+MONTH+' '+YEAR)
#root.configure(background='grey')

##### center the window
center_window(root, width, height)

###### main label where we will show input
main_lbl = tk.Label(root, text= 'Click to begin.')
main_lbl.config(font= (font, size, bold_ind))
main_lbl.config(fg= 'black')

main_lbl.grid(row=0, column=0, pady=pad, columnspan=2)

##### error label
error_lbl = tk.Label(root, text= error_str)
error_lbl.config(font= (font, size, bold_ind))
error_lbl.config(fg= 'red')

##### the button to begin -> next -> end program
btn_one = tk.Button(root, text= 'Begin', command= begin,
                highlightbackground= col1,
                highlightthickness= thick)
btn_one.config(font= (font, size, bold_ind))
btn_one.grid(row=1, column=0, pady=pad)

##### the button to close -> back
btn_two = tk.Button(root, text= 'Close', command= close_root,
                    highlightbackground= col2,
                    highlightthickness= thick)
btn_two.config(font= (font, size, bold_ind))
btn_two.grid(row=1, column=1, pady=pad)

##### monthly charge
## label
monthly_charge_lbl = tk.Label(root, text= 'Monthly Charge:')
monthly_charge_lbl.config(font= (font, size, bold_ind))
monthly_charge_lbl.config(fg= amt_color)
## entry
monthly_charge_entry = tk.Entry(root, width= entry_width)
monthly_charge_entry.config(font= (font, size))

##### additional charge
## label
add_charge_lbl = tk.Label(root, text= 'Additional Charge:')
add_charge_lbl.config(font= (font, size, bold_ind))
add_charge_lbl.config(fg= add_color)
## entry
add_charge_entry = tk.Entry(root, width= entry_width)
add_charge_entry.config(font= (font, size))

##### additional charge notes
## label
add_charge_notes_lbl = tk.Label(root, text= 'Notes on additional charge:')
add_charge_notes_lbl.config(font= (font, size, bold_ind))
add_charge_notes_lbl.config(fg= add_color_2)
## entry
add_charge_notes_entry = tk.Entry(root, width= entry_width)
add_charge_notes_entry.config(font= (font, size))

##### additional charge 2
## label
add_charge_lbl_2 = tk.Label(root, text= 'Additional Charge 2:')
add_charge_lbl_2.config(font= (font, size, bold_ind))
add_charge_lbl_2.config(fg= add_color)
## entry
add_charge_entry_2 = tk.Entry(root, width= entry_width)
add_charge_entry_2.config(font= (font, size))

##### additional charge notes
## label
add_charge_notes_lbl_2 = tk.Label(root, text= 'Notes on additional charge 2:')
add_charge_notes_lbl_2.config(font= (font, size, bold_ind))
add_charge_notes_lbl_2.config(fg= add_color_2)
## entry
add_charge_notes_entry_2 = tk.Entry(root, width= entry_width)
add_charge_notes_entry_2.config(font= (font, size))

##### my notes
## label
my_notes_lbl = tk.Label(root, text= 'My notes:')
my_notes_lbl.config(font= (font, size, bold_ind))
my_notes_lbl.config(fg= notes_color)
## entry
my_notes_entry = tk.Entry(root, width= entry_width)
my_notes_entry.config(font= (font, size))

##### reminders
## label
cust_reminder_lbl = tk.Label(root, text= 'Set Customer Reminder:')
cust_reminder_lbl.config(font= (font, size, bold_ind))
cust_reminder_lbl.config(fg= cust_remind_color)
## entry
cust_reminder_entry = tk.Entry(root, width= entry_width)
cust_reminder_entry.config(font= (font, size))

##### include check button
var = tk.IntVar()
chk = tk.Checkbutton(root, text='Do Not Include?', variable=var,
            selectcolor= 'blue', command= on_check, fg= off_color)
chk.config(font= (font, size, bold_ind))
# var.get()

##### main loop
root.mainloop()

# check if we completed the main program all the way through or not
# - if we did not, then exit with an error
if final_end == False:
    exit('Failed to complete full program.')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# INITIALIZE DOC WRITING

# ADD DATA i.e. total column, and current date for output doc
# create total column
tabl['TOTAL'] = tabl['MONTHLY_CHARGE'] + tabl['ADD_CHARGE'] + tabl['ADD_CHARGE_2']
# add included
tabl['INCLUDED'] = included

# current date
DATE = datetime.datetime.today().strftime('%B %d, %Y')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# HELPER FUNCTION to grab appropriate data

# grab a row from the table and process it for data to place in doc
def grab_data(row):
    # global data
    global data

    # convert to dict
    data = dict(tabl.iloc[row])
    # add dates
    data['DATE'] = DATE
    data['MONTH'] = MONTH
    data['YEAR'] = YEAR
    # if no additional charge then show nothing, else format float
    if data['ADD_CHARGE'] == 0:
        data['ADD_CHARGE'] = ''
    else:
        data['ADD_CHARGE'] = '${:,.2f}'.format(data['ADD_CHARGE'])

    # if no additional charge then show nothing, else format float
    if data['ADD_CHARGE_2'] == 0:
        data['ADD_CHARGE_2'] = ''
    else:
        data['ADD_CHARGE_2'] = '${:,.2f}'.format(data['ADD_CHARGE_2'])

    # format floats
    data['MONTHLY_CHARGE'] = '${:,.2f}'.format(data['MONTHLY_CHARGE'])
    data['TOTAL'] = '${:,.2f}'.format(data['TOTAL'])
    return

##### set data originally to hold nothining
data = None

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
print("Writing files....")

### first files
doc_pp = DocxTemplate(fpage)
doc_gg = DocxTemplate(fpage)
invoice_doc_path_pp = sub_path_pp+'/pp_invoice_'+MONTH+'_'+YEAR+'.docx'
invoice_doc_path_gg = sub_path_gg+'/gg_invoice_'+MONTH+'_'+YEAR+'.docx'
doc_pp.save(invoice_doc_path_pp)
doc_gg.save(invoice_doc_path_gg)

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# CONCATENATE DOC through loop

##### 'rp' indicator and values --- specific use
rp_inc = 0 # increment
rp_ind = False # rp process completion indicator
rp_months = []
rp_adds = []
rp_notes = []
rp_adds_2 = []
rp_notes_2 = []
#####

# loop
for i in range(N):
    # progress print
    print(i)

    # do not include the current row
    if included[i] == 0:
        continue # skip it

    ##### create temp doc

    # grab data and process it
    grab_data(i)

    ### temp doc
    # rp conditionals
    if tabl['RP_INDICATOR'][i] == 1:
        # increment
        rp_inc += 1
        # have we seen three?
        if rp_inc == 3:
            # first append
            rp_months.append(data['MONTHLY_CHARGE'])
            rp_adds.append(data['ADD_CHARGE'])
            rp_notes.append(data['ADD_CHARGE_NOTES'])
            rp_adds_2.append(data['ADD_CHARGE_2'])
            rp_notes_2.append(data['ADD_CHARGE_NOTES_2'])
            ### new data
            rm = data['CUST_REMINDER'] # cust reminder
            # save data for processing
            data = {}
            data['DATE'] = DATE
            data['MONTH'] = MONTH
            data['YEAR'] = YEAR
            data['M1'] = rp_months[0]
            data['M2'] = rp_months[1]
            data['M3'] = rp_months[2]
            data['A1'] = rp_adds[0]
            data['A2'] = rp_adds[1]
            data['A3'] = rp_adds[2]
            data['A1_NOTES'] = rp_notes[0]
            data['A2_NOTES'] = rp_notes[1]
            data['A3_NOTES'] = rp_notes[2]
            data['A1_2'] = rp_adds_2[0]
            data['A2_2'] = rp_adds_2[1]
            data['A3_2'] = rp_adds_2[2]
            data['A1_NOTES_2'] = rp_notes_2[0]
            data['A2_NOTES_2'] = rp_notes_2[1]
            data['A3_NOTES_2'] = rp_notes_2[2]
            data['CUST_REMINDER'] = rm
            ### loop for total sum
            total_sum = 0
            # month charge
            for ele in rp_months:
                if ele != '':
                    total_sum += float(ele.strip('$'))
            # add charge
            for ele in rp_adds:
                if ele != '':
                    total_sum += float(ele.strip('$'))
            # add charge 2
            for ele in rp_adds_2:
                if ele != '':
                    total_sum += float(ele.strip('$'))
            data['TOTAL'] = '${:,.2f}'.format(total_sum)
            rp_ind = True # finished the 'rp' process, note it
        else:
            # nothing to do yet but to append the data
            rp_months.append(data['MONTHLY_CHARGE'])
            rp_adds.append(data['ADD_CHARGE'])
            rp_notes.append(data['ADD_CHARGE_NOTES'])
            rp_adds_2.append(data['ADD_CHARGE_2'])
            rp_notes_2.append(data['ADD_CHARGE_NOTES_2'])
            continue

    ### temp doc
    if rp_ind: # rp stuff
        temp = DocxTemplate(invoice_template_mult)
        rp_ind = False # revert rp process completion indicator for next round
    elif tabl['Z_INDICATOR'][i] == 1: # Z stuff
        if tabl['DIRECTION'][i].strip() == 'PP':
            temp = DocxTemplate(invoice_template_z)
        else:
            temp = DocxTemplate(invoice_template_z_gg)
    elif tabl['V_INDICATOR'][i] == 1: # v stuff
        temp = DocxTemplate(invoice_template_v)
    else: # default
        temp = DocxTemplate(invoice_template)
    # render the doc
    temp.render(data)

    # if no email indicator, merge to paper invoice
    if tabl['EMAIL_IND'][i] == 0:
        ### merge docs
        # check for DIRECTION
        if tabl['DIRECTION'][i].strip() == 'PP':
            # path
            doc_path = invoice_doc_path_pp
        else:
            # path
            doc_path = invoice_doc_path_gg
        # load master doc
        doc = Document(doc_path)
        # add page break
        doc.add_page_break()
        # compose it
        composer = Composer(doc)
        # merge it with the temp doc
        composer.append(temp)

        # save the final output
        composer.save(doc_path)
    else:
        # check for DIRECTION
        if tabl['DIRECTION'][i].strip() == 'PP':
            # path
            temp_emails_path = emails_path_pp+'/'+tabl['EMAIL_ADDRESS'][i].strip()
        else:
            # path
            temp_emails_path = emails_path_gg+'/'+tabl['EMAIL_ADDRESS'][i].strip()
        # make directory
        os.mkdir(temp_emails_path)
        # temp email paths word doc file name
        temp_emails_path_doc = temp_emails_path+'/invoice_'+MONTH+'_'
        temp_emails_path_doc += '_'.join(tabl['FIRST'][i].strip().lower().split())
        temp_emails_path_doc += '_'
        temp_emails_path_doc += tabl['LAST'][i].strip().lower()
        temp_emails_path_doc += '.docx'
        # save doc
        temp.save(temp_emails_path_doc)

print("Files written.")

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# CREATE EXCEL SHEET for output
# order output table
tabl['AMT_PAID'] = [0]*N
tabl['AMT_LEFT'] = ['']*N
tabl['RECORD'] = [':::::']*N
tabl['REC_DATE'] = [':::::']*N
tabl['OBT_DATE'] = [':::::']*N
tabl['PAY_NOTES'] = [':::::']*N
tabl['CLR_DATE'] = [':::::']*N
tabl = tabl[['DIRECTION','FIRST','LAST','TOTAL',
            'AMT_PAID', 'AMT_LEFT','RECORD', 'REC_DATE', 'OBT_DATE', 'PAY_NOTES', 'CLR_DATE',
            'STREET_ADDRESS','CITY_ADDRESS','MY_NOTES','MONTHLY_CHARGE',
            'ADD_CHARGE','ADD_CHARGE_NOTES','ADD_CHARGE_2','ADD_CHARGE_NOTES_2',
            'CUST_REMINDER','INCLUDED']]

# letters
letters = ['A','B','C','D','E','F','G','H','I','J','K','L','M',
           'N','O','P','Q','R','S','T','U','V','W','X','Y','Z']
let_index = 0

# taken from:
# https://xlsxwriter.readthedocs.io/example_pandas_column_formats.html
# Create a Pandas Excel writer using XlsxWriter as the engine
invoice_data = path+'/data_'+MONTH+'_'+YEAR+'.xlsx' # file path
writer = pd.ExcelWriter(invoice_data, engine='xlsxwriter')
# Convert the dataframe to an XlsxWriter Excel object.
tabl.to_excel(writer, sheet_name='Sheet1', index= False)
# Get the xlsxwriter workbook and worksheet objects.
workbook  = writer.book
worksheet = writer.sheets['Sheet1']
# Set the column width and format.
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # direction
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # first
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # last
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # total
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # amount paid
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # amount left
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # record
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # rec date
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # obt date
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # pay notes
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) # clear date
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*1.5, None) # street
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*1.5, None) # city
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*2.5, None) # my notes
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # monthly charge
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # additional charge
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*2.5, None) # add charge notes
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # additional charge 2
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18*2.5, None) # add charge notes 2
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 18, None) # cust reminder
let_index += 1
worksheet.set_column(letters[let_index]+':'+letters[let_index], 9, None) #included

# Close the Pandas Excel writer and output the Excel file.
writer.save()

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# CREATE EXCEL SHEET for shorter output
# 'short' path
short_path = path+'/short_data_'+MONTH+'_'+YEAR+'.xlsx'
# create shorter data
short_tabl = tabl[['FIRST','LAST','STREET_ADDRESS']].copy()
# add month service column
short_tabl[MONTH+' '+YEAR] = tabl['TOTAL'].copy()
# add recrods column
short_tabl['Records'] = ['']*N

# create excel output
short_writer = pd.ExcelWriter(short_path, engine='xlsxwriter')
# Convert the dataframe to an XlsxWriter Excel object.
short_tabl.to_excel(short_writer, sheet_name='Sheet1', index= False)
# Get the xlsxwriter workbook and worksheet objects.
short_workbook  = short_writer.book
short_workbook = short_writer.sheets['Sheet1']
# Set the column width and format.
short_workbook.set_column('A:A', 15, None) # first
short_workbook.set_column('B:B', 15, None) # last
short_workbook.set_column('C:C', 25, None) # street
short_workbook.set_column('D:D', 15, None) # total
short_workbook.set_column('E:E', 15, None) # records
# MARGINS
short_workbook.set_margins(left=0.1, right=0.1, top=0.1, bottom=0.1)
# Close the Pandas Excel writer and output the Excel file.
short_writer.save()

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
## main program done
print('Main program done.')

# total total sum
print('Total: ${:,.2f}'.format(tabl['TOTAL'].values.sum()))

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# ## RUN AUX PROGRAM
# string_exe = 'python code/aux.py ' + .......
# os.system(string_exe)

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------

print('Complete!!!!!!')

# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------


from app import app, lm
from flask import Flask, render_template, request, url_for, flash, redirect
from datetime import datetime
from db import DBAccess
from flask.ext.login import login_user, logout_user, login_required, current_user
from forms import LoginForm, RegisterForm
from user import User
import json

dbManager = DBAccess()

@app.route("/", methods=['GET', 'POST'])
@login_required
def main():
    result = None
    if request.method == 'POST':
        form_data = request.form
        result = dbManager.log_projects(current_user.username, form_data)
        if result == "Hours updated successfully":
            flash(result, 'success')
        elif result == "No projects entered":
            flash(result, 'error')
    sorted_user_projects = {}
    total_hours = 0
    hours_required = 0
    date = datetime.now()
    year = str(date.year)
    month = date.strftime("%B")
    user = load_user(current_user.username)
    user_role = user.get_role()
    user_dict = dbManager.select_user(user.username)
    if year in user_dict and month in user_dict[year]:
        user_projects = user_dict[year][month]['projects']
        for x, y in user_projects.items():
            project = dbManager.select_project(x)
            sorted_user_projects[project['name']] = user_projects[x]
            print(sorted_user_projects)
        if 'total_hours' in user_dict[year][month]:
            total_hours = int(user_dict[year][month]['total_hours'])
        if 'hours_required' in user_dict[year][month]:
            hours_required = int(user_dict[year][month]['hours_required'])
    projects = dbManager.select_all_projects()
    return render_template(
        'staff-returns.html',
        title='home',
        month=month,
        year=year,
        projects=projects,
        user_role=user_role,
        user_projects=sorted_user_projects,
        total_hours=total_hours,
        hours_required=hours_required,
        result=result)

@app.route('/budget-tracking', methods=['GET'])
@login_required
def project_management():
    user = load_user(current_user.username)
    user_role = user.get_role()
    users = dbManager.select_all_users();
    date = datetime.now()
    year = str(date.year)
    month = date.strftime("%B")
    monthNames = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
    if user_role == 'Admin' or user_role == 'Delivery Manager':
        projects = dbManager.select_all_projects()
        return render_template('budget-tracking.html',
                               title='budget-tracking',
                               user_role=user_role,
                               projects=projects,
                               monthNames=monthNames,
                               current_month=month,
                               current_year=year,
                               users=users)
    else:
        return render_template('404.html'), 404

@app.route('/user-management', methods=['GET', 'POST'])
@login_required
def user_management():
    result = None
    if request.method == 'POST':
        form_data = request.form
        result = dbManager.update_user(form_data)
        if result == "User updated successfully":
            flash(result, 'success')
    user = load_user(current_user.username)
    user_role = user.get_role()
    if user_role == 'Admin':
        days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]
        roles = dbManager.select_all_roles()
        paygrades = dbManager.select_paygrades()
        users = dbManager.select_all_users()
        return render_template('user-management.html',
                               days=days,
                               paygrades=paygrades,
                               roles=roles,
                               title='user-management',
                               users=users,
                               user_role=user_role,
                               result=result)
    else:
        return render_template('404.html'), 404

@app.route('/load-user/<user_id>', methods=['GET'])
@login_required
def load_user_data(user_id=None):
    user = load_user(current_user.username)
    user_role = user.get_role()
    if user_role == 'Admin' or user_role == 'Delivery Manager':
        user = dbManager.select_user(user_id)
        user.pop('password')
        return json.dumps(user)

@app.route('/load-project/<project_name>', methods=['GET'])
@login_required
def load_project(project_name=None):
    user = load_user(current_user.username)
    user_role = user.get_role()
    if user_role == 'Admin' or user_role == 'Delivery Manager':
        project_id = dbManager.select_project_id_from_name(project_name)
        project = dbManager.select_project(project_id)
        return json.dumps(project)

@app.route('/login', methods=['GET', 'POST'])
def login():
    form = LoginForm(request.form)
    if request.method == 'POST' and form.validate_on_submit():
        user = dbManager.select_user(str.upper(form.username.data))
        if user != 'User not found':
            if user and User.validate_login(user['password'], form.password.data):
                user_obj = User(user['_id'])
                if form.remember.data:
                    login_user(user_obj, remember=True)
                else:
                    login_user(user_obj)
                flash("Logged in successfully", category='success')
                return redirect(request.args.get("next") or url_for('main'))
            else:
                flash("Wrong username or password", category='error')
        else:
            flash("Wrong username or password", category='error')
    return render_template('login.html', title='login', form=form)

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('login'))

@app.route('/register', methods=['GET', 'POST'])
def register():
    if current_user.is_authenticated():
        logout_user()
    form = RegisterForm(request.form)
    days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]
    if request.method == 'POST':
        if form.validate_on_submit() == False:
            flash('All fields are required.', category='error')
        else:
            result = dbManager.insert_user(form)
            if result == 'Success':
                flash('Account created', category='success')
                return redirect(url_for('login'))
            else:
                flash('Username taken', category='error')
    return render_template('register.html', title='register', form=form, days=days)

@app.errorhandler(404)
def page_not_found(e):
    if current_user.is_authenticated():
        user = load_user(current_user.username)
        user_roles = user.get_role()
        return render_template('404.html', user_roles=user_roles), 404
    else:
        return render_template('404.html'), 404

@lm.user_loader
def load_user(user_id):
    u = dbManager.select_user(user_id)
    if not u:
        return None
    return User(u['_id'])

if __name__ == "__main__":
    app.run(debug=True, host='0.0.0.0', port=8080)


import sys

test_Data = {
    'A variable must be assigned a value before it can be used?': 'yes',
    'What keyword is used to start function in Python?': 'def',
    'Is python compiled language?': 'no',
    'In python3 what does "//" operator do?': 'integer division',
    'If we have  a list x[1,2,3,4], how do we print the first element?': 'print(x[0])',
    'What is the binary number 00001010 indecimal?': '10',
    'What symbol can you use to comment out one line of code?': '#',
    'How would you cast the string variable "a" that is equal to "2" into the integer 2?': 'int(a)',
    'What keyword would you use to add an alternative condition to an if statement?': 'elif'}

wrong_Answers = 0
for question, right_Answer in test_Data.items():

        while True:
            print(question)
            try:
                answer = raw_input()
            except NameError:
                answer = input()

            if right_Answer != answer.lower():
                wrong_Answers += 1
                print('Sorry you are wrong, try once more...\n')
            else:
                print('Good answer! You are right!')
                break

print('Congradulations! Game is over ... \nWrong answers: ' + str(wrong_Answers))


# Frame rate settings
frames_per_second = 60
frame_rate = 1.0 / frames_per_second

# Movement settings
walk_speed = 0.1
mouse_sensitivity = 0.1

# Texture settings
texture_size = 64

# Window settings
window_width  = 1280
window_height = 900


import filecmp
import glob
import json
import logging
import os
import re
import shutil
import tarfile

import pandas as pd
import requests

import mistune
from bs4 import BeautifulSoup
from utils import file_ops

logging.getLogger().setLevel(logging.INFO)

def __parse_downloaded_model_file_list_response(response):
    html = mistune.markdown(response.text)
    soup = BeautifulSoup(html)
    link_nodes = soup.find_all("a")

    data = []
    for link in link_nodes:
        if "http://download.tensorflow.org/models/object_detection/" in link.attrs["href"]:
            model_name = link.text
            model_name = model_name.replace("☆", "")
            model_name = model_name.strip()

            model_url = link.attrs["href"]
            model_file_name = model_url.split("/")[-1]
            model_folder_name = os.path.splitext(os.path.basename(model_file_name))[0]
            model_folder_name = os.path.splitext(os.path.basename(model_folder_name))[0]
            try:
                model_release_date = re.search(r"\d{4}_\d{2}_\d{2}", model_file_name).group()
            except:
                model_release_date = None

            data.append(
                (model_release_date, model_folder_name, model_file_name, model_url, model_name)
            )

    logging.info(f"Parsed all model data from the webpage {response.url}")

    return pd.DataFrame(
        data,
        columns=[
            "model_release_date",
            "model_folder_name",
            "model_file_name",
            "model_url",
            "model_name",
        ],
    )

def validate_reference_model_list_exist_or_create(
    base_model_csv, positive_downstream, negative_downstream
):
    if file_ops.file_exist(base_model_csv):
        return positive_downstream
    else:
        return negative_downstream

def download_reference_model_list_as_csv(url, base_model_csv):
    try:
        response = requests.get(url, allow_redirects=True)
        new_models_reference_df = __parse_downloaded_model_file_list_response(response)
        new_models_reference_df.to_csv(base_model_csv)
        logging.info("Model list data saved to csv file")
    except requests.exceptions.RequestException as e:
        logging.error(f"An error occurred while downloading the file from {url}")

def download_and_extract_base_model(base_model_csv, base_model_folder, base_model_list=None):

    models_df = pd.read_csv(base_model_csv)
    models_subset = models_df[["model_folder_name", "model_file_name", "model_url", "model_name"]]

    if base_model_list is not None:
        models_subset = models_subset[models_df.model_name.isin(base_model_list)]

    models = [tuple(x) for x in models_subset.values]
    subfolders = file_ops.get_subfolders_names_in_directory(base_model_folder)

    for model_folder_name, model_file_name, model_url, model_name in models:
        if not model_folder_name in subfolders:
            logging.info(f"Model {model_folder_name} not found ")
            os.mkdir(os.path.join(base_model_folder, model_folder_name))
            try:
                response = requests.get(model_url, stream=True)
                logging.info(f"Downloading {model_url} .....")
                if response.status_code == 200:
                    tar_file = os.path.join(base_model_folder, model_file_name)
                    with open(tar_file, "wb") as f:
                        f.write(response.raw.read())

                    logging.info(f"Extracting {tar_file} .....")
                    shutil.unpack_archive(tar_file, os.path.join(base_model_folder))
                    os.remove(tar_file)
            except requests.exceptions.RequestException as e:
                logging.error(f"An error occurred while downloading the file from {model_url}")
        else:
            logging.info("All base models are already present")

def compare_label_map_file(base_tf_record_folder, video_source):

    subfolders = file_ops.get_directory_subfolders_subset(base_tf_record_folder, video_source)

    if len(subfolders) > 1:
        label_maps = []
        for subfolder in subfolders:
            label_maps.append(glob.glob(subfolder + "*.pbtxt")[0])
        reference_label_map = label_maps[0]
        labelmap_match = True
        for label_map in label_maps:
            print(label_map)
            logging.info(f"Reference File: {reference_label_map}")
            if filecmp.cmp(label_map, reference_label_map):
                print(f"[ MATCH ] | LabelMap:{label_map} ")
            else:
                print(f"[ FAILED ] | LabelMap:{label_map} ")
                labelmap_match = False

        return labelmap_match
    else:
        logging.warn(f"There were not enough dataset to compare i.g : Less than two")

def __create_training_folder_subtree(
    training_data_folder, video_source, object_names, execution_date, **kwargs
):

    # Base Folder
    data_folder = os.path.join(training_data_folder, "data")
    model_folder = os.path.join(training_data_folder, "model")
    training_folder = os.path.join(training_data_folder, "training")

    images_folder = os.path.join(data_folder, "images")
    annotations_folder = os.path.join(data_folder, "annotations")
    xmls_folder = os.path.join(data_folder, "annotations", "xmls")
    tf_record_folder = os.path.join(data_folder, "tf_record")
    base_model_folder = os.path.join(model_folder, "base")
    trained_model_folder = os.path.join(model_folder, "trained")

    train_data_folder = os.path.join(training_folder, "training")
    eval_data_folder = os.path.join(training_folder, "evaluation")

    training_folders = {
        "base_folder": training_data_folder,
        "data_folder": data_folder,
        "model_folder": model_folder,
        "training_folder": training_folder,
        "images_folder": images_folder,
        "annotations_folder": annotations_folder,
        "xmls_folder": xmls_folder,
        "tf_record_folder": tf_record_folder,
        "base_model_folder": base_model_folder,
        "trained_model_folder": trained_model_folder,
        "train_data_folder": train_data_folder,
        "eval_data_folder": eval_data_folder,
    }

    data_folders = [
        training_folders["images_folder"],
        training_folders["xmls_folder"],
        training_folders["tf_record_folder"],
        training_folders["base_model_folder"],
        training_folders["trained_model_folder"],
        training_folders["train_data_folder"],
        training_folders["eval_data_folder"],
    ]

    for folder in data_folders:
        os.makedirs(folder)

    ti = kwargs["ti"]
    ti.xcom_push(key="training_folders", value=training_folders)
    logging.info("Training folder subtree has been created successfully")

def create_training_folder(
    base_training_folder, tf_record_folder, video_source, execution_date, base_model, **kwargs,
):
    subfolders = file_ops.get_directory_subfolders_subset(tf_record_folder, video_source)

    object_names_set = set()
    for subfolder in subfolders:
        folder_name = os.path.basename(os.path.normpath(subfolder))
        if folder_name.startswith(video_source):

            object_names = subfolder.split("_")[1]
            object_names_set.update(object_names.split("-"))

    object_names = "-".join(list(object_names_set))

    training_folder = os.path.join(
        base_training_folder, f"{video_source}_{object_names}_{base_model}_{execution_date}"
    )

    __create_training_folder_subtree(
        training_folder, video_source, object_names, execution_date, **kwargs
    )

    ti = kwargs["ti"]
    ti.xcom_push(key="training_folder", value=training_folder)

def copy_labelbox_output_data_to_training(
    labelbox_output_data_folder,
    tf_record_folder,
    video_source,
    base_model,
    airflow_base_folder,
    gcp_base_bucket_url,
    **kwargs,
):
    ti = kwargs["ti"]
    training_folders = ti.xcom_pull(
        key="training_folders", task_ids=f"create_training_folder_tree_{video_source}_{base_model}"
    )

    filtered_subfolders = file_ops.get_directory_subfolders_subset(
        labelbox_output_data_folder, video_source
    )

    for subfolder in filtered_subfolders:
        subfolders = file_ops.get_subfolders_in_directory(subfolder)

        subfolders = [
            image_subfolder for image_subfolder in subfolders if image_subfolder.endswith("images")
        ]

        images_folder = subfolders[0]

        file_ops.copy_files_from_folder(images_folder, training_folders["images_folder"])

        logging.info("Image files copy completed")

        file_ops.copy_xml_files_from_folder(subfolder, training_folders["xmls_folder"])

        logging.info("XML files copy completed")

    subfolders = file_ops.get_directory_subfolders_subset(tf_record_folder, video_source)

    labelmap_files = []
    trainval_files = []
    tf_record_files = []
    for subfolder in subfolders:
        labelmap_files.extend(glob.glob(subfolder + "/*.pbtxt"))
        trainval_files.extend(glob.glob(subfolder + "/*.txt"))
        tf_record_files.extend(glob.glob(subfolder + "/*.record"))

    labelmap_file = f"{training_folders['annotations_folder']}/labelmap.pbtxt"
    trainval_file = f"{training_folders['annotations_folder']}/trainval.txt"

    with open(labelmap_file, "w") as outfile:
        with open(labelmap_files[0]) as infile:
            for line in infile:
                outfile.write(line)

    with open(trainval_file, "w") as outfile:
        for trainval_file in trainval_files:
            with open(trainval_file, "r") as infile:
                shutil.copyfileobj(infile, outfile)

    train_tf_records = []
    val_tf_records = []

    for tf_record_file in tf_record_files:

        if (tf_record_file).endswith("train.record"):
            train_tf_records.append(tf_record_file)
        else:
            val_tf_records.append(tf_record_file)

        shutil.copy2(tf_record_file, training_folders["tf_record_folder"])

    local_training_files = {
        "label_map_file": labelmap_file,
        "trainval_file": trainval_file,
        "train_tf_records": train_tf_records,
        "val_tf_records": val_tf_records,
    }

    gcp_training_files = {
        "label_map_file": labelmap_file.replace(airflow_base_folder, gcp_base_bucket_url),
        "trainval_file": trainval_file.replace(airflow_base_folder, gcp_base_bucket_url),
        "train_tf_records": [
            tf_record.replace(airflow_base_folder, gcp_base_bucket_url)
            for tf_record in train_tf_records
        ],
        "val_tf_records": [
            tf_record.replace(airflow_base_folder, gcp_base_bucket_url)
            for tf_record in val_tf_records
        ],
    }

    ti = kwargs["ti"]
    ti.xcom_push(key="local_training_files", value=local_training_files)
    ti.xcom_push(key="gcp_training_files", value=gcp_training_files)

def copy_base_model_to_training_folder(
    base_model_folder,
    base_model_csv,
    base_model,
    video_source,
    airflow_base_folder,
    gcp_base_bucket_url,
    **kwargs,
):
    ti = kwargs["ti"]
    training_folders = ti.xcom_pull(
        key="training_folders", task_ids=f"create_training_folder_tree_{video_source}_{base_model}"
    )

    gcp_training_files = ti.xcom_pull(
        key="gcp_training_files",
        task_ids=f"copy_labelbox_output_data_to_training_folder_{video_source}_{base_model}",
    )

    base_models_df = pd.read_csv(base_model_csv)

    model_df = base_models_df.loc[base_models_df["model_name"] == base_model]

    base_model_folder_name = model_df.iloc[0]["model_folder_name"]

    model_folder = os.path.join(base_model_folder, base_model_folder_name)

    file_ops.copy_files_from_folder(model_folder, training_folders["base_model_folder"])

    logging.info("Successfully copied all base model file to training folder")

    pipeline_file = os.path.join(training_folders["base_model_folder"], "pipeline.config")

    os.remove(pipeline_file)

    model_checkpoint = os.path.join(training_folders["base_model_folder"], "model.ckpt")

    logging.info("Successfully removed pipeline.config file")

    gcp_training_files["model_checkpoint"] = model_checkpoint.replace(
        airflow_base_folder, gcp_base_bucket_url
    )

    ti = kwargs["ti"]
    ti.xcom_push(key="gcp_training_files", value=gcp_training_files)

def generate_model_config(
    video_source, base_model, model_config_template, num_classes, **kwargs,
):

    ti = kwargs["ti"]
    training_folders = ti.xcom_pull(
        key="training_folders", task_ids=f"create_training_folder_tree_{video_source}_{base_model}"
    )
    gcp_training_files = ti.xcom_pull(
        key="gcp_training_files",
        task_ids=f"copy_base_model_to_training_folder_{video_source}_{base_model}",
    )

    # Replacing placeholders in airflow variables for values
    model_config_template = re.sub("NUM_CLASSES", str(num_classes), model_config_template)
    model_config_template = re.sub(
        "PRE_TRAINED_MODEL_CHECKPOINT_PATH",
        gcp_training_files["model_checkpoint"],
        model_config_template,
    )
    model_config_template = re.sub(
        "LABEL_MAP_PATH", gcp_training_files["label_map_file"], model_config_template
    )
    model_config_template = re.sub(
        "TRAIN_TF_RECORD_PATHS", str(gcp_training_files["train_tf_records"]), model_config_template
    )
    model_config_template = re.sub(
        "VAL_TF_RECORD_PATHS", str(gcp_training_files["val_tf_records"]), model_config_template
    )

    try:
        config_file = os.path.join(training_folders["base_folder"], "pipeline.config")
        with open(config_file, "w") as outfile:
            outfile.write(model_config_template)
        logging.info("Model config file has been created succesfully")
    except IOError as e:
        logging.error(
            "An error has been raised while trying to save the model config to a file on disk"
        )
        raise e

def archiving_training_folder(training_archiving_path, video_source, base_model, **kwargs):
    ti = kwargs["ti"]
    training_folders = ti.xcom_pull(
        key="training_folders", task_ids=f"create_training_folder_tree_{video_source}_{base_model}"
    )
    print(training_archiving_path)
    print(training_folders)
    folder_name = file_ops.get_folder_name(training_folders["base_folder"])
    archive_file = os.path.join(training_archiving_path, f"{folder_name}.tar")

    with tarfile.open(archive_file, "w:gz") as tar:
        tar.add(training_folders["base_folder"], arcname=folder_name)

    logging.info(
        f"Successfully created archive of training folder : {training_folders['base_folder']}"
    )

def remove_raw_images_and_annotations_from_training_folder(
    video_source, base_model, gcp_base_bucket_url, airflow_trainable_folder, **kwargs
):
    ti = kwargs["ti"]
    training_folders = ti.xcom_pull(
        key="training_folders", task_ids=f"create_training_folder_tree_{video_source}_{base_model}"
    )
    shutil.rmtree(training_folders["xmls_folder"])
    shutil.rmtree(training_folders["images_folder"])
    os.remove(os.path.join(training_folders["annotations_folder"], "trainval.txt"))

    training_folder = training_folders["base_folder"]

    prepared_cmd = f"gsutil -m cp -r {training_folder} {gcp_base_bucket_url}"

    training_folder_name = file_ops.get_folder_name(training_folder)

    json_data = {}
    json_data["gcp_url"] = f"{gcp_base_bucket_url}/{training_folder_name}"

    json_file = os.path.join(airflow_trainable_folder, f"{training_folder_name}.json")
    with open(json_file, "w") as outfile:
        json.dump(json_data, outfile, indent=4)

    ti.xcom_push(key="gcp_copy_cmd", value=prepared_cmd)

    logging.info("Successfully deleted useless files for training")

def clean_up_post_training_prep(folders, **kwargs):

    print(folders)
    for folder in folders:
        folder_content = glob.glob(f"{folder}/*")
        print(folder_content)

        for content in folder_content:
            if os.path.isdir(content):
                logging.info(f"Deleting Folder:{content}")
                shutil.rmtree(content)
            else:
                if not content.endswith(".gitignore"):
                    logging.info(f"Deleting File:{content}")
                    os.remove(content)


import sys, json
print(sys.path)
import numpy as np
from sklearn.externals import joblib

data = json.loads(sys.argv[1])
data = np.array(data['data'])
sav = joblib.load('/var/www/html/house_price.sav')
pred = sav.predict(data.reshape(1,-1))
result = format(int(round(pred[0],0)),',')
print(result)

mL imp/ML Project 1/ML Project 1/house_price_predict.py


from django.shortcuts import render
from rest_framework.response import Response
from rest_framework import generics
from rest_framework.views import APIView
import datetime

from rest_framework import status

from loggingapp.models import PageView
from loggingapp.serializers import PageViewSerializer
from common.views import GenericCRUDView
from party.models import Party
from partner.models import Partner

from django.db.models import Count, Min, Max

# Create your views here.

# top level uri: /session-logs/

# /page-views/
class PageViewCRUD(GenericCRUDView):
  queryset = PageView.objects.all()
  serializer_class = PageViewSerializer

  def get(self, request, format=None):
    params = request.GET
    obj = self.get_queryset()
    if 'startDate' in params:
      obj = obj.filter(pageViewDate__gte=params['startDate'])
    if 'endDate' in params:
      obj = obj.filter(pageViewDate__lte=params['endDate'])
    serializer = self.serializer_class(obj, many=True)
    return Response(serializer.data)

  def post(self,request, format=None):
    serializer = self.serializer_class(data=request.data)
    if serializer.is_valid():
      serializer.save()
      return Response(serializer.data, status=status.HTTP_201_CREATED)
    return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

  def delete(self, request, format=None):
    return Response({'message':'delete is not enabled for Page View'}, status=status.HTTP_400_BAD_REQUEST)

  def update(self, request):
    return Response({'message':'update is not enabled for Page View'}, status=status.HTTP_400_BAD_REQUEST)

# /sessions/counts/
class SessionCountView(generics.GenericAPIView):

  def get(self, request, format=None):
    startDate = request.GET.get('startDate')
    endDate = request.GET.get('endDate')
    ip = request.GET.get('ip')
    partyId = request.GET.get('partyId')

    filters = {}
    if startDate:
      filters['pageViewDate__gte']=startDate
    if endDate:
      filters['pageViewDate__lte']=endDate
    if ip:
      filters['ip']=ip
    if partyId:
      filters['partyId']=partyId

    distinctSessions = PageView.objects.values('sessionId').distinct().filter(**filters)
    return Response({'count':len(distinctSessions)})


# -*- coding: utf-8 -*-
"""RegressionTorchModel Base class for model with no cell specific parameters"""

import matplotlib.pyplot as plt
# +
import numpy as np
import pandas as pd

from cell2location.models.torch_model import TorchModel

class RegressionTorchModel(TorchModel):
    r"""RegressionTorchModel Base class for model with no cell specific parameters

    :param sample_col: str with column name in cell2covar that denotes sample
    :param cell2covar: pd.DataFrame with covariates in columns and cells in rows, rows should be named.
    :param cell_state_mat: Pandas data frame with gene signatures - genes in row, cell states or factors in columns
    :param X_data: Numpy array of gene expression (cols) in spatial locations (rows)
    :param learning_rate: ADAM learning rate for optimising Variational inference objective
    :param n_iter: number of training iterations
    :param total_grad_norm_constraint: gradient constraints in optimisation
    """

    def __init__(
            self,
            sample_id,
            cell2covar: pd.DataFrame,
            X_data: np.ndarray,
            data_type='float32',
            n_iter=200000,
            learning_rate=0.001,
            total_grad_norm_constraint=200,
            verbose=True,
            var_names=None, var_names_read=None,
            obs_names=None, fact_names=None,
            minibatch_size=None, minibatch_seed=[41, 56, 345],
            phi_hyp_prior=None, prior_eps=1e-8,
            nb_param_conversion_eps=1e-8,
            use_cuda=False,
            use_average_as_initial_value=True,
            stratify_cv=None
    ):

        ############# Initialise parameters ################
        # convert covariates to binary matrix
        # test for column types, get dummies for categorical / character, and just copy over continous
        cell2covar_df = pd.get_dummies(cell2covar.loc[:, ~cell2covar.columns.isin([sample_id])])
        cell2sample_df = pd.get_dummies(cell2covar[[sample_id]])
        cell2sample_covar_df = pd.concat([cell2sample_df, cell2covar_df], axis=1)

        fact_names = cell2sample_covar_df.columns
        n_fact = cell2sample_covar_df.shape[1]

        # extract obs names and sample id
        obs_names = cell2covar.index
        sample_id = cell2covar[sample_id]

        super().__init__(X_data, n_fact,
                         data_type, n_iter,
                         learning_rate, total_grad_norm_constraint,
                         verbose, var_names, var_names_read,
                         obs_names, fact_names, sample_id, use_cuda)

        self.nb_param_conversion_eps = nb_param_conversion_eps

        self.cell_factors_df = None
        self.minibatch_size = minibatch_size
        self.minibatch_seed = minibatch_seed
        self.n_cells_total = self.n_cells
        self.which_sample = self.fact_names.isin(cell2sample_df.columns)
        self.n_samples = np.sum(self.which_sample)
        self.n_covar = self.n_fact - self.n_samples

        self.phi_hyp_prior = phi_hyp_prior
        self.prior_eps = prior_eps

        self.cell2sample_df = cell2sample_df
        self.cell2sample_covar_df = cell2sample_covar_df
        # convert to np.ndarray
        self.cell2sample_mat = cell2sample_df.values
        self.cell2sample_covar_mat = cell2sample_covar_df.values

        # find mean and variance for each gene
        self.gene_mean = (self.X_data + self.prior_eps).mean(0).astype(self.data_type).reshape((1, self.n_genes))
        self.noise_gene_mean = (self.gene_mean / 10).astype(self.data_type).reshape((1, self.n_genes))
        self.prior_gene_mean = np.concatenate([self.noise_gene_mean, self.gene_mean], axis=0)

        self.stratify_cv = stratify_cv

        self.extra_data['cell2sample_covar'] = self.cell2sample_covar_mat

        if use_average_as_initial_value:
            # compute initial value for parameters: cluster averages
            self.cell2sample_covar_sig_mat = self.cell2sample_covar_mat / self.cell2sample_covar_mat.sum(0)
            self.clust_average_mat = np.dot(self.cell2sample_covar_sig_mat.T, self.X_data) + self.prior_eps
            self.clust_average_mat[self.which_sample, :] = self.clust_average_mat[self.which_sample, :] / 10

            # aver = get_cluster_averages(adata_snrna_raw, 'annotation_1') + self.prior_eps
            # variances = get_cluster_variances(adata_snrna_raw, 'annotation_1') + self.prior_eps
            # shape = aver ** 2 / variances
            # shape = shape.mean(1).values
            # overdisp_mean = shape.reshape((1, adata_snrna_raw.shape[1]))
            self.gene_E_mat = None  # np.sqrt(1 / overdisp_mean) # get gene_E ~ Exponential()
        else:
            self.clust_average_mat = None
            self.gene_E_mat = None

    # =====================Other functions======================= #
    def plot_gene_budget(self):

        plt.hist(np.log10(self.samples['post_sample_means']['gene_level'][:, 0]), bins=50)
        plt.xlabel('Gene expression level (hierarchical)')
        plt.title('Gene expression level (hierarchical)')
        plt.tight_layout()

    def sample2df(self, gene_node_name='gene_factors'):
        r""" Export cell factors as Pandas data frames.

        :param node_name: name of the cell factor model parameter to be exported
        :param gene_node_name: name of the gene factor model parameter to be exported
        :return: 8 Pandas dataframes added to model object:
                 .covariate_effects, .covariate_effects_sd, .covariate_effects_q05, .covariate_effects_q95
                 .sample_effects, .sample_effects_sd, .sample_effects_q05, .sample_effects_q95
        """

        # export parameters for covariate effects
        cov_ind = ~ self.which_sample
        self.covariate_effects = \
            pd.DataFrame.from_records(self.samples['post_sample_means'][gene_node_name][cov_ind, :].T,
                                      index=self.var_names,
                                      columns=['mean_' + 'cov_effect_' + i for i in self.fact_names[cov_ind]])

        # export parameters for sample effects
        sample_ind = self.which_sample
        self.sample_effects = \
            pd.DataFrame.from_records(self.samples['post_sample_means'][gene_node_name][sample_ind, :].T,
                                      index=self.var_names,
                                      columns=['mean_' + 'sample_effect' + i for i in self.fact_names[sample_ind]])

    def annotate_cell_adata(self, adata):
        r""" Add covariate and sample coefficients to anndata.var

        :param adata: anndata object to annotate
        :return: updated anndata object
        """

        if self.cell_factors_df is None:
            self.sample2df()

        ### Covariate effect
        # add gene factors to adata
        adata.var[self.covariate_effects.columns] = self.covariate_effects.loc[adata.var.index, :]

        ### Sample effects
        # add gene factors to adata
        adata.var[self.sample_effects.columns] = self.sample_effects.loc[adata.var.index, :]

        return (adata)


#!/usr/bin/env python3

import sys
from subprocess import call

if len(sys.argv) == 1:
    sys.argv=["build_lib_index.py","$library"]

file = sys.argv[1]

fasta = 'lib.fasta'

with open(fasta, 'w') as f:
    with open(file, 'r') as fp:
        ith = 0
        for line in fp:
            ith += 1
            line = line.split(',')
            if ith == 1:
                continue
            if line[1] == '+':
                f.write('>' + str(ith-1) + ':+\\n' + line[0] + '\\n')
            elif line[1] == '-':
                f.write('>' + str(ith-1) + ':-\\n' + line[0] + '\\n')
            else:
                f.write('>' + str(ith-1) + ':na\\n' + line[0]  + '\\n')

call('bowtie-build -f lib.fasta lib', shell=True)


from moabb.datasets.base import BaseDataset

from mne.io import read_raw_edf
from mne.channels import read_montage
import numpy as np

from . import download as dl

UPPER_LIMB_URL = 'https://zenodo.org/record/834976/files/'

class UpperLimb(BaseDataset):
    """Upper Limb motor dataset.

    Upper limb dataset :
    http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0182578

    Consist in 6 upper limb movement, recoded over 2 sessions.
    The first session is motor execution, the second session is imagination.

    """

    def __init__(self, imagined=True, executed=False):
        self.imagined = imagined
        self.executed = executed
        event_id = {"right_elbow_flexion": 1536,
                    "right_elbow_extension": 1537,
                    "right_supination": 1538,
                    "right_pronation": 1539,
                    "right_hand_close": 1540,
                    "right_hand_open": 1541,
                    "rest": 1542}

        n_sessions = int(imagined) + int(executed)
        super().__init__(
            subjects=list(range(1, 16)),
            sessions_per_subject=n_sessions,
            events=event_id,
            code='Upper Limb Imagery',
            interval=[2.5, 5],
            paradigm='imagery',
            doi='10.1371/journal.pone.0182578')

    def _get_single_subject_data(self, subject):
        """return data for a single subject"""

        sessions = []
        if self.imagined:
            sessions.append('imagination')

        if self.executed:
            sessions.append('execution')

        out = {}
        for session in sessions:
            paths = self.data_path(subject, session=session)

            eog = ['eog-l', 'eog-m', 'eog-r']
            montage = read_montage('standard_1005')
            data = {}
            for ii, path in enumerate(paths):
                raw = read_raw_edf(path, montage=montage, eog=eog,
                                   misc=range(64, 96), preload=True,
                                   verbose='ERROR')
                # there is nan in the data
                raw._data[np.isnan(raw._data)] = 0
                data['run_%d' % ii] = raw

            out[session] = data
        return out

    def data_path(self, subject, path=None, force_update=False,
                  update_path=None, verbose=None, session=None):
        if subject not in self.subject_list:
            raise(ValueError("Invalid subject number"))

        paths = []

        if session is None:
            sessions = []
            if self.imagined:
                sessions.append('imagination')

            if self.executed:
                sessions.append('execution')
        else:
            sessions = [session]

        for session in sessions:
            for run in range(1, 11):
                url = f"{UPPER_LIMB_URL}motor{session}_subject{subject}_run{run}.gdf"
                p = dl.data_path(url, 'UPPERLIMB', path, force_update,
                                 update_path, verbose)
                paths.append(p)

        return paths


"""
Module to find out the qualified name of a class.
"""

import ast
import inspect
import types

from collections import defaultdict

__all__ = ['qualname']

_cache = {}
_file_cache = {}

class _Visitor(ast.NodeVisitor):
    def __init__(self):
        super(_Visitor, self).__init__()
        self.stack = []

        # The keys here are line numbers of a method/function
        self.function_qualnames = {}

        # The keys here are unqualified names of classes
        self.class_qualnames = defaultdict(set)

    def current_qualname(self):
        return ".".join(self.stack)

    def visit_FunctionDef(self, node):
        self.stack.append(node.name)
        self.function_qualnames[node.lineno] = self.current_qualname()
        self.stack.append('<locals>')
        self.generic_visit(node)
        self.stack.pop()
        self.stack.pop()

    def visit_ClassDef(self, node):
        self.stack.append(node.name)
        self.class_qualnames[node.name].add(self.current_qualname())
        self.generic_visit(node)
        self.stack.pop()

def _fallback_to_name(obj):
    name = obj.__name__
    if inspect.isclass(obj):
        _cache[obj] = name
    else:
        try:
            obj.__qualname__ = name
        except (AttributeError, TypeError):
            _cache[obj] = name

    return name

def qualname(obj):
    """Find out the qualified name for a class or function."""

    # For Python 3.3+, this is straight-forward.
    # This attribute is also set where possible on functions processed
    # for the first time as a simple cache
    if hasattr(obj, '__qualname__'):
        return obj.__qualname__

    # This is for objects that can't have an attribute set on them
    # (e.g. builtins) and classes (to prevent inheritance issues)
    # See _fallback_to_name
    if obj in _cache:
        return _cache[obj]

    code = None
    if isinstance(obj, (types.FunctionType, types.MethodType)):
        # Extract function from unbound method (Python 2)
        obj = getattr(obj, 'im_func', obj)
        try:
            code = obj.__code__
        except AttributeError:
            code = obj.func_code

        # Different instances of the same local function share the same code object, so this
        # can be used to look them up in the cache
        if code in _cache:
            return _cache[code]
    elif not (inspect.isclass(obj) or inspect.isroutine(obj)):
        return obj.__qualname__  # This object isn't meant to have a qualname. Raise a sensible error

    # For older Python versions, things get complicated.
    # Obtain the filename where the
    # class/method/function is defined.
    try:
        filename = inspect.getsourcefile(obj)
    except TypeError:
        return _fallback_to_name(obj)

    # Re-parse the source file to figure out what the
    # __qualname__ should be by analysing the abstract
    # syntax tree. Use a cache to avoid doing this more
    # than once for the same file.
    visitor = _file_cache.get(filename)
    if visitor is None:
        with open(filename, 'r') as fp:
            source = fp.read()
        node = ast.parse(source, filename)
        visitor = _Visitor()
        visitor.visit(node)
        _file_cache[filename] = visitor

        # For classes accessible from the top level, directly associate
        # each class with its qualname
        module = inspect.getmodule(obj)
        for k, qname_set in visitor.class_qualnames.items():

            # iterate over a copy since we're going to modify it
            for qname in list(qname_set):
                val = module
                for attr in qname.split('.'):
                    val = getattr(val, attr, None)

                    # Ensure that we're getting the right thing
                    if not (
                            inspect.isclass(val)
                            and val.__name__ == attr
                            and inspect.getmodule(val) == module
                    ):
                        break
                else:
                    _cache[val] = qname
                    qname_set.discard(qname)

        # Check if that worked for the current argument
        if obj in _cache:
            assert _cache[obj].endswith(obj.__name__)
            return _cache[obj]

    if code:
        result = _cache[code] = obj.__qualname__ = visitor.function_qualnames[code.co_firstlineno]
    else:
        results = visitor.class_qualnames[obj.__name__]
        if not results:
            return _fallback_to_name(obj)

        if len(results) == 1:
            result = list(results)[0]
        else:  # This means several local classes in the file have the same short name

            # Since the qualname of a method is unambiguous, the qualname
            # of this class can be guessed pretty reliably from its methods.
            # Since you could theoretically take a method from one class and
            # assign it as an attribute on another class, we look for the most
            # common prefix.
            counts = defaultdict(int)
            for method in obj.__dict__.values():
                if not isinstance(method, types.FunctionType):
                    continue
                method_qualname = qualname(method)
                suffix = '.' + method.__name__
                owner_class_qualname = method_qualname[:-len(suffix)]
                if owner_class_qualname in results:
                    counts[owner_class_qualname] += 1

            if counts:
                result = max(counts.items(), key=lambda item: item[1])[0]
            else:
                result = list(results)[0]

        if '.<locals>.' not in result:  # avoid overloading the cache with local classes
            _cache[obj] = result

    assert result.endswith(obj.__name__)
    return result


import csv
import cv2
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.image as mpimg

# Lines in the csv file
lines = []

# Reading the csv file and save each line in lines array
with open('TrainingV3/driving_log.csv') as csvfile:
    reader = csv.reader(csvfile)
    for line in reader:
        lines.append(line)

# Each line contain: Center image, left image, right image, steering value, throttle, brake, speed
# and the tag of each one is found in lines[0], so the data starts from lines[1]
print("First row: ", lines[0])
print("Second row (data): ", lines[1])
print()
print("Lines Number: ", len(lines))

# correction for the left and the right images of the sample data
steering_correction = 0.2
# Images array for the input data
images = []
# Measurements will be the labels for the Images
measurements = []
counter = 0

for line in lines[1:]:
    center_path = line[0]
    left_path = line[1]
    right_path = line[2]
    cent_measurement = float(line[3])
    throttle = float(line[4])
    # print(cent_measurement)

    center_filename = (center_path.split('/')[-1]).split('\\')[-1]
    left_filename = (left_path.split('/')[-1]).split('\\')[-1]
    right_filename = (right_path.split('/')[-1]).split('\\')[-1]

    center_current_path = 'TrainingV3' + '/IMG/' + center_filename
    left_current_path = 'TrainingV3' + '/IMG/' + left_filename
    right_current_path = 'TrainingV3' + '/IMG/' + right_filename

    cent_image = cv2.imread(center_current_path)[..., ::-1]
    left_image = cv2.imread(left_current_path)[..., ::-1]
    right_image = cv2.imread(right_current_path)[..., ::-1]

    left_measurement = cent_measurement + steering_correction
    right_measurement = cent_measurement - steering_correction

    cent_image = cv2.resize(cent_image[40:140, :], (64, 64))
    images.append(cent_image)
    measurements.append(tuple((cent_measurement, throttle)))
    # images.append(cent_image_flipped)

    left_image = cv2.resize(left_image[40:140, :], (64, 64))
    images.append(left_image)
    measurements.append(tuple((left_measurement, throttle)))
    # images.append(left_image_flipped)

    right_image = cv2.resize(right_image[40:140, :], (64, 64))
    images.append(right_image)
    measurements.append(tuple((right_measurement, throttle)))
    counter += 1
    print(counter)

# Convert images and measurements into numpy arrays for keras

# Covert to numpy arrays
images = np.array(images)
print(images.shape)
measurements = np.array(measurements)
print(measurements.shape)
print(measurements[0])
print('data in arrays')

import tensorflow as tf

# Normalization, Mean Centering, and Cropping

# In[37]:

# from keras.models import Sequential
# from keras.layers import Flatten, Dense, Activation, Lambda, Cropping2D
# from keras.layers.convolutional import Convolution2D
# from keras.layers.pooling import MaxPooling2D
# from keras.layers.core import Dropout

from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda, Convolution2D, MaxPooling2D, Dropout, Cropping2D, Activation
from keras.optimizers import Adam
from keras.layers.advanced_activations import ELU

# We will depend on Nvidia Model

def Nvidia_model(input_shape):
    # Normalizing, Mean Cenetring, Cropping
    model = Sequential()
    model.add(Lambda(lambda x: x / 255.0 - 0.5, name="image_normalization", input_shape=input_shape))

    model.add(Convolution2D(24, 5, 5, name="convolution_1", subsample=(2, 2), activation='relu', border_mode="valid",
                            init='he_normal'))

    # model.add(Convolution2D(nb_filter=36,nb_row=5,nb_co                                                                                l=5,subsample=(2,2),activation='relu'))
    model.add(Convolution2D(36, 5, 5, name="convolution_2", subsample=(2, 2), border_mode="valid", init='he_normal',
                            activation='relu'))

    # model.add(Convolution2D(nb_filter=48,nb_row=5,nb_col=5,subsample=(2,2),activation='relu'))
    model.add(Convolution2D(48, 5, 5, name="convolution_3", subsample=(2, 2), border_mode="valid", init='he_normal',
                            activation='relu'))

    # model.add(Convolution2D(nb_filter=64,nb_row=3,nb_col=3,activation='relu'))
    model.add(Convolution2D(64, 3, 3, name="convolution_4", border_mode="valid", init='he_normal', activation='relu'))

    # model.add(Convolution2D(nb_filter=64,nb_row=3,nb_col=3,activation='relu'))
    model.add(Convolution2D(64, 3, 3, name="convolution_5", border_mode="valid", init='he_normal', activation='relu'))

    model.add(Flatten())

    # Adding Dropout layer
    model.add(Dropout(p=0.2))
    model.add(Activation('relu'))
    model.add(Dense(1000))

    # Adding Dropout layer
    model.add(Dropout(p=0.5))
    model.add(Activation('relu'))
    model.add(Dense(500))

    # Adding Dropout layer
    model.add(Dropout(p=0.5))
    model.add(Activation('relu'))
    model.add(Dense(100))

    # Adding Dropout layer
    model.add(Dropout(p=0.2))
    model.add(Activation('relu'))
    model.add(Dense(10))
    model.add(Dense(2))  # Steering angle action and speed

    return model

# In[38]:

# Training data

X_train = images
print(X_train.shape)
y_train = measurements
print(y_train.shape)

# Run the model
epochs_arr = [20, 30, 40, 50]

for x in range(0, len(epochs_arr)):
    Model = Nvidia_model(input_shape=(64, 64, 3))
    learning_rate = 0.001
    print('created Model')
    # Mean square error and adam optimizer
    adam = Adam(lr=learning_rate)
    Model.compile(optimizer=adam, loss='mse')
    print('compiled Model')
    Model.summary()
    epochs = epochs_arr[x]
    batch_size = 512
    # 15% validation and apply shuffling
    Model.fit(x=X_train, y=y_train, nb_epoch=epochs, batch_size=batch_size,  validation_split=0.15, shuffle=True)
    # Save the model for testing it and give it as a paramter when running drive.py
    Model.save('model_6_str_throttle_' + str(learning_rate) + 'lr_' + str(epochs) + 'epoch.h5')