import logging
import threading
import time
import sys

from metro.data import Data
from renderers.main import MainRenderer

logger = logging.getLogger("busses")

try:
    from rgbmatrix import RGBMatrix, __version__
    emulated = False
except ImportError:
    from RGBMatrixEmulator import RGBMatrix, version
    emulated = True

"""
The data object has methods for saving the data

The MainRenderer handles creating the matrix from Data
    it does this by calling a render method, busses.render_busses

The __render_xxx functions update Data every N seconds, which MainRenderer
   receives via reference

"""

def main(matrix):
    logger.info("(%sx%s)", matrix.width, matrix.height)

    if emulated:
        logger.debug("rgbmatrix not installed, falling back to emulator!")
        logger.debug("Using RGBMatrixEmulator version %s", version.__version__)
    else:
        logger.debug("Using rgbmatrix version %s", __version__)

    data = Data()

    render = threading.Thread(target=__render_main, args=[matrix, data], name="render_thread", daemon=True)
    time.sleep(1)
    render.start()

    __refresh_busses()

def __refresh_busses(render_thread, data):
    logger.debug("Main has selected the busses to refresh")
    while render_thread.is_alive():
        time.sleep(30)
        data.update_stop_statuses()

def __render_main(matrix, data):
    MainRenderer(matrix, data).render()

if __name__ == "__main__":
    matrix = RGBMatrix()
    try:
        main(matrix)
    except:
        logger.exception("Untrapped error in main!")
        sys.exit(1)
    finally:
        matrix.Clear()


from flask import Flask, jsonify, request, render_template, redirect, url_for

from werkzeug.utils import secure_filename
import time
from datetime import datetime
import os
import io
from PIL import Image

from predict import *
# from preprocess import preprocess_image

UPLOAD_FOLDER = 'static/uploaded_images'
ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg'])

print("__name__:", __name__)
app = Flask(__name__)
current_path = os.path.dirname(os.path.realpath(__file__))
print("current_path:", current_path)
app = Flask(__name__, static_folder=current_path)

app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config["IMAGE_UPLOADS"] = UPLOAD_FOLDER #"/mnt/c/wsl/projects/pythonise/tutorials/flask_series/app/app/static/img/uploads"
app.config["ALLOWED_IMAGE_EXTENSIONS"] = ["JPEG", "JPG", "PNG", "GIF"]
app.config["MAX_IMAGE_FILESIZE"] = 5 * 1024 * 1024
RELATIVE_TEMPLATE_PATH = ''#'../../'

def allowed_image(filename):
    if not "." in filename:
        return False

    ext = filename.rsplit(".", 1)[1]

    if ext.upper() in app.config["ALLOWED_IMAGE_EXTENSIONS"]:
        return True
    else:
        return False

def allowed_image_filesize(filesize):

    if int(filesize) <= app.config["MAX_IMAGE_FILESIZE"]:
        return True
    else:
        return False

@app.route("/", methods=["GET", "POST"])
@app.route("/upload-image", methods=["GET", "POST"])
def upload_image():
    predicted = False
    if request.method == "POST":
        if request.files:
#             if "filesize" in request.cookies:

#             if not allowed_image_filesize(request.cookies["filesize"]):
#                 print("Filesize exceeded maximum limit")
#                 return redirect(request.url)

            image = request.files["image"]

            if image.filename == "":
                print("No filename")
                return redirect(request.url)

            if allowed_image(image.filename):
                # create filename with datetime
                filename = secure_filename(image.filename)
                now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
                filename = filename.split('.')[0] + '_' + now +'.' + filename.split('.')[1]
                os.makedirs(app.config["IMAGE_UPLOADS"] +'/'+ filename.split('.')[0])
                file_path = os.path.join(app.config["IMAGE_UPLOADS"], filename.split('.')[0], filename)
                image.save(file_path)
                print("Image saved")
#                 return predict(file_path)
                image, cls_score, heat_map, bounding_box = get_prediction_and_heat_map(file_path)
                probability = round(cls_score*100, 2)
                predicted = True
                # Save images to folder for futher page rendering
                filepath_image_original, filepath_image_bounding_box, filepath_image_heat_map = save_images(filename, image, heat_map, bounding_box)
                # IF NORMAL THEN PLOT ONLY HEAT MAP
                if probability < 50:
                    filepath_image_bounding_box = filepath_image_original
                    message = "Model did not recognize abnormality on x-ray"
                    message_color = 'green'
                else:
                    message = "Model recognized abnormality on x-ray !"
                    message_color = 'red'
#                 print("message:", message)
                return render_template('public/analysis_report_page.html',
                                        filename                    = filename,
                                        filepath_image_original     = filepath_image_original,
                                        filepath_image_bounding_box = filepath_image_bounding_box,
                                        filepath_image_heat_map     = filepath_image_heat_map,
                                        probability                 = probability,
                                        message                     = message,
                                        message_color               = message_color)

#                 return redirect(url_for("analysis_report", filename = filename,
#                                         filepath_image_original     = filepath_image_original,
#                                         filepath_image_bounding_box = filepath_image_bounding_box,
#                                         filepath_image_heat_map     = filepath_image_heat_map,
#                                         probability                 = probability))
            else:
                print("That file extension is not allowed")
                return redirect(request.url)
    if not predicted:
        return render_template("public/upload_image.html")
#     if predicted:
#         pass

def save_images(filename, image, heat_map, bounding_box):
    # here draw images with prediction for report web page
    filepath_image_original = filename.split('.')[0]+'_prepr.'+filename.split('.')[1]
    filepath_image_original = os.path.join(app.config["IMAGE_UPLOADS"], filename.split('.')[0], filepath_image_original)
    image_original = Image.fromarray(image)
    image_original.save(filepath_image_original)

    # --- Image with bounding box
    image_bounding_box = image.copy()
    # scaling bounding box to 0.5 preserving center point
    x,y,w,h = bounding_box
    x_scaled, y_scaled, w_scaled, h_scaled = (x + int(0.25*w), y + int(0.25*h), x + int(0.75*w), y + int(0.75*h))
    color = (36,255,12) # green
    cv2.rectangle(image_bounding_box, (x_scaled, y_scaled), (w_scaled, h_scaled), color, thickness=3)
#     print("(x_scaled, y_scaled), (w_scaled, h_scaled):", (x_scaled, y_scaled), (w_scaled, h_scaled))
    # save
    filepath_image_bounding_box = filename.split('.')[0]+'_bb.'+filename.split('.')[1]
    filepath_image_bounding_box = os.path.join(app.config["IMAGE_UPLOADS"], filename.split('.')[0], filepath_image_bounding_box)

    image_bounding_box = Image.fromarray(image_bounding_box)
    image_bounding_box.save(filepath_image_bounding_box)

    # --- Image with heat map overlayed
    image_heat_map = image.copy()
#     heat_map = np.repeat(heat_map[:, :, np.newaxis], 3, axis=2)
#     image_heat_map = image[:,:,0]
    heat_map = (heat_map*255/1.5).astype('uint8')

    # Apply colormap
    colormap=cv2.COLORMAP_JET
    heat_map = cv2.applyColorMap(heat_map, colormap)
    heat_map = cv2.cvtColor(heat_map, cv2.COLOR_BGR2RGB)

#     def rgb(minimum, maximum, value):
#         minimum, maximum = float(minimum), float(maximum)
#         ratio = 2 * (value-minimum) / (maximum - minimum)
#         b = int(max(0, 255*(1 - ratio)))
#         r = int(max(0, 255*(ratio - 1)))
#         g = 255 - b - r
#         return r, g, b

    alpha = 0.5
    image_heat_map = cv2.addWeighted(heat_map, alpha, image_heat_map, 1-alpha, 0)
    # save
    filepath_image_heat_map = filename.split('.')[0]+'_map.'+filename.split('.')[1]
    filepath_image_heat_map = os.path.join(app.config["IMAGE_UPLOADS"], filename.split('.')[0], filepath_image_heat_map)

    image_heat_map = Image.fromarray(image_heat_map)
    image_heat_map.save(filepath_image_heat_map)

    return filepath_image_original, filepath_image_bounding_box, filepath_image_heat_map

@app.route("/analysis_report_page/<filename>",  methods=['GET', 'POST'])
def analysis_report(filename, filepath_image_original, filepath_image_bounding_box, filepath_image_heat_map, probability):

    if request.method == 'POST':
        print(" RENDERING ANALYSIS")
        return render_template('analysis_report_page.html', filename,  filepath_image_original, filepath_image_bounding_box, filepath_image_heat_map, probability)

# @app.route("/analysis_report_page/<filename>",  methods=['GET', 'POST'])
# def analysis_report(filename, image, probability, heat_map, bounding_box):
#     print("\n\nFILENAME:", filename)
#     # here draw images with prediction for report web page
#     image_original = image#.copy()
#     filepath_image_original = '/uploaded_images/{}'.format(filename)
#     os.makedirs(filepath_image_original)
#     cv2.imsave(image_original, filepath_image_original)

#     image_bounding_box = image.copy()
#     # scaling bounding box to 0.5 preserving center point
#     x,y,w,h = bounding_box
#     x_scaled, y_scaled, w_scaled, h_scaled = (x + int(0.25*w), y + int(0.25*h), x + int(0.75*w), y + int(0.75*h))
#     color = (36,255,12) # green
#     cv2.rectangle(image_bounding_box, (x_scaled, y_scaled), (w_scaled, h_scaled), color, thickness=3)

#     image_heat_map = image.copy()#         return render_template('analysis_report_page.html', filename, filepath_image_original, probability)#, filepath_image_bounding_box, filepath_image_heat_map, probability)

#     image_heat_map = cv2.addWeighted(image_heat_map,0.5,heat_map,0.1,0)

#     if request.method == 'POST':

@app.route('/predict', methods=['POST'])
def predict(file_path):
    print("-------- predict function")
    if request.method == 'POST':
#         file = request.files['file']
#         img_bytes = file.read()
#         image = Image.open(io.BytesIO(image_bytes))
        abnormality_score = get_prediction(file_path=file_path)
        return jsonify({'Abnormality_score': str(abnormality_score), 'Is_abnormal': str(1 if abnormality_score>=0.5 else 0)})

# @app.route('/upload')
# def upload_file():
#     return render_template('upload.html')

# @app.route('/uploader', methods = ['GET', 'POST'])
# def upload_file():
#     if request.method == 'POST':
#         f = request.files['file']
#         f.save(secure_filename(f.filename))
#         return 'file uploaded successfully'

# def upload_file():
#     if request.method == 'POST':
#         # check if the post request has the file part
#         if 'file' not in request.files:
#             flash('No file part')
#             return redirect(request.url)
#         file = request.files['file']
#         # if user does not select file, browser also
#         # submit a empty part without filename
#         if file.filename == '':
#             flash('No selected file')
#             return redirect(request.url)
#         if file and allowed_file(file.filename):
#             filename = secure_filename(file.filename)
#             file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))

if __name__ == '__main__':
    app.run(debug=True)


import requests
import re
import csv
from openpyxl import Workbook
from openpyxl.styles import PatternFill
from urllib.request import urlopen
from bs4 import BeautifulSoup
from datetime import date

import pandas as pd
import os
from pathlib import Path
# pip install xlsxwriter
import xlsxwriter
import glob

#The list of all potential FI's websites that we might visit
pages = ["https://www.cibc.com/en/special-offers/fall-savings-promotion.html",
"https://www.scotiabank.com/ca/en/personal/rates-prices/savings-account-rates.html",
# "https://www.scotiabank.com/ca/en/personal/bank-accounts/savings-accounts/momentum-plus-savings-account.html",
"https://www.tangerine.ca/en/landing-page/raptors"]

#Today date in order to generate Excel timestamp
todayDate = str(date.today())

#This is write header to excel file
headers =['Bank','Account','Details','Special Offer']
book = Workbook()
sheet = book.active

for index, header in enumerate(headers):
    sheet.cell(row=1, column=index+1).value=header

for index, link in enumerate(pages):

    #SCO_MOME
    if link == "https://www.scotiabank.com/ca/en/personal/rates-prices/savings-account-rates.html":
        sheet.cell(row=index+2, column=1).value="Scotiabank"
        page = requests.get(link)
        soup = BeautifulSoup(page.text, 'html.parser')

        #Account name
        account_name = soup.find(class_='heading').text.strip()
        sheet.cell(row=index+2, column=2).value=account_name

        # table = soup.find('div', class_ = '_momentum-plus-savings-rates')

        # #Offer detail
        # print(table.tbody)
        detail = soup.find('small').find('p').text.strip()
        # sheet.cell(row=index+2, column=3).value=detail

        perks = soup.find(class_='cmp cmp-text').find('p').text
        #sheet.cell(row=index+2, column=4).value=perks

    #CIBC
    elif link =="https://www.cibc.com/en/special-offers/fall-savings-promotion.html" :
        sheet.cell(row=index+2, column=1).value="CIBC"

        page = requests.get(link)
        soup = BeautifulSoup(page.text, 'html.parser')

        account_name= soup.find(text=re.compile('eAdvantage'))
        sheet.cell(row=index+2, column=2).value = account_name

        data = soup.find_all('div', class_ = 'longformtext base parbase')

        sheet.cell(row=index+2, column=3).value = data[4].text.strip()

        #Offer detail
        sheet.cell(row=index+2, column=4).value = data[3].text.strip()

    #Tangerine
    elif link =="https://www.tangerine.ca/en/landing-page/raptors" :
        sheet.cell(row=index+2, column=1).value="Tangerine"

        page= requests.get(link)
        soup = BeautifulSoup(page.text, 'html.parser')

        #Account name
        account_name= soup.find(text=re.compile('Savings'))
        sheet.cell(row=index+2, column=2).value=account_name

        data = soup.find_all('div', class_ = 'tngFlex')

        #Offer 1
        offer1 = soup.find(class_ = 'tngWrapper').find('p').text.strip()
        sheet.cell(row=index+2, column=3).value = offer1

        #Offer 2
        sheet.cell(row=index+2, column=4).value = data[2].text.strip()

book.save("specialOffer"+todayDate+".xlsx")
####################################################

# def compare_changed_Special_Offer(previousFile,todayFile):

#Today date in order to generate Excel timestamp
todayDate = str(date.today())

#Get correct file
excelFile=glob.glob("specialOffer[0-9]*.xlsx")
excelFile = sorted(excelFile)
#path to files
currentDirectory = os.getcwd()+"\\"
path_OLD=Path(currentDirectory+excelFile[len(excelFile)-2])
path_NEW=Path(currentDirectory+excelFile[len(excelFile)-1])

# Read in the two excel files and fill NA
df_OLD = pd.read_excel(path_OLD,header=None, names=None).fillna(0)
df_NEW = pd.read_excel(path_NEW,header=None, names=None).fillna(0)

dfDiff = df_OLD.copy()
for row in range(dfDiff.shape[0]):
    for col in range(dfDiff.shape[1]):
        value_OLD = df_OLD.iloc[row,col]
        try:
            value_NEW = df_NEW.iloc[row,col]
            if value_OLD==value_NEW and value_NEW!=0:
                dfDiff.iloc[row,col] = df_NEW.iloc[row,col]
            elif value_OLD==value_NEW and value_NEW==0:
                dfDiff.iloc[row,col] = ""
            elif( value_OLD!=value_NEW and value_NEW==0):
                dfDiff.iloc[row,col] = ('Expired: {}').format(value_OLD)
            else:
                dfDiff.iloc[row,col] = ('Update: {}').format(value_NEW)

        except:
            dfDiff.iloc[row,col] = ('{}-->{}').format(value_OLD, 'NaN')
writer = pd.ExcelWriter("specialOffer_compare"+todayDate+".xlsx", engine='xlsxwriter') # pylint: disable=abstract-class-instantiated
dfDiff.to_excel(writer, sheet_name='DIFF', index=False, header=None)

workbook  = writer.book
worksheet = writer.sheets['DIFF']

# define formats
highlight_fmt_red = workbook.add_format({'font_color': '#000000', 'bg_color':'#FF0000'})
highlight_fmt_yellow = workbook.add_format({'font_color': '#000000', 'bg_color':'#FFFF00'})

## highlight Update cells
worksheet.conditional_format('A1:ZZ1000', {'type': 'text',
                                        'criteria': 'containing',
                                        'value':'Update',
                                        'format': highlight_fmt_yellow})
## highlight Expired cells
worksheet.conditional_format('A1:ZZ1000', {'type': 'text',
                                        'criteria': 'containing',
                                        'value':'Expired',
                                        'format': highlight_fmt_red})
# save
writer.save()


import sys
import plotly.graph_objects as go
import itertools
import copy
from settings import filterWidth, channelWidth, filterCountMax
from gui import drawSplit, drawUncoveredChannel, drawConflict, renderGUI

channelWidthHalf = int(channelWidth/2)
filterWidthHalf = int(filterWidth/2)
searchGranularity = int(channelWidthHalf/1) #TODO find minimum value

class Channel:
    def __init__(self, freqCenter, width=channelWidth):
        self.freqCenter = freqCenter
        self.freqLow = freqCenter - int(width/2)
        self.freqHigh = freqCenter + int(width/2)
    def __lt__(self, other):
        return self.freqCenter < other.freqCenter
    def __str__(self):
        return str(self.freqCenter)

class Filter:
    def __init__(self, freqCenter, width=filterWidth):
        self.freqCenter = freqCenter
        self.freqLow = freqCenter - int(width/2)
        self.freqHigh = freqCenter + int(width/2)
        self.width = width
        self.channels = []
        self.channelScore = 0
        self.centerScore = 0
    def __str__(self):
        return str(self.freqCenter)
    def calcFilterChannelScore(self):
        self.channelScore = -1 * (len(self.channels) * len(self.channels))
        return
    def calcFilterCenterScore(self):
        freqSum = 0
        for channel in self.channels:
            freqSum += channel.freqCenter
        freqAverage = int(freqSum / len(self.channels))
        delta = (self.freqCenter - freqAverage) / 1000000
        self.centerScore = -1 * (delta * delta)
        return

class SubSolution:
    def __init__(self, filters, channels):
        self.filters = filters
        self.filterCount = len(filters)
        self.channels = channels
        self.channelScore = sum(f.channelScore for f in filters) / self.filterCount
        self.centerScore = sum(f.centerScore for f in filters) / self.filterCount

class Solution:
    def __init__(self, subSolutions):
        self.subSolutions = subSolutions
        self.filterCount = sum(s.filterCount for s in subSolutions)
        self.channelScore = sum(s.channelScore for s in subSolutions) / len(subSolutions)
        self.centerScore = sum(s.centerScore for s in subSolutions) / len(subSolutions)

# Given channels and a filter, return all filters passed by the channel
def getChannelsInFilter(filter, channels):
    channelsInFilter = []
    for channel in channels:
        if(checkChannelInFilter(filter, channel)):
            channelsInFilter.append(channel)
    return channelsInFilter

# Given a potential filter, existing filters, and channels, check if new filter is valid.
# If filter is valid, return True.
# If not valid, return False.
def validateFilter(newFilter, filters, channels):
    filterValid = True
    # Filter can't split a channel. Channel must be fully in or out of filter.
    for channel in channels:
        filterValid = filterValid and checkFilterSplit(newFilter, channel)

    # Filter can't overlap any other filter.
    for filter in filters:
        filterValid = filterValid and not checkFilterOverlap(newFilter, filter)

    # Filter must fully pass at least one channel.
    filterContainsChannel = False
    for channel in channels:
        filterContainsChannel = filterContainsChannel or checkChannelInFilter(newFilter, channel)

    filterValid = filterValid and filterContainsChannel
    return filterValid

# Given a filter and channel, check if channel is within filter
# Given a filter and channel, check if the channel is fully passed by filter.
# If channel fully passes, return True.
# If not, return False.
def checkChannelInFilter(filter, channel):
    return (filter.freqLow <= channel.freqLow <= filter.freqHigh)\
        and (filter.freqLow <= channel.freqHigh <= filter.freqHigh)

# Given a filter and channel, check if the filter edge falls within the channel
def checkFilterSplit(filter, channel):
    return not ((channel.freqLow < filter.freqLow < channel.freqHigh)\
        or (channel.freqLow < filter.freqHigh < channel.freqHigh))

# Given two filters of identical width, check if filters overlap.
# If filters overlap (invalid), return True.
# If filters don't overlap, return False.
def checkFilterOverlap(filter1, filter2):
    return (filter1.freqLow < filter2.freqHigh) and (filter2.freqLow < filter1.freqHigh)

# Check if channel is fully passed by only one filter in filter list
def checkChannel(fig, channel, filters):
    filterCount = 0
    for filter in filters:
        if (filter.freqLow <= channel.freqLow <= filter.freqHigh)\
            and (filter.freqLow <= channel.freqHigh <= filter.freqHigh):
            filterCount += 1
    if filterCount == 0:
        print("Channel", channel.freqCenter, "not passed!")
        drawUncoveredChannel(fig, channel)
    elif filterCount > 1:
        print("Channel", channel.freqCenter, "covered by multiple filters!")
        drawUncoveredChannel(fig, channel)
    return filterCount == 1

# Check if set of channels and filters are valid.
# Solution is valid if no filters overlap and if all channels are fully passed.
# Return True if valid.
def checkSolution(fig, channels, filters):
    validSolution = True
    filterCombos = list(itertools.combinations(filters, 2))

    for filterPair in filterCombos:
        if checkFilterOverlap(filterPair[0], filterPair[1]):
            # Channels overlap
            validSolution = False
            print("Filter overlap between ", filterPair[0], filterPair[1])
            drawConflict(fig, filterPair[0])
            drawConflict(fig, filterPair[1])

    for channel in channels:
        if checkChannel(fig, channel, filters) == False:
            validSolution = False

    print("Valid solution:", validSolution)
    return validSolution

# Check if channels are far enough apart such that no valid filter on one could affect the other.
# If indpendent, return True.
# If not, return False.
def channelsAreIndependent(channel1, channel2):
    return (channel2.freqCenter-channel1.freqCenter) + channelWidth >= 2 * filterWidth

# Given a list of channels, split channels into independent groups.
# Return list of list of channels.
def splitChannels(fig, channels):
    channels.sort()
    channelRanges = []
    lowIndex = 0
    for i in range(0, len(channels)+1):
        if i == len(channels):
            channelRanges.append(channels[lowIndex:i])
        elif i != lowIndex:
            if channelsAreIndependent(channels[i-1], channels[i]):
                channelRanges.append(channels[lowIndex:i])
                drawSplit(fig, 0.5*(channels[i].freqCenter+channels[i-1].freqCenter))
                lowIndex = i
    return channelRanges

# Recursive portion of solveChannels.
def solveChannelsRec(subChannels, solutions, channelsCopy, newFilters):
    # If no channels remain to be covered, we're done.
    if len(channelsCopy) == 0:
        solution = SubSolution(newFilters, subChannels)
        # Reduce solutions to best solution per number of filters
        if solution.filterCount in solutions:
            # If we already have a solution for this number of filters, only save best.
            solutionOptions = [solution, solutions[solution.filterCount]]
            solutionOptions.sort(key=lambda i: (i.filterCount, i.channelScore, i.centerScore))
            solutions[solution.filterCount] = solutionOptions[-1]
        else:
            solutions[solution.filterCount] = solution
        return
    # Find range of possible filter frequencies.
    fLow = min(channelsCopy).freqCenter + channelWidthHalf - filterWidthHalf
    fHigh = (max(channelsCopy).freqCenter - channelWidthHalf) + filterWidthHalf
    # Create filter and check if valid.
    for grid in range(fLow, fHigh + channelWidthHalf, searchGranularity):
        newFiltersCopy = copy.copy(newFilters)
        channelsCopyRec = copy.copy(channelsCopy)
        newFilter = Filter(grid)
        if(validateFilter(newFilter, newFiltersCopy, channelsCopy)):
            channelsInFilter = getChannelsInFilter(newFilter, channelsCopyRec)
            newFilter.channels = channelsInFilter
            newFilter.calcFilterCenterScore()
            newFilter.calcFilterChannelScore()
            # Add filter to existing filter list.
            newFiltersCopy.append(newFilter)
            # Remove channels covered by new filter to create new channel subset.
            for channel in channelsInFilter:
                channelsCopyRec.remove(channel)
            # Solve remaining channels recurrsively.
            solveChannelsRec(channelsCopy, solutions, channelsCopyRec, newFiltersCopy)
    return

# Initial recursive driver to find solutions for a set of channels.
# Return dict mapping filterCount to SubSolution.
def solveChannels(channels):
    # Find range of possible filter frequencies.
    fLow = min(channels).freqCenter + channelWidthHalf - filterWidthHalf
    fHigh = (max(channels).freqCenter - channelWidthHalf) + filterWidthHalf
    solutions = {}
    # Create filter and check if valid.
    for grid in range(fLow, fHigh + searchGranularity, searchGranularity):
        channelsCopy = copy.copy(channels)
        newFilters = []
        # Pick filter
        newFilter = Filter(grid)
        if(validateFilter(newFilter, newFilters, channelsCopy)):
            channelsInFilter = getChannelsInFilter(newFilter, channelsCopy)
            newFilter.channels = channelsInFilter
            newFilter.calcFilterCenterScore()
            newFilter.calcFilterChannelScore()
            # Add filter to existing filter list.
            newFilters.append(newFilter)
            # Remove channels covered by new filter to create new channel subset.
            for channel in channelsInFilter:
                channelsCopy.remove(channel)
            # Solve remaining channels recurrsively.
            solveChannelsRec(channels, solutions, channelsCopy, newFilters)
    return solutions

def getChannelsFromFile(filePath, channelsAll):
    f = open(filePath,"r")
    for line in f.read().splitlines():
        freq = float(line.replace(",",""))
        # If input is in MHz, convert to Hz.
        if freq < 1000:
            freq = freq * 1000000
        # Check if input is within US Dl ranges.
        if not((758000000 <= freq <= 775000000) or (851000000 <= freq <= 869000000)):
            print("Error:", freq, "is outside of US DL ranges.")
            exit(1)

        freq = int(freq)
        # Check for duplicate channels
        if any(channel.freqCenter == freq for channel in channelsAll):
            print("Error:", freq, "appears in input twice.")
            exit(1)
        # Add channel to list
        channelsAll.append(Channel(freq))
    return

def main():
    if(len(sys.argv)==1):
        print("Error: Please provide path to channel lists (csv/txt) as argument.")
        exit(1)

    # Get channels from file arguments
    channelsAll = []
    for path in sys.argv[1:]:
        getChannelsFromFile(path, channelsAll)

    print("Channels:", len(channelsAll))

    fig = go.Figure()

    # Split channels into independent subgroups.
    # For each subgroup, find best solution per filter count.
    channelSubgroups = splitChannels(fig, channelsAll)
    subgroupSolutions = [None] * len(channelSubgroups)
    for i in range(0, len(channelSubgroups)):
        channelSubgroup = channelSubgroups[i]
        print("Solving subset of size", len(channelSubgroup))
        if len(channelSubgroup) == 1:
            # Add filters 1:1 to single isolated channels
            newFilter = Filter(channelSubgroup[0].freqCenter)
            newFilter.channels = [channelSubgroup[0]]
            newFilter.calcFilterChannelScore()
            newFilter.calcFilterCenterScore()
            subgroupSolutions[i] = {1: SubSolution([newFilter], channelSubgroup)}
        else:
            subgroupSolutions[i] = solveChannels(channelSubgroup)

    # Get all combinations of subgroupSolutions.
    # Create Solution for each combination.
    solutionCombos = list(itertools.product(*subgroupSolutions))
    solutions = []
    for s in solutionCombos:
        subSolutionList = []
        for i in range(0, len(s)):
            subSolutionList.append(subgroupSolutions[i][s[i]])
        solutions.append(Solution(subSolutionList))
    # Sort solutions from worst to best.
    solutions.sort(key=lambda i: (i.filterCount, i.channelScore, i.centerScore))

    print("Max filter solution uses", solutions[-1].filterCount, "filters.")
    print("Min filter solution uses", solutions[0].filterCount, "filters.")

    # Traverse solution list until filter count is within limit or until end of list.
    filters = []
    for i in reversed(range(0, len(solutions))):
        if solutions[i].filterCount <= filterCountMax:
            # Solution within limits has been found.
            # Print results and write to file.
            # Render GUI.
            print("Best solution within limits:", solutions[i].filterCount,\
                solutions[i].channelScore, solutions[i].centerScore)
            for s in solutions[i].subSolutions:
                filters += s.filters
            checkSolution(fig, channelsAll, filters)
            # Print filters and write to file.
            print("Filters used:",len(filters))
            f = open("filters.txt", "w")
            for filter in filters:
                print(filter)
                f.write(str(filter)+",\n")
            f.close()
            renderGUI(fig, channelsAll, filters)
            exit(0)
    print("Error: No solution for", filterCountMax, "filters.")
    exit(0)

if __name__ == "__main__":
    main()


import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
import pickle
from sklearn.utils import shuffle
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

for number in range(6, 10):

    df = pd.read_excel(r'C:\Users\gaura\OneDrive\Desktop\Simpoint_data_gem5\user_experience_project\d+i_cache\data_400M\d+i_COMBINE_DATA.xlsx', 'final_dataset')
    #df = pd.read_excel(r'C:\Users\gaura\OneDrive\Desktop\Simpoint_data_gem5\user_experience_project\d+i_cache\data_400M\MOBY\d+i_Moby.xlsx', 'Tosi_data_moby')

    df = df.fillna(df.mean())
    df = pd.DataFrame(df)

    # clustering algo

    wcss = []
    from sklearn.cluster import KMeans
    for i in range(1, 15):
        kmeans = KMeans(n_clusters = i).fit(df.iloc[:, 4:-1])
        wcss.append(kmeans.inertia_)
    plt.plot(range(1, 15), wcss)
    #plt.show()

    kmeans = KMeans(n_clusters = 4, random_state = 0)
    y_kmeans = kmeans.fit_predict(df.iloc[:, 4:-1])

    cluster_0 = df.iloc[:, 4:-1][y_kmeans == 0].index.tolist()
    cluster_1 = df.iloc[:, 4:-1][y_kmeans == 1].index.tolist()
    cluster_2 = df.iloc[:, 4:-1][y_kmeans == 2].index.tolist()
    cluster_3 = df.iloc[:, 4:-1][y_kmeans == 3].index.tolist()

    clusters = [cluster_0,cluster_1,cluster_2,cluster_3]
    application_0, application_1, application_2, application_3 = [], [], [], []
    for n,i in enumerate(clusters):
        application = []
        for row in i:
            application.append(df.iloc[row, 0:4].values)
        if n ==0: application_0 = application
        if n ==1: application_1 = application
        if n ==2: application_2 = application
        if n ==3: application_3 = application

    df = pd.concat([df[y_kmeans == 0]])
    feat_df = [i for i in df.columns if 'icache' in i]
    df_ind = [ind for ind, name in enumerate(df.columns.tolist()) if name in feat_df]
    df_y_e = df.loc[df.final_class == 'e']
    df_y_a = df.loc[df.final_class == 'a']
    df_y_b = df.loc[df.final_class == 'b']
    df_y_c = df.loc[df.final_class == 'c']
    df_y_d = df.loc[df.final_class == 'd']

    df_y_e = shuffle(df_y_e)
    df_y_e = df_y_e.iloc[0:20, :]
    df = pd.concat([df_y_a, df_y_b, df_y_c, df_y_d, df_y_e], axis = 0)
    df = shuffle(df)
    X = df.iloc[:, 4:-1]
    y = df.iloc[:, -1]
    from sklearn.feature_selection import SelectKBest, chi2

    best_features = SelectKBest(score_func = chi2, k = 30)
    fit = best_features.fit(X,y)

    # finding the scores and columns to concatenate them into a table

    scores = pd.DataFrame(fit.scores_)
    columns = pd.DataFrame(pd.DataFrame(X).columns)

    score_table = pd.concat([columns, scores], axis = 1, ignore_index = True)
    score_table.columns = ['specs', 'score']
    sc_table = score_table.nlargest(25, 'score')
    index_sc_table = list(pd.DataFrame(sc_table).index.values)
    from sklearn.preprocessing import StandardScaler
    sc_x = StandardScaler()
    X = sc_x.fit_transform(X)
    from sklearn.model_selection import train_test_split
    x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)

    x_train = pd.DataFrame(x_train)
    x_test = pd.DataFrame(x_test)

    ind= [13,31, 32, 11, 29,66, 67, 68,  5, 48, 50, 14, 10, 18, 49]

    X_train = x_train.iloc[:,ind]
    X_test = x_test.iloc[:,ind]

    pipeline_feat_DT= Pipeline([('classifier_feat_DT', DecisionTreeClassifier( criterion = 'entropy', random_state = 0))])
    pipeline_feat_RF= Pipeline([('classifier_feat_RF', RandomForestClassifier(n_estimators = 300, criterion = 'entropy', random_state = 0))])
    pipeline_feat_LT = Pipeline([('classifier_feat_LT', LogisticRegression(random_state = 0))])
    pipeline_feat_SVC = Pipeline([('classifier_feat_SVC', SVC(C = 1, kernel = 'rbf', gamma =0.1 , random_state = 0, cache_size = 200))])
    pipeline_feat_Knn = Pipeline([('classifier_feat_Knn', KNeighborsClassifier(n_neighbors = 3, metric = 'minkowski', p = 2))])
    pipeline_feat_Naive_Bayes = Pipeline([('classifier_feat_Naive_Bayes', GaussianNB())])
    pipeline_feat_Adaboost = Pipeline([('classifier_feat_Aaboost', AdaBoostClassifier(n_estimators=300, learning_rate=1).fit(X_train, y_train))])

    pipe_feat_list = [pipeline_feat_LT, pipeline_feat_DT, pipeline_feat_RF, pipeline_feat_SVC, pipeline_feat_Knn, pipeline_feat_Naive_Bayes, pipeline_feat_Adaboost]

    pipe_feat_dict = {0: 'LogisticRegression_Feat', 1: 'DecisionTree_Feat', 2: 'RandomForest_Feat', 3: 'SVC_Feat', 4: 'K-NN_Feat', 5: 'Naive_Bayes_Feat', 6:'Adaboost'}

    for pipe in pipe_feat_list:
        pipe.fit(X_train, y_train)

    Feat_table = []
    for i, model in enumerate(pipe_feat_list):
        accuracy = '{}_accuracy {}'.format(pipe_feat_dict[i], model.score(X_test, y_test))
        Feat_table.append(accuracy.split(' ')[1])

    col_1 = ['LogisticRegression', 'Decision Tree', 'Random Forest', 'SVC', 'K-NN', 'Naive_Bayes', 'Adaboost' ]
    col_2 = Feat_table
    Final_table = pd.concat([pd.DataFrame(col_1), pd.DataFrame(col_2)], axis = 1, ignore_index = True)
    Final_table.columns = ['ML_Classifier' , 'Feat_accuracy']
    Final_table = pd.DataFrame(Final_table)
    print(Final_table)

    #knn tuning
    clf_knn = KNeighborsClassifier(n_neighbors = 3, metric = 'minkowski', p = 1).fit(X_train, y_train)
    print('Before tuning Knn: {}'.format(clf_knn.score(X_test, y_test)))

    from sklearn.model_selection import GridSearchCV

    parameters = [
                  {'n_neighbors' : [1, 2, 3, 4, 5, 10], 'metric':['minkowski'], 'p':[1, 2]},

                    ]
    GS_knn = GridSearchCV(estimator = clf_knn, param_grid = parameters, scoring = 'accuracy', cv = 10, n_jobs = -1 )
    GS_knn.fit(X_train, y_train)

    accuracy_knn = GS_knn.best_score_
    print("Knn GS_cv accuracy: {}".format(accuracy_knn))
    print("GS_cv_knn best params: {}".format(GS_knn.best_params_))

    # Rf tuning

    clf_RF= RandomForestClassifier(n_estimators = 20, criterion = 'entropy', random_state = 0).fit(X_train, y_train)
    clf_RF.score(X_test, y_test)

    parameters = [
                  {'n_estimators': [20, 30, 50, 70, 100, 200, 300, 500, 1000], 'criterion' : ['gini'],'max_features' :['auto'], 'random_state' : [0] },
                  {'n_estimators' : [20, 30, 50, 70, 100, 200, 300, 500, 1000], 'criterion': ['gini'], 'max_features' : ['sqrt'], 'random_state' : [0]},
                  {'n_estimators' : [20, 30, 50, 70, 100, 200, 300, 500, 1000], 'criterion' : ['gini'], 'max_features' : ['log2'], 'random_state' : [0]},
                  {'n_estimators': [20, 30, 50, 70, 100, 200, 300, 500, 1000], 'criterion' : ['entropy'],'max_features' :['auto'], 'random_state' : [0] },
                  {'n_estimators' : [20, 30, 50, 70, 100, 200, 300, 500, 1000], 'criterion': ['entropy'], 'max_features' : ['sqrt'],  'random_state' : [0]},
                  {'n_estimators' : [20, 30, 50, 70, 100, 200, 300, 500, 1000], 'criterion' : ['entropy'], 'max_features' : ['log2'],  'random_state' : [0]}]

    GS_RF = GridSearchCV(estimator = clf_RF, param_grid = parameters, scoring = 'accuracy', cv = 10, n_jobs = -1 )
    GS_RF.fit(X_train, y_train)

    accuracy_RF = GS_RF.best_score_
    print("RF GS_cv accuracy: {}".format(accuracy_RF))
    print("GS_cv_RF best params: {}".format(GS_RF.best_params_))

    # applying the parameters got from the tuning

    clf_RF= RandomForestClassifier(n_estimators = GS_RF.best_params_['n_estimators'], max_features = GS_RF.best_params_['max_features'], criterion = GS_RF.best_params_['criterion'], random_state = 0).fit(X_train, y_train)
    clf_RF.score(X_test, y_test)
    print('RF_tuned accuracy: {}'.format(clf_RF.score(X_test, y_test)))
    with open('i_RF_Tuned.sav', 'wb') as iDTt:
        pickle.dump(clf_RF, iDTt)

    clf_knn = KNeighborsClassifier(n_neighbors = GS_knn.best_params_['n_neighbors'], metric = GS_knn.best_params_['metric'], p = GS_knn.best_params_['p']).fit(X_train, y_train)
    print('knn_tuned accuracy: {}'.format(clf_knn.score(X_test, y_test)))
    with open('i_knn_Tuned.sav', 'wb') as iDTt:
        pickle.dump(clf_knn, iDTt)

    clf_DT_tuned= DecisionTreeClassifier(criterion = 'entropy', random_state = 0).fit(X_train, y_train)
    print('DT_tuned_accuracy: {}'.format(clf_DT_tuned.score(X_test, y_test)))

    #with open('i_DT_Tuned.sav', 'wb') as iDTt:
    #    pickle.dump(clf_DT_tuned, iDTt)

    os.mkdir(r'C:\Users\gaura\OneDrive\Desktop\Simpoint_data_gem5\user_experience_project\SPEC\d_cache\RF_DT\icache\Round_{}'.format(number))

    Final_table.loc[Final_table['ML_Classifier'] == 'Decision Tree', 'Feat_accuracy'] = clf_DT_tuned.score(X_test, y_test)
    Final_table.loc[Final_table['ML_Classifier'] == 'Random Forest', 'Feat_accuracy'] = clf_RF.score(X_test, y_test)
    Final_table.loc[Final_table['ML_Classifier'] == 'K-NN', 'Feat_accuracy'] = clf_knn.score(X_test, y_test)

    Final_table.to_csv(r'C:\Users\gaura\OneDrive\Desktop\Simpoint_data_gem5\user_experience_project\SPEC\d_cache\RF_DT\icache\accuracy_table_{}.csv'.format(number), index = False)

    y_pred_RF = clf_RF.predict(X_test)
    y_pred_array_RF = np.array(y_pred_RF)
    y_pred_RF
    y_pred_DT = clf_DT_tuned.predict(X_test)
    y_pred_array_DT = np.array(y_pred_DT)
    y_pred_DT
    y_pred_knn = clf_knn.predict(X_test)
    y_pred_array_knn = np.array(y_pred_knn)
    y_pred_knn
    y_test_array = np.array(y_test)
    y_test_array

    from openpyxl import Workbook

    df_alldata = pd.read_excel(r'C:\Users\gaura\OneDrive\Desktop\Simpoint_data_gem5\user_experience_project\d+i_cache\data_400M\d+i_COMBINE_DATA.xlsx', 'LE_calc')
    pred_arr = [y_pred_array_DT, y_pred_array_RF, y_pred_array_knn]
    name = ['DT', 'RF', 'knn']

    for array, name in zip(pred_arr, name):

        index = y_test.index
        df_array = []
        df_phase = []
        df_names = df_alldata.iloc[:, 0]
        df_phases = df_alldata.iloc[:, 3]
        for i in index:
            df_array.append(df_names[index])
            df_phase.append(df_phases[index])

        book = Workbook()
        sheet= book.active

        df_array = pd.DataFrame(df_array)
        df_phase = pd.DataFrame(df_phase)
        f = df_array.T.iloc[:,0]
        f_phs = df_phase.T.iloc[:,0]
        f.columns = ['index', 'app_name']
        f_phs.columns = ['index','phase']

        moby_arr = ['360buy', 'adobe', 'baidumap', 'bbench', 'frozenbubble', 'k9mail', 'netease', 'sinaweibo', 'mxplayer', 'ttpod' , 'kingsoftoffice']

        k = 2

        sheet.cell(row = 1, column = 1).value = 'application'
        sheet.cell(row = 1, column = 2).value = 'phase'
        sheet.cell(row = 1, column = 3).value = 'Predicted_class'
        sheet.cell(row = 1, column = 4).value = 'Actual_class'
        sheet.cell(row = 1, column = 5).value = 'i_PREDICTED_LAT_RET'
        sheet.cell(row = 1, column = 6).value = 'i_PRED_LAT_RET'
        sheet.cell(row = 1, column = 7).value = 'i_ACTUAL_ENERGY_RET'
        sheet.cell(row = 1, column = 8).value = 'i_ACTUAL_ENERGY_RET'
        sheet.cell(row = 1, column = 9).value = 'i_Latency_predicted'
        sheet.cell(row = 1, column = 10).value = 'i_Latency_actual'
        sheet.cell(row = 1, column = 11).value = 'i_Energy_predicted'
        sheet.cell(row = 1, column = 12).value = 'i_Energy_actual'

        for index, i, j in zip(index, array, y_test_array):

            if f[index] in moby_arr:

                sheet['A{}'.format(k)] = f[index]
                sheet['B{}'.format(k)] = f_phs[index]
                sheet['C{}'.format(k)] = i
                sheet['D{}'.format(k)] = j

                if i == 'a':
                    pair = ['1ms']

                elif i == 'b':
                    pair = ['50ms']

                elif i == 'c':
                    pair = ['100ms']

                elif i == 'd':
                    pair = ['1s']

                elif i == 'e':
                    pair = ['10ms']

                if j == 'a':
                    pair_2 = ['1ms']

                elif j == 'b':
                    pair_2 = ['50ms']

                elif j == 'c':
                    pair_2 = ['100ms']

                elif j == 'd':
                    pair_2 = ['1s']

                elif i == 'e':
                    pair_2 = ['10ms']

                lat_ret_pred = pair[0]
                energy_ret_pred = pair[0]
                lat_ret_actual = pair_2[0]
                energy_ret_actual = pair_2[0]

                df_1 = df_alldata
                df_1 = df_1.loc[:,['application','phase no.','d_ret_time', 'i_ret_time', 'i_latency','i_energy']]

                df_row_lat_pred = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index]) & (df_1['i_ret_time'] == lat_ret_pred)]
                df_row_energy_pred = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index]) & (df_1['i_ret_time'] == energy_ret_pred)]
                df_row_lat_actual = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index]) & (df_1['i_ret_time'] == lat_ret_actual )]
                df_row_energy_actual = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index]) & (df_1['i_ret_time'] == energy_ret_actual)]

                # for base = 1s

                df_row_lat_1s = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index]) & (df_1['i_ret_time'] == '1s')]
                df_row_energy_1s = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index]) & (df_1['i_ret_time'] == '1s')]

                #exhautive
                df_row_lat_exh = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index])]
                df_row_energy_exh = df_1.loc[(df_1['application'] == f[index]) & (df_1['phase no.'] == f_phs[index])]

                i_Lat_pred = df_row_lat_pred.iloc[:,-2]
                i_Lat_pred = np.array(i_Lat_pred)
                i_Lat_pred = i_Lat_pred[0]

                i_energy_pred = df_row_energy_pred.iloc[:,-1]
                i_energy_pred = np.array(i_energy_pred)
                i_energy_pred = i_energy_pred[0]

                i_Lat_actual = df_row_lat_actual.iloc[:,-2]
                i_Lat_actual = np.array(i_Lat_actual)
                i_Lat_actual = i_Lat_actual[0]

                i_energy_actual = df_row_energy_actual.iloc[:,-1]
                i_energy_actual = np.array(i_energy_actual)
                i_energy_actual = i_energy_actual[0]

                i_exh_lat = min(df_row_lat_exh.iloc[:,-2].values)
                i_exh_energy = min(df_row_energy_exh.iloc[:,-1].values)

                i_Lat_1s = df_row_lat_1s.iloc[:,-2]
                i_Lat_1s = np.array(i_Lat_1s)
                i_Lat_1s = i_Lat_1s[0]

                i_energy_1s = df_row_energy_1s.iloc[:,-1]
                i_energy_1s = np.array(i_energy_1s)
                i_energy_1s = i_energy_1s[0]

                sheet['E{}'.format(k)] = lat_ret_pred
                sheet['F{}'.format(k)] = energy_ret_pred
                sheet['I{}'.format(k)] = i_Lat_pred
                sheet['J{}'.format(k)] = i_Lat_actual
                sheet['K{}'.format(k)] = i_energy_pred
                sheet['L{}'.format(k)] = i_energy_actual
                sheet['G{}'.format(k)] = lat_ret_actual
                sheet['H{}'.format(k)] = energy_ret_actual
                sheet['M{}'.format(k)] = i_exh_lat
                sheet['N{}'.format(k)] = i_exh_energy
                sheet['O{}'.format(k)] = i_Lat_1s
                sheet['P{}'.format(k)] = i_energy_1s
                sheet['R{}'.format(k)] = (i_Lat_1s - i_Lat_pred)/i_Lat_1s
                sheet['T{}'.format(k)] = (i_Lat_1s - i_Lat_actual)/i_Lat_1s
                sheet['V{}'.format(k)] = (i_Lat_1s - i_exh_lat)/i_Lat_1s
                sheet['S{}'.format(k)] = (i_energy_1s - i_energy_pred)/i_energy_1s
                sheet['U{}'.format(k)] = (i_energy_1s - i_energy_actual)/i_energy_1s
                sheet['W{}'.format(k)] = (i_energy_1s - i_exh_energy)/i_energy_1s
                k+=1
                print(name, f[index],i, j, i_Lat_pred, i_Lat_actual)

        book.save(r'C:\Users\gaura\OneDrive\Desktop\Simpoint_data_gem5\user_experience_project\SPEC\d_cache\RF_DT\icache\Round_{}\icache_{}_{} .xlsx'.format(number, name, number))


from setuptools import setup
import pathlib
import re

curdir = pathlib.Path(__file__).parent

README = (curdir / "README.md").read_text()
VERSION = (curdir / "tidyfasta" / "__init__.py").read_text()
version_number=re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", VERSION).group(1)

setup(
    name="tidyfasta",
    version=version_number,
    packages=['tidyfasta', 'tidyfasta.common'],
    url='https://github.com/maxhebditch/tidyfasta',
    license='MIT',
    author='Max Hebditch',
    author_email='max@maxhebditch.co.uk',
    description='Sanitise protein FASTA files / data',
    long_description=README,
    long_description_content_type="text/markdown",
    entry_points={
        "console_scripts": [
            "tidyfasta=tidyfasta.__main__:main",
        ]
    }
)


# %%
#Mihai Cira - B. 29. Walmart Sales Forecast
#topic: analysis of the influence of holidays on sales

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

from scipy import stats
import statsmodels.api as sm

from sklearn.preprocessing import MinMaxScaler

from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression

from sklearn.ensemble import RandomForestRegressor

# %%

# %%
#load the data
data = pd.read_csv('/kaggle/input/walmart-sales-forecast/train.csv')
stores = pd.read_csv('/kaggle/input/walmart-sales-forecast/stores.csv')
features = pd.read_csv('/kaggle/input/walmart-sales-forecast/features.csv')

# %%
#inspect the loaded data for the DATA table
data.shape
data.head()
data.info()
data.isna().sum()

# %%
#inspect the loaded data for the STORES table
stores.shape
stores.head()
stores.info()

# %%
#inspect the loaded data for the FEATURES table
features.shape
features.head()
features.info()
features.isna().sum()

# %%
# filling missing values
features['CPI'].fillna(features['CPI'].median(),inplace=True)
features['Unemployment'].fillna(features['Unemployment'].median(),inplace=True)

# %%
# Count unique values in the 'MarkDown1' column
features['MarkDown1'].value_counts().unique()

# %%
# Data cleaning for 'MarkDown' columns
from pandas.core.ops import flex_arith_method_FRAME
for i in range(1,6):
  features["MarkDown"+str(i)] = features["MarkDown"+str(i)].apply(lambda x: 0 if x<0 else x )
  features["MarkDown"+str(i)].fillna(value=0,inplace=True)

# %%
features.info()
features.head()

# %%
# Merge the 'data' dataframe ith the 'stores' and 'features'
data = pd.merge(data,stores,on='Store',how='left')
data = pd.merge(data,features,on=['Store','Date'],how='left')

data.head()

# %%
#convert 'date' column to datetime format, sort the dataframe based on 'date', set the 'date' as index
data['Date'] = pd.to_datetime(data['Date'],errors='coerce')
data.sort_values(by=['Date'],inplace=True)
data.set_index(data.Date, inplace=True)
data.head()

# %%
#check whether the column IsHoliday_x and IsHoliday_y are same or not
data['IsHoliday_x'].isin(data['IsHoliday_y']).all()

# %%
#Since this two columns are same so drop any one column and make another column as IsHoliday
data.drop(columns='IsHoliday_x',inplace=True)
data.rename(columns={"IsHoliday_y" : "IsHoliday"}, inplace=True)
data.info()

# %%
#exctracting the date values
data.head()
data['Year'] = data['Date'].dt.year
data['Month'] = data['Date'].dt.month
data['Week'] = data['Date'].dt.week
data.head()

# %%
#Outlier Detection and Abnormalities
#group the 'data' dataframe by 'Store' and 'Dept' columns and calculates various statistical measures (maximum, minimum, mean, median, and standard deviation) of the 'Weekly_Sales' column for each group.

agg_data = data.groupby(['Store', 'Dept']).Weekly_Sales.agg(['max', 'min', 'mean', 'median', 'std']).reset_index()
agg_data.head()

# %%
agg_data.isnull().sum()

# %%
#merge 'data' with 'agg_data' on 'Store' and 'Dept' using a left join, resulting in 'store_data'.
store_data = pd.merge(left=data,right=agg_data,on=['Store', 'Dept'],how ='left')
store_data.head(2)

# %%
#drop rows with missing values from 'store_data' and assigns the modified DataFrame back to 'data'
store_data.dropna(inplace=True)
data = store_data.copy()

# %%
#Converts the 'Date' column in the 'data' DataFrame to datetime format and sort the dataframe based on 'date' column
data['Date'] = pd.to_datetime(data['Date'],errors='coerce')
data.sort_values(by=['Date'],inplace=True)
data.set_index(data.Date, inplace=True)
data.head()

# %%
#calculate the total markdown value by summing the individual markdown columns
data['Total_MarkDown'] = data['MarkDown1']+data['MarkDown2']+data['MarkDown3']+data['MarkDown4']+data['MarkDown5']
data.drop(['MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5'], axis = 1,inplace=True)
data.head()

# %%
#crate a new dataframe called data_numeric by selecting only the columns specified in the numeric_col list from 'data' dataframe
print(data.shape)
numeric_col = ['Weekly_Sales','Size','Temperature','Fuel_Price','CPI','Unemployment','Total_MarkDown']
data_numeric = data[numeric_col].copy()
print(data_numeric.head())

# %%
#filter the data
data = data[(np.abs(stats.zscore(data_numeric)) < 2.5).all(axis = 1)]
data.shape

# %%
data=data[data['Weekly_Sales']>=0]
data.shape

# %%
data['IsHoliday'] = data['IsHoliday'].astype('int')
data.head()

# %%
#Average Monthly Sales
plt.figure(figsize=(14,8))
sns.barplot(x='Month',y='Weekly_Sales',data=data)
plt.ylabel('Sales',fontsize=14)
plt.xlabel('Months',fontsize=14)
plt.title('Average Monthly Sales',fontsize=16)
plt.savefig('avg_monthly_sales.png')
plt.grid()

# %%
# Monthly Sales for Each Year
data_monthly = pd.crosstab(data["Year"], data["Month"], values=data["Weekly_Sales"],aggfunc='sum')
data_monthly

# %%
#creates a grid of subplots to display the monthly sales for each year. Line graphs are plotted for each month of each year using the 'data_monthly' dataframe.
fig, axes = plt.subplots(3,4,figsize=(16,8))
plt.suptitle('Monthly Sales for each Year', fontsize=18)
k=1
for i in range(3):
    for j in range(4):
      sns.lineplot(ax=axes[i,j],data=data_monthly[k])
      plt.subplots_adjust(wspace=0.4,hspace=0.32)
      plt.ylabel(k,fontsize=12)
      plt.xlabel('Years',fontsize=12)
      k+=1

plt.savefig('monthly_sales_every_year.png')
plt.show()

# %%
# Holiday Distribution
plt.figure(figsize=(8,8))
plt.pie(data['IsHoliday'].value_counts(),labels=['No Holiday','Holiday'],autopct='%0.2f%%')
plt.title("Pie chart distribution",fontsize=14)
plt.legend()
plt.savefig('holiday_distribution.png')
plt.show()

# %%
#One hot encoding
cat_col = ['Store','Dept','Type']
data_cat = data[cat_col].copy()
data_cat.head()

# %%
data_cat = pd.get_dummies(data_cat,columns=cat_col)
data_cat.head()

# %%
print(data.shape)
data = pd.concat([data, data_cat],axis=1)
print(data.shape)

# %%
data.drop(columns=cat_col,inplace=True)
data.drop(columns=['Date'],inplace=True)
data.shape

# %%
#performs min-max normalization on specified numerical columns of the dataframe
num_col = ['Weekly_Sales','Size','Temperature','Fuel_Price','CPI','Unemployment','Total_MarkDown','max','min','mean','median','std']

minmax_scale = MinMaxScaler(feature_range=(0, 1))
def normalization(df,col):
  for i in col:
    arr = df[i]
    arr = np.array(arr)
    df[i] = minmax_scale.fit_transform(arr.reshape(len(arr),1))
  return df

# %%
print(data.head())
data = normalization(data.copy(),num_col)
data.head()

# %%
# Finding Correlation between features
# generate a heatmap to visualize the correlation matrix between the selected numerical features in the dataset
plt.figure(figsize=(15,8))
corr = data[num_col].corr()
sns.heatmap(corr,vmax=1.0,annot=True)
plt.title('Correlation Matrix',fontsize=16)
plt.savefig('correlation_matrix.png')
plt.show()

# %%
# Feature Elimination
feature_col = data.columns.difference(['Weekly_Sales'])
feature_col

# %%
#train a random forest regressor model with 23 decision trees on the selected features and target variable.
radm_clf = RandomForestRegressor(oob_score=True,n_estimators=23)
radm_clf.fit(data[feature_col], data['Weekly_Sales'])

# %%
#computes the feature importance using a random forest regressor and creates a dataframe to store the rankings
indices = np.argsort(radm_clf.feature_importances_)[::-1]
feature_rank = pd.DataFrame(columns = ['rank', 'feature', 'importance'])

for f in range(data[feature_col].shape[1]):
    feature_rank.loc[f] = [f+1,
                           data[feature_col].columns[indices[f]],
                           radm_clf.feature_importances_[indices[f]]]

feature_rank

# %%
#selects the top 25 features from the feature_rank DataFrame and stores them in a list
x=feature_rank.loc[0:25,['feature']]
x=x['feature'].tolist()
print(x)

# %%
X = data[x]
Y = data['Weekly_Sales']

# %%
data = pd.concat([X,Y],axis=1)
data.head()

# %%
#building the model

#Splitting data into train and test data
X = data.drop(['Weekly_Sales'],axis=1)
Y = data.Weekly_Sales

# %%
#Split the data into training and testing sets
X_train,X_test,y_train,y_test = train_test_split(X,Y,test_size=0.20, random_state=50)

# %%
rf = RandomForestRegressor()
rf.fit(X_train, y_train)

# %%
y_pred = rf.predict(X_test)

# %%
print("MAE" , metrics.mean_absolute_error(y_test, y_pred))
print("MSE" , metrics.mean_squared_error(y_test, y_pred))
print("RMSE" , np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
print("R2" , metrics.explained_variance_score(y_test, y_pred))

# %%
rf_df = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})
rf_df.head()

# %%
# generate a plot comparing the predicted values with the actual values for a subset of the test data
plt.figure(figsize=(15,8))
plt.title('Comparison between actual and predicted values',fontsize=16)
plt.plot(rf.predict(X_test[:100]), label="prediction", linewidth=3.0,color='blue')
plt.plot(y_test[:100].values, label="real_values", linewidth=3.0,color='red')
plt.legend(loc="best")
plt.show()


import argparse
import os
import pickle
import re
from collections import defaultdict

import numpy as np

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("filename", nargs="+")
    args = parser.parse_args()

    datasets = defaultdict(list)
    for filename in args.filename:
        match = re.match(r"(.*)-(simcse|bert).pkl", filename)
        if match:
            datasets[match.group(1)].append(filename)

    for dataset_name, dataset_filenames in datasets.items():
        dataset_dict = {"text_embeddings": [], "text_columns": []}

        for filename in sorted(dataset_filenames):
            with open(filename, "rb") as fp:
                current_dict = pickle.load(fp)
            text_embeddings = current_dict.pop("text_embeddings")
            text_columns = current_dict.pop("text_columns")

            dataset_dict.update(current_dict)
            dataset_dict["text_embeddings"].append(text_embeddings)
            dataset_dict["text_columns"] += text_columns

            os.remove(filename)

        dataset_dict["text_embeddings"] = np.concatenate(
            dataset_dict["text_embeddings"], axis=1
        )
        with open(f"{dataset_name}.pkl", "wb") as fp:
            pickle.dump(dataset_dict, fp)


from ghapi.all import GhApi
from doozerlib.pushd import Dir
from dockerfile_parse import DockerfileParser

class CommentOnPr:
    def __init__(self, distgit_dir: str, token: str):
        self.distgit_dir = distgit_dir
        self.token = token
        self.owner = None
        self.repo = None
        self.commit = None
        self.gh_client = None  # GhApi client
        self.pr_url = None
        self.pr_no = None

    def list_comments(self):
        """
        List the comments in a PR
        """
        # https://docs.github.com/rest/reference/issues#list-issue-comments
        return self.gh_client.issues.list_comments(issue_number=self.pr_no, per_page=100)

    def check_if_comment_exist(self, comment):
        """
        Check if the same comment already exists in the PR
        """
        issue_comments = self.list_comments()
        for issue_comment in issue_comments:
            if issue_comment["body"] == comment:
                return True
        return False

    def post_comment(self, comment):
        """
        Post the comment in the PR if the comment doesn't exist already
        """
        # https://docs.github.com/rest/reference/issues#create-an-issue-comment
        if not self.check_if_comment_exist(comment):
            self.gh_client.issues.create_comment(issue_number=self.pr_no, body=comment)
            return True
        return False

    def set_pr_from_commit(self):
        """
        Get the PR from the merge commit
        """
        # https://docs.github.com/rest/commits/commits#list-pull-requests-associated-with-a-commit
        prs = self.gh_client.repos.list_pull_requests_associated_with_commit(self.commit)
        if len(prs) == 1:
            # self._logger.info(f"PR from merge commit {sha}: {pull_url}")
            self.pr_url = prs[0]["html_url"]
            self.pr_no = prs[0]["number"]
            return
        raise Exception(f"Multiple PRs found for merge commit {self.commit}")

    def set_repo_details(self):
        """
        Get the owner, commit and repo from the dfp label
        """
        with Dir(self.distgit_dir):
            dfp = DockerfileParser(str(Dir.getpath().joinpath('Dockerfile')))

            # eg: "https://github.com/openshift/origin/commit/660e0c785a2c9b1fd5fad33cbcffd77a6d84ccb5"
            source_commit_url = dfp.labels["io.openshift.build.commit.url"]
            url_split = source_commit_url.split("/")
            commit = url_split[-1]  # eg: 660e0c785a2c9b1fd5fad33cbcffd77a6d84ccb5
            repo = url_split[-3]  # eg: origin
            owner = url_split[-4]  # eg: openshift

            self.owner = owner
            self.commit = commit
            self.repo = repo

    def set_github_client(self):
        """
        Set the gh client after the get_source_details function is run
        """
        self.gh_client = GhApi(owner=self.owner, repo=self.repo, token=self.token)


import http.client
import hmac
import hashlib
import json
import time
from urllib.parse import urlencode
from typing import Union, Dict

class CBAuth:
    """
    Singleton class for Coinbase authentication.
    """

    _instance = None  # Class attribute to hold the singleton instance

    def __new__(cls):
        """
        Override the __new__ method to control the object creation process.
        :return: A single instance of CBAuth
        """
        if cls._instance is None:
            print("Creating CBAuth instance")
            cls._instance = super(CBAuth, cls).__new__(cls)
            cls._instance.init()
        return cls._instance

    def init(self):
        """
        Initialize the CBAuth instance with API credentials.
        """
        self.key = None
        self.secret = None

    def set_credentials(self, api_key, api_secret):
        """
        Update the API credentials used for authentication.
        :param api_key: The API Key for Coinbase API
        :param api_secret: The API Secret for Coinbase API
        """
        self.key = api_key
        self.secret = api_secret

    def __call__(self, method: str, path: str, body: Union[Dict, str] = '', params: Dict[str, str] = None) -> Dict:
        """
        Prepare and send an authenticated request to the Coinbase API.

        :param method: HTTP method (e.g., 'GET', 'POST')
        :param path: API endpoint path
        :param body: Request payload
        :param params: URL parameters
        :return: Response from the Coinbase API as a dictionary
        """
        path = self.add_query_params(path, params)
        body_encoded = self.prepare_body(body)
        headers = self.create_headers(method, path, body)
        return self.send_request(method, path, body_encoded, headers)

    def add_query_params(self, path, params):
        if params:
            query_params = urlencode(params)
            path = f'{path}?{query_params}'
        return path

    def prepare_body(self, body):
        return json.dumps(body).encode('utf-8') if body else b''

    def create_headers(self, method, path, body):
        timestamp = str(int(time.time()))
        message = timestamp + method.upper() + \
            path.split('?')[0] + (json.dumps(body) if body else '')
        signature = hmac.new(self.secret.encode(
            'utf-8'), message.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()

        return {
            "Content-Type": "application/json",
            "CB-ACCESS-KEY": self.key,
            "CB-ACCESS-SIGN": signature,
            "CB-ACCESS-TIMESTAMP": timestamp
        }

    def send_request(self, method, path, body_encoded, headers):
        conn = http.client.HTTPSConnection("api.coinbase.com")
        try:
            conn.request(method, path, body_encoded, headers)
            res = conn.getresponse()
            data = res.read()

            if res.status == 401:
                print("Error: Unauthorized. Please check your API key and secret.")
                return None

            response_data = json.loads(data.decode("utf-8"))
            if 'error_details' in response_data and response_data['error_details'] == 'missing required scopes':
                print(
                    "Error: Missing Required Scopes. Please update your API Keys to include more permissions.")
                return None

            return response_data
        except json.JSONDecodeError:
            print("Error: Unable to decode JSON response. Raw response data:", data)
            return None
        finally:
            conn.close()


from abc import ABC, abstractmethod
from ..common_imports import *
from typing import Literal
from ..core.config import parse_output_idx, Config
from ..core.model import Ref
from ..core.wrapping import unwrap
from ..core.tps import ListType, StructType, DictType, SetType
from .weaver import (
    ValNode,
    CallNode,
    StructOrientations,
    get_items,
    get_elts,
    get_elt_and_struct,
    get_idx,
    is_key,
    is_idx,
    traverse_all,
    get_vq_orientation,
)
from .graphs import InducedSubgraph, get_canonical_order
from ..ui.viz import (
    Node,
    Edge,
    SOLARIZED_LIGHT,
    to_dot_string,
    write_output,
    HTMLBuilder,
    Cell,
)
import textwrap

class ValueLoaderABC(ABC):
    @abstractmethod
    def load_value(self, full_uid: str) -> Any:
        raise NotImplementedError

class GraphPrinter:
    def __init__(
        self,
        vqs: Set[ValNode],
        fqs: Set[CallNode],
        value_loader: Optional[ValueLoaderABC] = None,
        names: Optional[Dict[ValNode, str]] = None,
    ):
        self.vqs = vqs
        self.fqs = fqs
        self.value_loader = value_loader
        self.g = InducedSubgraph(vqs=vqs, fqs=fqs)
        self.v_labels, self.f_labels, self.vq_to_node = self.g.canonicalize()
        self.full_topsort, self.sources, self.sinks = self.g.topsort(
            canonical_labels={**self.v_labels, **self.f_labels}
        )
        if names is None:
            names = get_names(
                hints={}, canonical_order=get_canonical_order(vqs=vqs, fqs=fqs)
            )
        self.names = names

    def get_struct_comment(
        self,
        vq: ValNode,
        elt_names: Tuple[str, ...],
        idx_names: Optional[Tuple[str, ...]] = None,
    ) -> str:
        id_to_name = {"__list__": "list", "__dict__": "dict", "__set__": "set"}
        if len(elt_names) == 1:
            s = f"{self.names[vq]} will match any {id_to_name[vq.tp.struct_id]} containing a match for {elt_names[0]}"
            if idx_names is not None:
                s += f" at index {idx_names[0]}"
        else:
            s = f'{self.names[vq]} will match any {id_to_name[vq.tp.struct_id]} containing matches for each of {", ".join(elt_names)}'
            if idx_names is not None:
                s += f" at indices {', '.join(idx_names)}"
        return s

    def get_source_comment(self, vq: ValNode) -> str:
        if is_idx(vq=vq):
            return "index into list"
        elif is_key(vq=vq):
            return "key into dict"
        else:
            return "input to computation; can match anything"

    def get_construct_computation_rhs(self, node: ValNode) -> str:
        """
        Given a constructive struct, return
            [name_0, ..., name_1] for lists
            {key_0: name_0, ..., key_n: name_n} for dicts
            {name_0, ..., name_n} for sets
        """
        if isinstance(node.tp, (ListType, DictType)):
            idxs_and_elts = list(get_items(vq=node).values())
            idxs = [x[0] for x in idxs_and_elts]
            idx_values = [
                unwrap(self.value_loader.load_value(full_uid=idx.constraint[0]))
                for idx in idxs
            ]
            elts = [x[1] for x in idxs_and_elts]
            elt_names = tuple([str(self.names[elt]) for elt in elts])
            idx_value_to_elt_name = {
                idx_value: elt_name
                for idx_value, elt_name in zip(idx_values, elt_names)
            }
            if isinstance(node.tp, ListType):
                assert sorted(idx_value_to_elt_name.keys()) == list(range(len(elts)))
                rhs = f'[{", ".join([idx_value_to_elt_name[i] for i in range(len(elts))])}]'
            elif isinstance(node.tp, DictType):
                elt_strings = [
                    f"'{idx_value}': {elt_name}"
                    for idx_value, elt_name in idx_value_to_elt_name.items()
                ]
                rhs = f'{", ".join(elt_strings)}'
                rhs = f"{{{rhs}}}"
            else:
                raise RuntimeError
        elif isinstance(node.tp, SetType):
            elts = list(get_elts(vq=node).values())
            elt_names = tuple([str(self.names[elt]) for elt in elts])
            rhs = f'{", ".join(elt_names)}'
            rhs = f"{{{rhs}}}"
        else:
            raise ValueError
        return rhs

    def get_construct_query_rhs(self, node: ValNode) -> str:
        if isinstance(node.tp, (ListType, DictType)):
            idxs_and_elts = list(get_items(vq=node).values())
            idxs = [x[0] for x in idxs_and_elts]
            elts = [x[1] for x in idxs_and_elts]
            elt_names = tuple([str(self.names[elt]) for elt in elts])
            idx_names = tuple(
                [str(self.names[idx]) if idx is not None else "?" for idx in idxs]
            )
            comment = self.get_struct_comment(
                vq=node, elt_names=elt_names, idx_names=idx_names
            )
            if isinstance(node.tp, ListType):
                rhs = f'ListQ(elts=[{", ".join(elt_names)}], idxs=[{", ".join(idx_names)}]) # {comment}'
            elif isinstance(node.tp, DictType):
                elt_strings = [
                    f"{idx_name}: {elt_name}"
                    for idx_name, elt_name in zip(idx_names, elt_names)
                ]
                rhs = f'DictQ(dct={{..., {", ".join(elt_strings)}, ...}}) # {comment}'
            else:
                raise RuntimeError
        elif isinstance(node.tp, SetType):
            elts = list(get_elts(vq=node).values())
            elt_names = tuple([str(self.names[elt]) for elt in elts])
            comment = self.get_struct_comment(vq=node, elt_names=elt_names)
            rhs = f'SetQ(elts=[{", ".join(elt_names)}]) # {comment}'
        else:
            raise ValueError
        return rhs

    def get_op_computation_line(self, node: CallNode) -> str:
        full_returns = node.returns_interp
        full_returns_names = [
            self.names[vq] if vq is not None else "_" for vq in full_returns
        ]
        lhs = ", ".join(full_returns_names)
        # rhs
        args_dict = {arg_name: self.names[vq] for arg_name, vq in node.inputs.items()}
        for inp_name in node.func_op.sig.input_names:
            if inp_name not in args_dict:
                raise RuntimeError
        args_string = ", ".join([f"{k}={v}" for k, v in args_dict.items()])
        rhs = f"{node.func_op.sig.ui_name}({args_string})"
        if len(lhs) == 0:
            return rhs
        else:
            return f"{lhs} = {rhs}"

    def get_op_query_line(self, node: CallNode) -> str:
        # lhs
        full_returns = node.returns_interp
        full_returns_names = [
            self.names[vq] if vq is not None else "_" for vq in full_returns
        ]
        lhs = ", ".join(full_returns_names)
        # rhs
        args_dict = {arg_name: self.names[vq] for arg_name, vq in node.inputs.items()}
        for inp_name in node.func_op.sig.input_names:
            if inp_name not in args_dict:
                args_dict[inp_name] = "Q()"
        args_string = ", ".join([f"{k}={v}" for k, v in args_dict.items()])
        rhs = f"{node.func_op.sig.ui_name}({args_string})"
        if len(lhs) == 0:
            return rhs
        else:
            return f"{lhs} = {rhs}"

    def get_destruct_computation_line(self, node: CallNode) -> str:
        elt, struct = get_elt_and_struct(fq=node)
        idx = get_idx(fq=node)
        lhs = f"{self.names[elt]}"
        if idx is None:
            raise NotImplementedError
        # inline the index value if the index is a source in the graph
        if idx in self.sources:
            assert idx.constraint is not None
            assert len(idx.constraint) == 1
            idx_ref = self.value_loader.load_value(full_uid=idx.constraint[0])
            idx_value = unwrap(idx_ref)
            if isinstance(idx_value, int):
                rhs = f"{self.names[struct]}[{idx_value}]"
            elif isinstance(idx_value, str):
                rhs = f"{self.names[struct]}['{idx_value}']"
            else:
                raise RuntimeError
        else:
            rhs = f"{self.names[struct]}[{self.names[idx]}]"
        return f"{lhs} = {rhs}"

    def get_destruct_query_line(self, node: CallNode) -> str:
        elt, struct = get_elt_and_struct(fq=node)
        idx = get_idx(fq=node)
        lhs = f"{self.names[elt]}"
        if idx is None:
            idx_label = "?"
        else:
            idx_label = self.names[idx]
        rhs = f"{self.names[struct]}[{idx_label}] # {self.names[elt]} will match any element of a match for {self.names[struct]} at index matching {idx_label}"
        return f"{lhs} = {rhs}"

    def print_computational_graph(self, show_sources_as: str = "values") -> str:
        res = []
        for node in self.full_topsort:
            if isinstance(node, ValNode):
                if node in self.sources:
                    if is_idx(node) or is_key(node):
                        # exclude indices/keys if they are sources (we will inline them)
                        continue
                    assert node.constraint is not None
                    assert len(node.constraint) == 1
                    if show_sources_as == "values":
                        ref = self.value_loader.load_value(full_uid=node.constraint[0])
                        value = unwrap(ref)
                        rep = textwrap.shorten(str(value), 25)
                        res.append(f"{self.names[node]} = {rep}")
                    elif show_sources_as == "uids":
                        uid, causal_uid = Ref.parse_full_uid(
                            full_uid=node.constraint[0]
                        )
                        res.append(
                            f"{self.names[node]} = Ref(uid={uid}, causal_uid={causal_uid})"
                        )
                    else:
                        raise ValueError
                elif isinstance(node.tp, StructType):
                    if get_vq_orientation(node) != StructOrientations.construct:
                        continue
                    rhs = self.get_construct_computation_rhs(node=node)
                    lhs = f"{self.names[node]}"
                    line = f"{lhs} = {rhs}"
                    res.append(line)
            elif isinstance(node, CallNode):
                if not node.func_op.is_builtin:
                    res.append(self.get_op_computation_line(node=node))
                else:
                    if node.orientation == StructOrientations.destruct:
                        res.append(self.get_destruct_computation_line(node=node))
        return "\n".join(res)

    def print_query_graph(
        self,
        selection: Optional[List[ValNode]],
        pprint: bool = False,
    ):
        res = []
        for node in self.full_topsort:
            if isinstance(node, ValNode):
                if node in self.sources:
                    comment = self.get_source_comment(vq=node)
                    res.append(f"{self.names[node]} = Q() # {comment}")
                elif isinstance(node.tp, StructType):
                    if get_vq_orientation(node) != StructOrientations.construct:
                        continue
                    rhs = self.get_construct_query_rhs(node=node)
                    lhs = f"{self.names[node]}"
                    line = f"{lhs} = {rhs}"
                    res.append(line)
            elif isinstance(node, CallNode):
                if not node.func_op.is_builtin:
                    res.append(self.get_op_query_line(node=node))
                else:
                    if node.orientation == StructOrientations.destruct:
                        res.append(self.get_destruct_query_line(node=node))
        if selection is not None:
            res.append(
                f"result = storage.df({', '.join([self.names[vq] for vq in selection])})"
            )
        res = [textwrap.indent(line, "    ") for line in res]
        if Config.has_rich and pprint:
            from rich.syntax import Syntax

            highlighted = Syntax(
                "\n".join(res),
                "python",
                theme="solarized-light",
                line_numbers=False,
            )
            # return Panel.fit(highlighted, title="Computational Graph")
            return highlighted
        else:
            return "\n".join(res)

def print_graph(
    vqs: Set[ValNode],
    fqs: Set[CallNode],
    names: Dict[ValNode, str],
    selection: Optional[List[ValNode]],
    pprint: bool = False,
):
    printer = GraphPrinter(vqs=vqs, fqs=fqs, names=names)
    s = printer.print_query_graph(selection=selection)
    if Config.has_rich and pprint:
        rich.print(s)
    else:
        print(s)

def graph_to_dot(
    vqs: List[ValNode],
    fqs: List[CallNode],
    names: Dict[ValNode, str],
    layout: Literal["computational", "bipartite"] = "computational",
    memoization_tables: Optional[Dict[CallNode, pd.DataFrame]] = None,
) -> str:
    # should work for subgraphs
    assert set(vqs) <= set(names.keys())
    nodes = {}  # val/op query -> Node obj
    edges = []
    col_names = []
    counter = 0
    for vq in vqs:
        if names[vq] is not None:
            col_names.append(names[vq])
        else:
            col_names.append(f"unnamed_{counter}")
            counter += 1
    for vq, col_name in zip(vqs, col_names):
        html_label = HTMLBuilder()
        if hasattr(vq, "_hidden_message"):
            msg = vq._hidden_message
            text = f"{col_name} ({msg})"
        else:
            text = str(col_name)
        html_label.add_row(
            cells=[
                Cell(
                    text=text,
                    port=None,
                    bold=True,
                    bgcolor=SOLARIZED_LIGHT["orange"],
                    font_color=SOLARIZED_LIGHT["base3"],
                )
            ]
        )
        node = Node(
            internal_name=str(id(vq)),
            # label=str(val_query.column_name),
            label=html_label.to_html_like_label(),
            color=SOLARIZED_LIGHT["blue"],
            shape="plain",
        )
        nodes[vq] = node
    for func_query in fqs:
        html_label = HTMLBuilder()
        func_preview = (
            f'{func_query.displayname}({", ".join(func_query.inputs.keys())})'
        )
        if hasattr(func_query, "_hidden_message"):
            msg = func_query._hidden_message
            func_preview = f"{func_preview} ({msg})"
        title_cell = Cell(
            text=func_preview,
            port=None,
            bgcolor=SOLARIZED_LIGHT["blue"],
            bold=True,
            font_color=SOLARIZED_LIGHT["base3"],
        )
        input_cells = []
        output_cells = []
        for input_name in func_query.inputs.keys():
            input_cells.append(Cell(text=input_name, port=input_name, bold=True))
            # html_label.add_row(elts=[Cell(text=input_name, port=input_name)])
        for output_idx in range(len(func_query.outputs)):
            output_cells.append(
                Cell(
                    text=f"output_{output_idx}", port=f"output_{output_idx}", bold=True
                )
            )
        if layout == "bipartite":
            html_label.add_row(cells=[title_cell])
            if len(input_cells + output_cells) > 0:
                html_label.add_row(cells=input_cells + output_cells)
            if memoization_tables is not None:
                column_names = [cell.text for cell in input_cells + output_cells]
                port_names = [cell.port for cell in input_cells + output_cells]
                # remove ports from the table column cells
                for cell in input_cells + output_cells:
                    cell.port = None
                df = memoization_tables[func_query][column_names]
                rows = list(df.head().itertuples(index=False))
                for tup in rows:
                    html_label.add_row(
                        cells=[
                            Cell(text=textwrap.shorten(str(x), 25), bold=True)
                            for x in tup
                        ]
                    )
                # add port names to the cells in the *last* row
                for cell, port_name in zip(html_label.rows[-1], port_names):
                    cell.port = port_name
        elif layout == "computational":
            if len(input_cells) > 0:
                html_label.add_row(input_cells)
            html_label.add_row([title_cell])
            if len(output_cells) > 0:
                html_label.add_row(output_cells)
        else:
            raise ValueError(f"Unknown layout: {layout}")
        node = Node(
            internal_name=str(id(func_query)),
            # label=str(func_query.func_op.sig.ui_name),
            label=html_label.to_html_like_label(),
            color=SOLARIZED_LIGHT["red"],
            shape="plain",
        )
        nodes[func_query] = node
        for input_name, vq in func_query.inputs.items():
            if not vq in nodes:
                continue
            if layout == "bipartite":
                edges.append(
                    Edge(
                        target_node=nodes[vq],
                        source_node=nodes[func_query],
                        source_port=input_name,
                        arrowtail="none",
                        arrowhead="none",
                    )
                )
            elif layout == "computational":
                edges.append(
                    Edge(
                        source_node=nodes[vq],
                        target_node=nodes[func_query],
                        target_port=input_name,
                    )
                )
            else:
                raise ValueError(f"Unknown layout: {layout}")
        for output_name, vq in func_query.outputs.items():
            if not vq in nodes:
                continue
            edges.append(
                Edge(
                    source_node=nodes[func_query],
                    target_node=nodes[vq],
                    source_port=output_name,
                    arrowtail="none" if layout == "bipartite" else None,
                    arrowhead="none" if layout == "bipartite" else None,
                )
            )
    return to_dot_string(nodes=list(nodes.values()), edges=edges, groups=[])

def visualize_graph(
    vqs: Set[ValNode],
    fqs: Set[CallNode],
    names: Optional[Dict[ValNode, str]],
    layout: Literal["computational", "bipartite"] = "computational",
    memoization_tables: Optional[Dict[CallNode, pd.DataFrame]] = None,
    output_path: Optional[Path] = None,
    show_how: Literal["none", "browser", "inline", "open"] = "none",
):
    if names is None:
        names = get_names(
            hints={}, canonical_order=get_canonical_order(vqs=vqs, fqs=fqs)
        )
    dot_string = graph_to_dot(
        vqs=list(vqs),
        fqs=list(fqs),
        names=names,
        layout=layout,
        memoization_tables=memoization_tables,
    )
    if output_path is None:
        tempfile_obj, output_name = tempfile.mkstemp(suffix=".svg")
        output_path = Path(output_name)
    write_output(
        output_path=output_path,
        dot_string=dot_string,
        output_ext="svg",
        show_how=show_how,
    )
    return output_path

def show(*vqs: ValNode):
    vqs, fqs = traverse_all(vqs=list(vqs), direction="both")
    visualize_graph(vqs=vqs, fqs=fqs, show_how="browser")

def extract_names_from_scope(scope: Dict[str, Any]) -> Dict[ValNode, str]:
    """
    Heuristic to get deterministic name for all ValQueries we can find in the
    scope.
    """
    names_per_vq = defaultdict(list)
    for k, v in scope.items():
        if k.startswith("_"):
            continue
        if isinstance(v, Ref) and v._query is not None:
            names_per_vq[v.query].append(k)
        elif isinstance(v, ValNode):
            names_per_vq[v].append(k)
    res = {}
    for vq, names_per_vq in names_per_vq.items():
        if len(names_per_vq) > 1:
            logger.warning(
                f"Found multiple names for {vq}: {names_per_vq}, choosing {sorted(names_per_vq)[0]}"
            )
        res[vq] = names_per_vq[0]
    return res

def get_names(
    hints: Dict[ValNode, str], canonical_order: List[ValNode]
) -> Dict[ValNode, str]:
    """
    Get names for the given oredered list of ValQueries, using the following
    priority:
        - from the object itself;
        - the hints;
        - a_{counter} for the rest.
    """
    counter = Count()
    idx_counter = Count()
    key_counter = Count()
    existing_names = set(hints.values()) | {
        vq.name for vq in canonical_order if vq.name is not None
    }
    res = {}
    for vq in canonical_order:
        if vq.name is not None:
            res[vq] = vq.name
        elif vq in hints:
            res[vq] = hints[vq]
        else:
            if is_key(vq):
                c, prefix = key_counter, "key"
            elif is_idx(vq):
                c, prefix = idx_counter, "idx"
            else:
                c, prefix = counter, "a"
            while f"{prefix}{c.i}" in existing_names:
                c.i += 1
            res[vq] = f"{prefix}{c.i}"
            c.i += 1
    return res

class Count:
    def __init__(self):
        self.i = 0


from django.shortcuts import render
from django.views.generic import TemplateView
from gandhara.models import Users

class IndexView(TemplateView):
    template_name = 'index.html'

    def get_context_data(self):
        ctxt = super().get_context_data()
        ctxt['users'] = Users.objects.all()
        return ctxt

class AboutView(TemplateView):
    template_name = 'about.html'

    def get_context_data(self):
        ctxt = super().get_context_data()
        ctxt['num_services'] = 123456790
        ctxt['skills'] = [
            "Python",
            "C++",
            "JavaScript",
            "PHP",
        ]
        return ctxt


'''
@Description:
@Author: Jack Huang
@Github: https://github.com/HuangJiaLian
@Date: 2019-10-10 21:57:24
@LastEditors: Jack Huang
@LastEditTime: 2019-10-10 22:34:46
'''

import pandas as pd
import os
import matplotlib.pyplot as plt

class logger:
    def __init__(self, logger_name, logger_path, col_names, restore=False):
        self.logger_name = logger_name
        if not os.path.exists(logger_path):
            os.makedirs(logger_path)
        self.logger_path = logger_path
        self.col_names = col_names
        self.save_path = os.path.join(self.logger_path, self.logger_name + '.csv')
        if restore:
            self.dataLogs = self.load()
        else:
            self.dataLogs = pd.DataFrame(columns=col_names)

    def add_row_data(self, one_row, saveFlag=False):
        last_index = len(self.dataLogs)
        self.dataLogs.loc[last_index] = one_row
        if saveFlag:
            self.save()

    def save(self):
        self.dataLogs.to_csv(self.save_path)

    def load(self):
        dataLogs_Saved = pd.read_csv(self.save_path)
        return dataLogs_Saved

    def plotToFile(self, title, showFlag = False):
        plt.clf()
        plt.title(title)
        ax = plt.gca()
        self.dataLogs.plot(kind='line', x = self.col_names[0], y=self.col_names[1], ax=ax)
        self.dataLogs.plot(kind='line', x = self.col_names[0], y=self.col_names[2], ax=ax)
        self.dataLogs.plot(kind='line', x = self.col_names[0], y=self.col_names[3], ax=ax)
        self.dataLogs.plot(kind='line', x = self.col_names[0], y=self.col_names[4], ax=ax)
        plt.savefig(os.path.join(self.logger_path, self.logger_name + '.png'))
        if showFlag:
            plt.show()

    def close(self):
        self.save()


import json
import os

import data
from constants import SC_EMOJI, UP_ARROW_EMOJI

items = []

class Item():
    def __init__(self, item_dict):
        self.name = item_dict["name"]
        self.cost = item_dict.get("cost", None)
        self.emoji = item_dict["emoji"]
        self.description = item_dict["description"]
        self.aliases = item_dict["aliases"]
        self.shop_item = item_dict.get("shop_item", False)
        self.type = item_dict.get("type", "standard")
        self.boost_bonus = item_dict.get("boost", {}).get("bonus", None)
        self.boost_max_bonus = item_dict.get("boost", {}).get("max_bonus", None)
        self.boost_category = item_dict.get("boost", {}).get("category", None)

    def can_be_in_shop(self):
        return self.shop_item

    def has_value(self):
        return not self.cost is None

    def is_booster(self):
        return not self.boost_bonus is None

    def get_shop_string(self):
        # Brackets combine the multiple lines into one
        return (
            f'{self.emoji} **{self.name}** ─ {SC_EMOJI}{self.cost}' +
            (f' ─ {UP_ARROW_EMOJI}**{self.boost_category.capitalize()}**' if self.boost_category else '') +
            f'\n{self.description}\n\n'
        )

def import_items():
    global items
    items = []
    items_data = data.get_items()
    for i in items_data:
        items.append(Item(i))

def get_by_name(name):
    global items
    for i in items:
        if name.lower() == i.name.lower() or name.lower() in i.aliases:
            return i
    return None

def get_player_boost(player_id, *categories):
    """Get a player boost for a specific category.
    An output of 0 means a 0% bonus, 1 means an 100% boost.
    """
    boost = 0
    for name, amount in data.get_data(player_id, "inv", default_val={}).items():
        item = get_by_name(name)
        if item.is_booster():
            if item.boost_category in categories:
                boost += get_boost_from_item_amount(item, amount)
    return boost

def get_boost_from_item_amount(item, amount):
    """Get the amount of boost a player gets from a specific item."""
    if not item.is_booster():
        return 0
    boost = min(amount * item.boost_bonus, item.boost_max_bonus)
    return boost

def get_player_boost_from_item(player_id, item):
    """Get the amount of boost a player gets from a specific item."""
    if not item.is_booster():
        return 0
    amount = data.get_data(player_id, "inv", item.name, default_val=0)
    boost = min(amount * item.boost_bonus, item.boost_max_bonus)
    return boost


# -*- coding: utf-8 -*-
"""
Created on September 2019

Description:
In this script we synthesize the weighting function for windowing in time domain.
%Author: Dami�n Dellavale.

%Refs:
%Matlab -> Help -> window
%http://www.mathworks.com/help/signal/ref/window.html
%https://en.wikipedia.org/wiki/Window_function

@author: Osvaldo M Velarde

@title: Window.
"""

import scipy.signal

# Observation - Relations (sigma-alpha-XdB):
# sigma = (n-1)/(2*alpha)
# alpha = sqrt(-2*XdB/(10 log10 e)) // Power attenuation
# alpha = sqrt(-XdB/(10 log10 e)) // Amplitude attenuation (usual)
# En general, XdB=-30.

def function_window(windowParam, windowLength):
    """
    Inputs:
        - windowParam: Structure. Parameters of the window function.
                       - 'name':  String {'gausswin','hamming','hann','tukey','rectwin'}.
                                  Name defining the window type.
                       - 'sigma': Numeric value.
                                  Parameter for the gausswin:standard deviation of a
                                  Gaussian random variable.
                       - 'sflag': {False ('periodic'), True('symmetric')(default)}.
                                  Sampling parameter for hamming and hann windows.
                       - 'r':     Numeric value.
                                  *If r in (0,1): A Tukey window is a rectangular window with the
                                  first and last 100*r/2 percent of the samples equal to parts of a cosine.
                                  *If you input r ≤ 0, you obtain a rectwin window.
                                  *If you input r ≥ 1, you obtain a hann window.

        - windowLength: Int value. Length of the window function.

    Outputs:
        - windowFunction. Numeric array (). Synthesized window function.

    % Info sflag:
    The 'periodic' flag is useful for DFT/FFT purposes, such as in spectral analysis.
    The DFT/FFT contains an implicit periodic extension and the periodic flag enables a signal windowed
    with a periodic window to have perfect periodic extension. When 'periodic' is specified,
    hamming/hann computes a length "windowLength+1" window and returns the first "windowLength" points.

    When using windows for filter design, the 'symmetric' flag should be used.
    """

# %Argument completion ------------------------------------------------------
# if (nargin < 2)||isempty(windowParam)||isempty(windowLength),...
#    error('MATLAB:function_window','Input argument error.');
# end
# %--------------------------------------------------------------------------

# %Check the input arguments ------------------------------------------------
# assert(isstruct(windowParam), 'Input argument error in function "function_window": windowParam must be a structure array.');
# %--------------------------------------------------------------------------

    LIST_WINDOWS = {'gausswin':scipy.signal.gaussian,
                    'hamming':scipy.signal.hamming,
                    'hann':scipy.signal.hann,
                    'tukey':scipy.signal.tukey}

    LIST_PARAMETERS = {'gausswin':'sigma',
                       'hamming':'sflag',
                       'hann':'sflag',
                       'tukey':'r'}

    name = windowParam['name']

    if name in LIST_WINDOWS.keys():
        functwin = LIST_WINDOWS.get(name, lambda:"Invalid method")
        parameter = windowParam[LIST_PARAMETERS[name]]
        return functwin(windowLength,parameter)
    else:
        return scipy.signal.boxcar(windowLength)


# -*- coding: utf-8 -*-
import sys
import argparse
import math
# http://braeunig.us/space/index.htm
# http://braeunig.us/space/comb.htm

# Gas constant (J/(K*kmol))
R = 8314.4598

def exhaustVelocity(chamberP, chamberT, k, molarM, exitP):
	v = math.sqrt((2*k/(k-1))*(R*chamberT/molarM)*(1.0-(exitP/chamberP)**((k-1.0)/k)))
	return v

def throatP(chamberP, k):
	p = chamberP*((1.0 + (k-1)/2.0)**(-k/(k-1.0)))
	return p

def throatT(chamberT, k):
	t = chamberT/(1.0 + (k-1.0)/2.0)
	return t

def MachNum(chamberP, k, exitP):
	Mn = math.sqrt( (2.0/(k-1.0))*((chamberP/exitP)**((k-1.0)/k) - 1.0) )
	return Mn

def exitArea(chamberP, chamberT, k, molarM, exitP, A_t):
	Mn = MachNum(chamberP, k, exitP)
	#print ("MachN: " + str(Mn))
	A_e = (A_t/Mn)*( (1.0 + (k-1.0)/2.0 * Mn*Mn)/((k+1.0)/2.0) )**((k+1.0)/(2.0*(k-1.0)))
	return A_e

def main():

	# Parse commandline
	cmdLine = argparse.ArgumentParser()

	cmdLine.add_argument("massFlow", type=float, help="Thrust chamber mass flow")
	cmdLine.add_argument("chamberPress", type=float, help="Thrust chamber pressure")
	cmdLine.add_argument("temp", type=float, help="Thrust chamber temperature")
	cmdLine.add_argument("k", type=float, help="Spesific heat ratio")
	cmdLine.add_argument("areaRatio", type=float, help="Nozzle area ratio")
	cmdLine.add_argument("molarMass", type=float, help="Exhaust molar mass")

	args = cmdLine.parse_args()

	lowerP = 0.0
	# exit pressure can't be higher than throat pressure
	upperP = throatP(args.chamberPress, args.k)
	testP = (lowerP+upperP)/2.0
	#print ("starting p: " + str(testP))

	# Throat parameters
	P_t = throatP(args.chamberPress, args.k)
	T_t = throatT(args.temp, args.k)
	A_t = args.massFlow/P_t * math.sqrt(R*T_t/(args.molarMass*args.k))

	# Loop to find correct pressure on nozzle-exit
	iters = 0;
	while (1):
		iters = iters + 1
		v_e = exhaustVelocity(args.chamberPress, args.temp, args.k, args.molarMass, testP)
		A_e = exitArea(args.chamberPress, args.temp, args.k, args.molarMass, testP, A_t)

		# Do a binary search to find correct exit-pressure for the given area ratio
		if (A_e > A_t*args.areaRatio):
			lowerP = testP
			testP = (upperP+testP)/2.0
		else:
			upperP = testP
			testP = (lowerP+testP)/2.0

		#print ("New p: " + str(testP) + " area ratio with prev iter: " + str(A_e/A_t))

		# Exit loop when acceptable precision is reached
		if (math.fabs(args.areaRatio-(A_e/A_t)) < 0.01):
			break

	#print ("Iterations: " + str(iters))

	thrust_m = v_e * args.massFlow
	thrust_s = thrust_m + (testP - 101325)*A_e
	thrust_v = thrust_m + testP*A_e

	# 80% bell-nozzle length
	# Length percentage is relative to 15 deg half-angle conical nozzle
	aRatio = A_e/A_t
	n_radius = math.sqrt(A_e/math.pi)
	n_length = n_radius/math.tan(math.radians(15))*(math.sqrt(aRatio)-1)/math.sqrt(aRatio)
	n_length = n_length*0.8

	print ("Exhaust velocity: {0:.1f}".format(v_e) + " m/s")
	print ("Throat: {0:.6f}".format(A_t) + " m² {0:.0f}".format(T_t) + " K")
	print ("Exit: {0:.6f}".format(A_e) + " m² Area ratio: {0:.3f}".format(aRatio))
	print ("Pressure at nozzle exit: {0:.0f}".format(testP) + " Pa")
	print ("\nThrust")
	print (" Sea level: {0:.0f}".format(thrust_s) + " N")
	print (" Vacuum:    {0:.0f}".format(thrust_v) + " N")

	print ("\nBell-nozzle 80%:\n Length:   {0:.3f}".format(n_length) + " m")
	print (" Diameter: {0:.3f}".format(n_radius*2) + " m")

if __name__ == '__main__':
        try:
                main()
        except KeyboardInterrupt:
                sys.exit(1)