import sys
import calendar
import numpy as np

from PyQt5 import QtGui, QtCore
from PyQt5.QtCore import QPropertyAnimation
from PyQt5.QtWidgets import *

from main_gui import Ui_MainWindow
from formating import *
from calendar_widget import *
from stylesheets import *
from sql_functions import *
from uiFunctions import *
from flowlayout import FlowLayout

from matplotlib.patches import FancyBboxPatch
from mlpcanvas import *
from datetime import datetime, date, timedelta

class MainWindow(QMainWindow):
    widget_task_ids = {}
    task_overviews_per_month = [None for i in range(12)]

    def __init__(self):
        QMainWindow.__init__(self)
        self.db_task = SQLTaskConnection("task_data")
        self.db_joint = SQLJointConnection("data_joint")
        self.year = datetime.today().year
        self.month = datetime.today().month

        self.resize(QtCore.QSize(850, 600))
        self.setMinimumSize(QtCore.QSize(850, 600))
        self.setStyleSheet(scrollArea_style)

        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.init_ui()
        self.init_menu_width = self.ui.frame_left_menu.minimumWidth()

        self.show()

    def init_ui(self):
        self.init_flow_layouts()
        self.init_tasks_todo()
        self.init_tasks()
        self.init_month_statistics()
        self.init_effects()
        self.connect_buttons()
        self.compare_accomplished_tasks_statistics()

        self.ui.Pages_Widget.setCurrentWidget(self.ui.page_show_tasks)
        self.ui.day_field.setValidator(QtGui.QIntValidator())

    def init_flow_layouts(self):
        # flow layouts need to be initialized by code
        layout = FlowLayout()
        widget = self.ui.tasks_todo_widget
        widget.setLayout(layout)

    def init_effects(self):
        add_shadow_effect(self.ui.frame_left_menu)
        add_shadow_effect(self.ui.task_overview)
        add_shadow_effect(self.ui.history)
        add_shadow_effect(self.ui.tasks_todo)
        add_shadow_effect(self.ui.add_task_widget)
        add_shadow_effect(self.ui.widget_2)
        add_shadow_effect(self.ui.highlights_widget)
        add_shadow_effect(self.ui.effect_test_2, 5)

    def connect_buttons(self):
        self.ui.Btn_Menu_2.clicked.connect(lambda: self.change_page(self.ui.page_add_task))
        self.ui.Btn_Menu_1.clicked.connect(lambda: self.change_page(self.ui.page_show_tasks))
        self.ui.Btn_Menu_3.clicked.connect(self.show_statistics)
        self.ui.Button_AddTask.clicked.connect(self.insert_new_task)
        self.ui.button_last_year.clicked.connect(lambda: self.change_calendar_year(-1))
        self.ui.button_next_year.clicked.connect(lambda: self.change_calendar_year(+1))
        self.ui.button_last_month.clicked.connect(lambda: self.change_tasks_month(-1))
        self.ui.button_next_month.clicked.connect(lambda: self.change_tasks_month(+1))
        self.ui.button_last_month_2.clicked.connect(lambda: self.change_tasks_month(-1))
        self.ui.button_next_month_2.clicked.connect(lambda: self.change_tasks_month(+1))

    def custom_context_menu(self, event):
        contextMenu = QMenu(self)
        delete = contextMenu.addAction("Delete")
        cancel_task = contextMenu.addAction("Cancel Task")
        action = contextMenu.exec_(self.mapToGlobal(self.mapFromGlobal(QtGui.QCursor.pos())))
        if action == delete:
            self.show_delete_warning(self.sender())
        elif action == cancel_task:
            self.show_cancel_task_warning(self.sender())

    def show_cancel_task_warning(self, sender):
        msg = QMessageBox()
        msg.setWindowTitle("Cancel task")
        msg.setText("Do you really want to stop this task early?")
        msg.setIcon(QMessageBox.Warning)
        msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
        msg.setDefaultButton(QMessageBox.Cancel)
        msg.buttonClicked.connect(lambda x: self.cancel_task(x, msg, sender))
        msg.exec()

    def show_delete_warning(self, sender):
        msg = QMessageBox()
        msg.setWindowTitle("Deleting Task")
        msg.setText("Do you really want to delete this task entirely?")
        msg.setIcon(QMessageBox.Warning)
        msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
        msg.setDefaultButton(QMessageBox.Cancel)
        msg.buttonClicked.connect(lambda x: self.delete_task(x, msg, sender))
        msg.exec()

    def delete_task(self, button, msg, sender):
        if button == msg.button(QMessageBox.Cancel):
            return
        id = self.widget_task_ids[sender]
        sender.deleteLater()

        # delete label on todo tasks
        name = self.db_task.sql_query("SELECT taskName FROM tasks WHERE id = {}".format(id))
        name = name[0][0]

        label = self.ui.tasks_todo_widget.findChild(QLabel, name)
        if label:
            # label was on the todo list
            label.deleteLater()

        # delete from db
        res = self.db_task.sql_query_commit("DELETE FROM tasks WHERE id={}".format(id))
        res = self.db_joint.sql_query_commit("DELETE FROM data_joint WHERE task_id={}".format(id))

    def cancel_task(self, button, msg, sender):
        if button == msg.button(QMessageBox.Cancel):
            return
        id = self.widget_task_ids[sender]
        task = self.db_task.sql_query("SELECT * FROM tasks WHERE id = {}".format(id))
        task = task[0]

        new_end_date = datetime.now() - timedelta(days=1)
        new_days = (new_end_date - datetime.strptime(task[4],  "%Y-%m-%d")).days
        # convert to date
        new_end_date = new_end_date.date()
        self.db_task.update_row(id, task[1], new_days, task[3], task[4], new_end_date, task[6])

        # delte task from task todo bar
        self.update_tasks_todo(id, 1)
        # delete task from task overview
        sender.deleteLater()
        # insert task with updated parameters
        self.update_task_overview(id, task[1], new_days, task[3], new_end_date, task[6])

    def change_page(self, page):
        self.ui.Pages_Widget.setCurrentWidget(page)

    def toggle_menu(self, max_width):
        current_width = self.ui.frame_left_menu.width()
        end_width = self.init_menu_width
        if current_width == self.init_menu_width:
            end_width = max_width
        else:
            self.ui.Btn_Menu_1.setText("")
            self.ui.Btn_Menu_2.setText("")
            self.ui.Btn_Menu_3.setText("")
        self.animation = QPropertyAnimation(self.ui.frame_left_menu, b"minimumWidth")
        self.animation.setDuration(100)
        self.animation.setStartValue(current_width)
        self.animation.setEndValue(end_width)
        #self.animation.setEasingCurve(QtCore.QEasingCurve.InOutQuart)
        self.animation.start()
        if end_width == max_width:
            self.animation.finished.connect(self.show_button_text)

    def show_button_text(self):
        self.ui.Btn_Menu_1.setText(" TASKS")
        self.ui.Btn_Menu_2.setText(" EDIT")
        self.ui.Btn_Menu_3.setText(" HISTORY")

    def select_tasks_todo(self):
        current_tasks = self.db_task.sql_query("SELECT * FROM tasks WHERE endDate > strftime('%Y-%m-%d', 'now')")
        if not current_tasks:
            return []
        # sort by start date to get oldest start date of a running task
        current_tasks.sort(key=lambda x: x[4])
        oldest_start_date = current_tasks[0][4]

        # differences[i] states how many days task i is younger than the oldest relevant task
        differences = {task[0]: (datetime.strptime(task[4],  "%Y-%m-%d") - datetime.strptime(oldest_start_date,  "%Y-%m-%d")).days for task in current_tasks}
        # id of check bar for today (with respect to the oldest current task)
        today_day_id = (datetime.today() - datetime.strptime(oldest_start_date,  "%Y-%m-%d")).days
        current_task_ids = set([x[0] for x in current_tasks])

        query = format_list_sql_query("SELECT * FROM data_joint WHERE check_number != 0 and task_id", current_task_ids)
        result = self.db_joint.sql_query(query)

        # day_id is different for newer tasks -> subtract number in differences at the respective id
        # task_id starts at 1
        done_task_ids = set([task[0] for task in result if task[1] == today_day_id-differences[task[0]]])
        tasks_todo_ids = current_task_ids - done_task_ids

        return tasks_todo_ids

    def init_tasks_todo(self):
        tasks_todo_ids = self.select_tasks_todo()

        layout = self.ui.tasks_todo_widget.layout()
        layout.setSpacing(8)

        query = format_list_sql_query("SELECT * FROM tasks WHERE id", tasks_todo_ids)
        rows = self.db_task.sql_query(query)
        names = [x[1] for x in rows]

        for i, name in enumerate(names):
            widget = create_tasks_todo_widget(name, rows[i][6])
            layout.addWidget(widget)

    # check 0 means to add
    # check 1 means to delete
    def update_tasks_todo(self, id, checked):
        res = self.db_task.sql_query("SELECT taskName, color FROM tasks WHERE id = {}".format(id))
        name = res[0][0]

        if checked != 0:
            label = self.ui.tasks_todo_widget.findChild(QLabel, name)
            if label:
                label.deleteLater()
        else:
            layout = self.ui.tasks_todo_widget.layout()
            color = res[0][1]
            widget = create_tasks_todo_widget(name, color)
            layout.addWidget(widget)

    def insert_new_task(self):
        if not self.ui.task_field.text() or not self.ui.day_field.text():
            return

        max_input = 24
        # insert new task into database
        name_input = self.ui.task_field.text()[:max_input]
        days_input = min(31, int(self.ui.day_field.text()))
        duration_input = self.ui.time_field.time().toString("HH:mm")

        # clear inputs
        self.ui.task_field.clear()
        self.ui.day_field.clear()
        self.ui.time_field.clear()

        start_date = date.today()
        rnd_color = get_random_color()
        task_id = self.db_task.insert_task(name_input, days_input, duration_input, start_date, rnd_color)

        # insert new task into ui
        self.update_task_overview(task_id, name_input, days_input, duration_input, start_date, rnd_color)

        self.update_tasks_todo(task_id, 0)
        self.change_page(self.ui.page_show_tasks)

    def update_task_overview(self, task_id, name, days, duration, start, color):
        # get current scrollArea
        scrollArea = self.task_overviews_per_month[self.month-1]

        # add new task to task overview
        task_widget = create_task_widget(self, task_id, name, days, duration, start, color, self.year, self.month)
        self.widget_task_ids[task_widget] = task_id
        scrollArea.widget().layout().addWidget(task_widget)

    def init_tasks(self):
        self.setStyleSheet(scrollArea_style)
        scrollArea = self.task_overviews_per_month[self.month-1]

        # show month name
        month_name = calendar.month_abbr[self.month]
        self.ui.month_label.setText("{} {}".format(month_name, str(self.year)[-2:]))

        if scrollArea:
            scrollArea.setVisible(True)
        else:
            scrollArea = QScrollArea()
            self.ui.task_overview.layout().addWidget(scrollArea)
            new_widget = QWidget()
            scrollArea.setWidget(new_widget)
            scrollArea.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
            scrollArea.setWidgetResizable(True)

            layout = FlowLayout()
            new_widget.setLayout(layout)
            layout.setSpacing(10)

            self.task_overviews_per_month[self.month-1] = scrollArea

            sql_end_date = date(self.year, self.month, 1)
            sql_start_date = add_delta_month_datetime(self.year, self.month, 1, 1).date()
            tasks = self.db_task.sql_query("SELECT * FROM tasks WHERE startDate < strftime('%Y-%m-%d', '{}') and endDate >= strftime('%Y-%m-%d', '{}')".format(sql_start_date, sql_end_date))
            if not tasks:
                return
            tasks.sort(key=lambda x: x[2], reverse=True)

            for task in tasks:
                task_end_date = datetime.strptime(task[4],  "%Y-%m-%d")
                task_widget = create_task_widget(self, task[0], task[1], task[2], task[3], task_end_date.date(), task[6], self.year, self.month)
                self.widget_task_ids[task_widget] = task[0]
                layout.addWidget(task_widget)

    def compute_tasks_count_quantiles(self):
        res = self.db_joint.sql_query("SELECT date, count(task_id) FROM data_joint WHERE check_number != 0 GROUP BY date")
        if not res:
            return [], []
        tasks_per_day = {x[0]: x[1] for x in res}
        quantiles = np.quantile(list(tasks_per_day.values()), [0.25, 0.5, 0.75, 1.0])

        return quantiles, tasks_per_day

    def compare_accomplished_tasks_statistics(self):
        this_month_end = datetime.now()
        last_month_begin = add_delta_month_datetime(this_month_end.year, this_month_end.month, 1, -1)

        accomplished = self.db_joint.sql_query("""SELECT strftime(\"%m-%Y\", date), ifnull(count(*),0)
                                FROM data_joint
                                WHERE check_number > 0 AND date >= strftime('%Y-%m-%d', '{}') AND date <= strftime('%Y-%m-%d', '{}')
                                GROUP BY strftime(\"%m-%Y\", date)""".format(last_month_begin, this_month_end))

        total = self.db_joint.sql_query("""SELECT strftime(\"%m-%Y\", date), ifnull(count(*),0)
                                FROM data_joint
                                WHERE check_number != -1 AND day_id >= 0 AND date >= strftime('%Y-%m-%d', '{}') AND date <= strftime('%Y-%m-%d', '{}')
                                GROUP BY strftime(\"%m-%Y\", date)""".format(last_month_begin, this_month_end))

        accomplished = dict(accomplished)
        total = dict(total)

        last_month_str = datetime.strftime(last_month_begin, "%m-%Y")
        this_month_str = datetime.strftime(this_month_end, "%m-%Y")

        percentage_last_month = accomplished.get(last_month_str, 0) / total.get(last_month_str, 1)
        percentage_this_month = accomplished.get(this_month_str, 0) / total.get(this_month_str, 1)
        highlights_text = []

        if percentage_last_month:
            accomplished_dif = percentage_this_month - percentage_last_month
            month_dif = accomplished_dif / percentage_last_month
            month_dif_text = interpret_accomplished_tasks_month_difference(month_dif)
            highlights_text.append(month_dif_text)

        accomplished_tasks_text = interpret_accomplished_tasks_percentage(accomplished.get(this_month_str, 0), total.get(this_month_str, 0))
        highlights_text.append(accomplished_tasks_text)

        self.ui.textBrowser.setText(" ".join(highlights_text))

        y = [percentage_last_month * 100, percentage_this_month * 100]
        x = [calendar.month_abbr[last_month_begin.month], calendar.month_abbr[this_month_end.month]]
        sc = MplCanvas(width=4, height=3, dpi=100)
        sc.axes.barh(x, y, height=0.4, color="#dde6f6")
        sc.axes.xaxis.set_visible(False)

        # new_patches = []
        # for patch in reversed(sc.axes.patches):
        #     bb = patch.get_bbox()
        #     color=patch.get_facecolor()
        #     p_bbox = FancyBboxPatch((bb.xmin, bb.ymin), abs(bb.width), abs(bb.height),
        #                             boxstyle="round,pad=0.0040,rounding_size=0.10",
        #                             ec="none", fc=color, mutation_aspect=5)
        #     patch.remove()
        #     if bb.height == 0:
        #         new_patches.append(patch)
        #     else:
        #         new_patches.append(p_bbox)
        # for patch in new_patches:
        #     sc.axes.add_patch(patch)
        # sc.fig.tight_layout()

        sc.setFixedHeight(100)
        self.ui.widget_8.layout().insertWidget(0, sc)

    def init_month_statistics(self):
        month_name = calendar.month_abbr[self.month]
        self.ui.month_label_2.setText("{} {}".format(month_name, str(self.year)[-2:]))

        end_date = add_delta_month_datetime(self.year, self.month, 1, 1)
        start_date = add_delta_month_datetime(self.year, self.month, 1, -5)

        res = self.db_joint.sql_query("""SELECT strftime(\"%m-%Y\", date), ifnull(count(*),0)
                                        FROM data_joint
                                        WHERE check_number > 0 AND date >= strftime('%Y-%m-%d', '{}') AND date < strftime('%Y-%m-%d', '{}')
                                        GROUP BY strftime(\"%m-%Y\", date)""".format(start_date, end_date))

        res = dict(res)
        x =  []
        y = []

        for i in reversed(range(6)):
            current_date = add_delta_month_datetime(self.year, self.month, 1, -i).date()
            x.append(calendar.month_abbr[current_date.month])
            current_date = current_date.strftime('%m-%Y')

            if current_date in res:
                y.append(res[current_date])
            else:
                y.append(0)

        sc = MplCanvas(width=5, height=4, dpi=100)
        sc.axes.bar(x, y, width=0.8, color="#dde6f6")
        max_y_lim = 10 + max(y)//10 * 10
        sc.axes.set_ylim(0, max_y_lim)
        new_patches = []
        for patch in reversed(sc.axes.patches):
            bb = patch.get_bbox()
            color=patch.get_facecolor()
            p_bbox = FancyBboxPatch((bb.xmin, bb.ymin), abs(bb.width), abs(bb.height),
                                    boxstyle="round,pad=0.0040,rounding_size={}".format(0.002*max_y_lim),
                                    ec="none", fc=color, mutation_aspect=10)
            patch.remove()
            if bb.height == 0:
                new_patches.append(patch)
            else:
                new_patches.append(p_bbox)
        for patch in new_patches:
            sc.axes.add_patch(patch)
        sc.fig.tight_layout()
        sc.setFixedHeight(300)
        self.ui.monthsplot.layout().itemAt(0).widget().deleteLater()
        self.ui.monthsplot.layout().insertWidget(0, sc)

    def show_statistics(self):
        if self.ui.calendar.count() < 1:
            # create new calendar
            self.create_calendar_statistics()
        else:
            # update calendar if there already exists one
            self.update_calendar_statistics()

        # show statistics page
        self.change_page(self.ui.page_statistics)

    def change_tasks_month(self, i):
        widget = self.task_overviews_per_month[self.month-1]
        widget.setVisible(False)

        self.year, self.month = add_delta_month(self.year, self.month, i)

        self.init_tasks()
        self.init_month_statistics()

    def change_calendar_year(self, i):
        # change year if updated year is not in the future
        current_year = datetime.today().year
        year = self.year + i
        self.year = year if year <= current_year else current_year
        self.ui.year_label.setText("{}".format(self.year))
        # update shown calendar using the updated year
        self.update_calendar_statistics()

    def update_calendar_statistics(self):
        # delete old calendar
        for i in range(self.ui.calendar.count()):
            self.ui.calendar.itemAt(i).widget().deleteLater()
        # create new one
        self.create_calendar_statistics()

    def create_calendar_statistics(self):
        quantiles, tasks_per_day = self.compute_tasks_count_quantiles()

        columns = 7
        calendar_grid = self.ui.calendar
        calendar_grid.setSpacing(2)
        d = 0
        for month in range(1, 13):
            month_calendar = calendar.monthcalendar(self.year, month)
            for week in month_calendar:
                for day in week:
                    count = 0
                    # skip first 0 days at the beginning of the year
                    # count rows in order for monday to begin in row 0
                    if month == 1 and day == 0:
                        d += 1
                        continue
                    # do not count rows for all other 0 days
                    elif day == 0:
                        continue
                    current_date = date(self.year, month, day)
                    color = 0
                    try:
                        count = tasks_per_day[str(current_date)]
                        color = 1
                        for i in range(len(quantiles)-1, 0, -1):
                            if count >= quantiles[i]:
                                color = i+1
                                break
                    except:
                        color = 0

                    current_date = date(self.year, month, day)
                    day_widget = create_calendar_day_widget(self, current_date, count, color)
                    calendar_grid.addWidget(day_widget, d%columns, d//columns)

                    d += 1
                    # for january skip 0 days at the end of month
                    if day == 31:
                        break

app = QApplication(sys.argv)
window = MainWindow()

try:
    sys.exit(app.exec_())
except:
    print("Exit")


import os
import pickle

import nltk
from nltk import word_tokenize
from nltk.util import ngrams

def find_accuracy(filepath, filepath2):
    file = open(os.path.join(os.getcwd(), filepath), 'r')
    file2 = open(os.path.join(os.getcwd(), filepath2), 'r')

    total = 0
    correct = 0
    incorrect = list()

    while True:
        line1 = file.readline()
        line2 = file2.readline()
        total += 1

        if not line1:
            break

        if line1 == line2:
            correct += 1
        else:
            incorrect.append(line1.split()[0])

    file.close()
    file2.close()

    acc = correct / total

    print("accuracy %.2f" % acc)
    print("incorrect line numbers", incorrect)

def find_most_likely(filepath, u_eng, b_eng, u_fre, b_fre, u_ita, b_ita):
     with open(os.path.join(os.getcwd(), filepath), 'r') as file:

        count = 1

        with open('results.txt', 'w') as f:
            for sentence in file:
                most_likely = get_lang(sentence, u_eng, b_eng, u_fre, b_fre, u_ita, b_ita)
                f.write(str(count) + " " + most_likely + "\n")
                count += 1

def get_lang(sentence, u_eng, b_eng, u_fre, b_fre, u_ita, b_ita):
     eng = compute_prob(sentence, u_eng, b_eng, len(u_eng), len(b_eng))
     fre = compute_prob(sentence, u_fre, b_fre, len(u_fre), len(b_fre))
     ita = compute_prob(sentence, u_ita, b_ita, len(u_ita), len(b_ita))

     biggest = max(eng, fre, ita)

     if eng == biggest:
        return "English"
     elif fre == biggest:
        return "French"
     else:
         return "Italian"

def compute_prob(text, unigram_dict, bigram_dict, N, V):
    # N is the number of tokens in the training data
    # V is the vocabulary size in the training data (unique tokens)

    unigrams_test = word_tokenize(text)
    bigrams_test = list(ngrams(unigrams_test, 2))

    p_laplace = 1  # calculate p using Laplace smoothing

    for bigram in bigrams_test:
        n = bigram_dict[bigram] if bigram in bigram_dict else 0
        d = unigram_dict[bigram[0]] if bigram[0] in unigram_dict else 0

        p_laplace = p_laplace * ((n + 1) / (d + V))

    return p_laplace

if __name__ == '__main__':

        # read the pickle files
        unidict_eng = pickle.load(open('unidict_eng.p', 'rb'))  # read binary
        bidict_eng = pickle.load(open('bidict_eng.p', 'rb'))  # read binary

        # read the pickle files
        unidict_fre = pickle.load(open('unidict_fre.p', 'rb'))  # read binary
        bidict_fre = pickle.load(open('bidict_fre.p', 'rb'))  # read binary

        # read the pickle files
        unidict_ita = pickle.load(open('unidict_ita.p', 'rb'))  # read binary
        bidict_ita = pickle.load(open('bidict_ita.p', 'rb'))  # read binary

        find_most_likely("data/LangId.test", unidict_eng, bidict_eng, unidict_fre, bidict_fre, unidict_ita, bidict_ita)
        find_accuracy("data/LangId.sol", "results.txt")


import cv2
import numpy as np
import imutils
from collections import deque

img = cv2.imread("../images/VY_Canis_Majoris.png")
out = img.copy()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.5, 500)

if circles is not None:
    # convert the (x, y) coordinates and radius of the circles to integers
    circles = np.round(circles[0, :]).astype("int")
    # loop over the (x, y) coordinates and radius of the circles
    for (x, y, r) in circles:
        # draw the circle in the output image, then draw a rectangle
        # corresponding to the center of the circle
        cv2.circle(out, (x, y), r, (0, 255, 0), 4)
        cv2.rectangle(out, (x - 5, y - 5), (x + 5, y + 5), (0, 128, 255), -1)
    # show the output image
    cv2.imshow("output", np.hstack([img, out]))
    cv2.waitKey(0)

src/object detection/steph 3 pt contest tracking/tracking tests/hough_circles_test.py


# circle in NURBS
sr2i = 1.0 / (2**0.5)
xs = [1.0,  sr2i,  0.0, -sr2i, -1.0, -sr2i,  0.0,  sr2i,  1.0]
ys = [0.0,  sr2i,  1.0,  sr2i,  0.0, -sr2i, -1.0, -sr2i,  0.0]
ws = [1.0,  sr2i,  1.0,  sr2i,  1.0,  sr2i,  1.0,  sr2i,  1.0]
n = len(xs)
knot = [0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4];
degree = 2;

def index_in_knots(t, nurbs_knot):
	for i in range(len(nurbs_knot) - 1):
		if t >= nurbs_knot[i] and t < nurbs_knot[i+1]:
			return i
	return -1

def nurbs_in_t(t, control_xs, control_ys, control_ws, knot, p):
	# de Boor
	k = index_in_knots(t, knot)

	d_x = []
	d_y = []
	d_w = []
	for j in range(p + 1):
		d_x += [control_xs[j + k - p]]
		d_y += [control_ys[j + k - p]]
		d_w += [control_ws[j + k - p]]

	for r in range(1, p+1):
		for j in reversed(range(r, p+1)):
			a = (t - knot[j+k-p]) / (knot[j+1+k-r] - knot[j+k-p])
			d_x[j] = (1.0 - a) * d_x[j-1] + a * d_x[j]
			d_y[j] = (1.0 - a) * d_y[j-1] + a * d_y[j]
			d_w[j] = (1.0 - a) * d_w[j-1] + a * d_w[j]

	return [d_x[p] / d_w[p], d_y[p] / d_w[p]]

def circle(t):
	global xs, ys, ws, knot, degree
	return nurbs_in_t(t, xs, ys, ws, knot, degree)

# make the plot
import numpy as np
import matplotlib.pyplot as plt

import matplotlib
matplotlib.interactive(True)
matplotlib.use('WebAgg')

# polynomial curve
ts = np.arange(0.00, 4.00, 0.01)
Pxs = []
Pys = []
for t in ts:
	xy = circle(t)
	Pxs += [xy[0]]
	Pys += [xy[1]]

fig, ax = plt.subplots()
ax.plot(Pxs, Pys)
ax.set(xlabel='x', ylabel='y', title='de Boor\'s circle')
ax.grid()
ax.set_aspect('equal', adjustable='box')
plt.scatter([x/w for x,w in zip(xs,ws)], [y/w for y,w in zip(ys,ws)])
fig.savefig("nurbs_circle.png")
plt.show()


from collections import defaultdict

with open("data/6.txt") as f:
    timers = [int(n) for n in f.read().rstrip().split(",")]

def one():
    # 80 days later
    for _ in range(80):
        for i in range(len(timers)):
            timers[i] -= 1
            if timers[i] == -1:
                # Produce new lanternfish with timer 8 for first cycle
                timers[i] = 6
                timers.append(8)
    return len(timers)

def two():
    fish = defaultdict(int)
    for timer in timers:
        fish[timer] += 1

    for _ in range(256):
        new_fish = fish[0]
        for i in range(1, 9):
            fish[i - 1] = fish[i]
        fish[8] = new_fish
        fish[6] += new_fish  # Parents requeue
    return sum(fish.values())

if __name__ == "__main__":
    print(two())


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.13.1
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
sns.set()

def read_heartbeats_log(filename: str):
    df = pd.read_csv(filename, delimiter=" ",
                     skipinitialspace=True, index_col="Beat")
    df = df.groupby("Tag").mean()
    return df

# # Benchmark AVX2 (without POET )

# +
import numpy as np
from sklearn.linear_model import LinearRegression
from utils import system_cpu_freqs

def rate_power(folder,freq):
    df_hb = read_heartbeats_log(f"../experiments/{folder}/heartbeat_{freq}.log")
    power = df_hb["Global_Power"].mean()
    rate =  df_hb["Global_Rate"].mean()
    return rate,power

freqs = np.array(list(system_cpu_freqs()))

without_avx2_32 = np.array([rate_power("without_avx2_32",f) for f in freqs])
without_avx2_64 = np.array([rate_power("without_avx2_64",f) for f in freqs])

with_avx2_32 = np.array([rate_power("with_avx2_32",f) for f in freqs])/without_avx2_32
with_avx2_64 = np.array([rate_power("with_avx2_64",f) for f in freqs])/without_avx2_64

reg32 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_32[:,0])
reg64 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_64[:,0])

plt.figure(figsize=(9,5))
plt.plot(freqs,with_avx2_32[:,0],"--o",label="AVX2 32-bit",color="C0")
plt.plot(freqs,with_avx2_64[:,0],"--o",label="AVX2 64-bit",color="C1")

plt.plot(freqs,np.array(freqs)*reg32.coef_+reg32.intercept_,"-",color="C0")
plt.text(1.8e6,1.11,"avg. "+str(with_avx2_32[:,0].mean().round(3)),ha="center",color="C0")

plt.plot(freqs,np.array(freqs)*reg64.coef_+reg64.intercept_,"-",color="C1")
plt.text(2.3e6,1.055,"avg. "+str(with_avx2_64[:,0].mean().round(3)),ha="center",color="C1")
plt.xticks(freqs)
plt.ylabel("Speedup")
plt.xlabel("CPU Frequency (GHz)")
plt.legend()
plt.savefig("../figures/avx2_speedup.pdf",dpi=300,bbox_inches='tight')

# +
reg32 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_32[:,1])
reg64 = LinearRegression().fit(np.array(freqs).reshape(-1,1), with_avx2_64[:,1])

plt.figure(figsize=(9,5))
plt.plot(freqs,with_avx2_32[:,1],"o",label="AVX2 32-bit",color="C0")
plt.plot(freqs,with_avx2_64[:,1],"o",label="AVX2 64-bit",color="C1")

plt.plot(freqs,np.array(freqs)*reg32.coef_+reg32.intercept_,"-",color="C0")
plt.text(2.3e6,1.055,"avg. "+str(with_avx2_32[:,1].mean().round(3)),ha="center",color="C0")

plt.plot(freqs,np.array(freqs)*reg64.coef_+reg64.intercept_,"-",color="C1")
plt.text(1.8e6,1.0,"avg. "+str(with_avx2_64[:,1].mean().round(3)),ha="center",color="C1")
plt.xticks(freqs)
plt.ylabel("Powerup")
plt.xlabel("CPU Frequency (GHz)")
plt.legend()
plt.savefig("../figures/avx2_powerup.pdf",dpi=300,bbox_inches='tight')
# -

# # Benchmark with POET

# +
df_poet = pd.read_csv("../experiments/poet_no_avx2_32/poet.log", delimiter=" ",
                      skipinitialspace=True).groupby("HB_NUM").first()
df_heartbeats = read_heartbeats_log("../experiments/poet_no_avx2_32/heartbeat.log")

df_poet_avx2 = pd.read_csv("../experiments/poet_avx2_32/poet.log", delimiter=" ",
                      skipinitialspace=True).groupby("HB_NUM").first()
df_heartbeats_avx2 = read_heartbeats_log("../experiments/poet_avx2_32/heartbeat.log")

# +
df_avx2 = df_poet_avx2.join(df_heartbeats_avx2)
df_avx2["Timestamp"] = pd.to_datetime(df_avx2["Timestamp"] - df_avx2["Timestamp"].iloc[0]).iloc[5:]

df = df_poet.join(df_heartbeats)
df["Timestamp"] = pd.to_datetime(df["Timestamp"] - df["Timestamp"].iloc[0]).iloc[5:]

# +
fig, ax1 = plt.subplots(figsize=(20, 5))

ax1.plot([df["Timestamp"].min(),df["Timestamp"].max()],[180]*2,label="Target Rate",color="green")

ax1.plot(df["Timestamp"], df["Global_Rate"], label="Rate", color="C0")
ax1.plot(df_avx2["Timestamp"], df_avx2["Global_Rate"], label="Rate (AVX2)", color="C2")

ax1.tick_params(axis='y', labelcolor="C0")
ax1.set_ylabel('Rate', color="C0")
ax1.set_xlabel('Time', color="black")

ax2 = ax1.twinx()
ax2.plot(df["Timestamp"], df["Global_Power"], label="Power", color="C1")
ax2.plot(df_avx2["Timestamp"], df_avx2["Global_Power"], label="Power (AVX2)", color="C3")

ax2.tick_params(axis='y', labelcolor="C1")
ax2.set_ylabel('Power (Watt)', color="C1")
ax2.grid(False)

step=20
num_threads = [1,2,3,4,5,6,5,4,3,2,1]
areas = [[t,[i*step,(i+1)*step]] for i,t in enumerate(num_threads)]

for cores, a in areas:
    x_min = pd.Timestamp(0)+pd.Timedelta(a[0], unit="sec")
    x_max = pd.Timestamp(0)+pd.Timedelta(a[1], unit="sec")
    ax1.axvspan(x_min, x_max,
                color=f"C{int(cores)+2}", alpha=0.3)
    y = sum(ax1.get_ylim())/2
    ax1.text(pd.Timestamp(0)+pd.Timedelta((a[0]+a[1])/2, unit="sec"),
             y, f"{cores} thread(s)",
             color=f"C{int(cores)+2}", horizontalalignment="center")

fig.legend(ncol=5,loc="upper center")
fig.savefig("../figures/poet_benchmark.pdf",dpi=300,bbox_inches='tight')
# -

# # Datasets Benchmark

# +
from os import path
import glob

ds_n = []

for fn in glob.glob("../experiments/datasets_32/heartbeat_*.log"):
    parts = path.basename(fn).split(".")[0].split("_")[2:5]
    dc = dict(zip(["Size","Distribution","Ratio"],parts))
    df_h = read_heartbeats_log(fn)
    dc["Operations / Second"] = df_h["Instant_Rate"].mean() * 10000
    watt = df_h["Instant_Power"].mean()
    delta =pd.to_datetime(df_h.iloc[-1]["Timestamp"])-pd.to_datetime(df_h.iloc[0]["Timestamp"])

    joules = watt*delta.total_seconds()
    dc["Operations/Joule"] = int(dc["Operations / Second"]/joules)
    dc["Operations / Second"] = int(dc['Operations / Second'])
    ds_n.append(dc)

df_ds = pd.DataFrame(ds_n)
df_ds
# -

df_ds_n = df_ds.sort_values(["Size","Distribution","Ratio"],ascending=False).reset_index()
df_ds_n.index = list(range(1,len(df_ds)+1))
del df_ds_n["index"]
df_ds_n

print(df_ds_n.to_latex())


import torch.nn as nn

class EncoderDecoder(nn.Module):
    def __init__(self):
        super(EncoderDecoder, self).__init__()

    """
        Retrieves intermediate representation's number of channels
    """

    @property
    def _encoder_channels(self):
        currentLevel, chnl_sizes, tmpParam = "_level1", [], None
        # Get the paramters from the model and their names
        for name, param in self._encoder.named_parameters():
            tmpLevel = name.split(".")[0]
            # If it is the last layer of a level, add its name and size
            if not tmpLevel == currentLevel:
                chnl_sizes.append((currentLevel, tmpParam.size()[0]))
                currentLevel = tmpLevel
            tmpParam = param

        chnl_sizes.append((tmpLevel, tmpParam.size()[0]))
        return chnl_sizes


## Example code on how to properly use the xlwt lbrary to organize data dumps
import xlwt
from xlrd import open_workbook

target_column = 0     # 1st column is 0 indexed

book = open_workbook('test.xlsx')
sheet = book.sheets()[0]
data = [sheet.row_values(i) for i in xrange(sheet.nrows)]
labels = data[0]    # Don't sort our headers
data = data[1:]     # Data begins on the second row
data.sort(key=lambda x: x[target_column])

bk = xlwt.Workbook()
sheet = bk.add_sheet(sheet.name)

for idx, label in enumerate(labels):
     sheet.write(0, idx, label)

for idx_r, row in enumerate(data):
    for idx_c, value in enumerate(row):
        sheet.write(idx_r+1, idx_c, value)

bk.save('result.xls')

## problem: this code only ouputs .xls files. need to convert that
##          to .xlsx some how


from datetime import datetime, timedelta
import time

class CronTab(object):
  """Simulates basic cron functionality by checking for firing jobs every
  minute."""

  def __init__(self, xbmc):
    self.xbmc = xbmc
    self.jobs = []
    self.__enabled = True

  def switch(self):
      self.__enabled = True

  def stop(self):
    """Stops the crontab."""
    self.__enabled = False
    self.xbmc.log("PVRReminder: Stopped Cron", self.xbmc.LOGDEBUG)

  def start(self):
    """Starts to check every minute, if the registered jobs should run."""
    t = datetime(*datetime.now().timetuple()[:5])
    self.xbmc.log("PVRReminder: Cron is enabled:" + str(self.__enabled), self.xbmc.LOGDEBUG)
    while self.__enabled:
      if self.xbmc and not self.xbmc.abortRequested:
        self.xbmc.log("PVRReminder: Cron jobs:" + str(self.jobs), self.xbmc.LOGDEBUG)
        for job in self.jobs:
          self.xbmc.log("PVRReminder: checking job %s against %s" % (str(job), str(t)),
                          self.xbmc.LOGDEBUG)
          job.check(t)

        t += timedelta(minutes=1)
        if datetime.now() < t:
          if self.xbmc:
            self.xbmc.sleep((t - datetime.now()).seconds * 1000)
          else:
            time.sleep((t - datetime.now()).seconds)

class AllMatch(set):
  """Universal set - match everything"""
  def __contains__(self, item): return True

class Job(object):
  """Cron job abstraction."""

  def conv_to_set(self, obj):
    """Convert obj to a set containing obj if necessary."""
    if isinstance(obj, (int,long)):
      return set([obj])
    if not isinstance(obj, set):
      obj = set(obj)
    return obj
# day=AllMatch(), month=AllMatch(), dow=AllMatch(),
  def __init__(self, action, min=AllMatch(), hour=AllMatch(),
                     day=AllMatch(), month=AllMatch(),
                     args=(), kwargs={}):
    self.mins = self.conv_to_set(min)
    self.hours = self.conv_to_set(hour)
    self.days = self.conv_to_set(day)
    self.months = self.conv_to_set(month)
    #self.dow = self.conv_to_set(dow)
    self.action = action
    self.args = args
    self.kwargs = kwargs

  def __str__(self):
    return str(self.mins) + ", " + str(self.hours) + ", "\
                 + str(self.days) + ", " + str(self.months) + ", "\
                 + str(self.action) + ", "\
                 + str(self.args) + ", " + str(self.kwargs)
# + str(self.dow) + ", " + str(self.action) + ", "\

  def is_matchtime(self, t):
    """Is t the job's scheduled time"""
    return ((t.minute in self.mins) and
            (t.hour in self.hours) and
            (t.day in self.days) and
            (t.month in self.months)) #and
          #  (t.weekday() in self.dow))

  def check(self, t):
    """Checks if t is the scheduled time and executes the job if so."""
    if self.is_matchtime(t):
      self.action(*self.args, **self.kwargs)


import findspark
import pyspark
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import *
from pyspark import SparkConf, SparkContext
import os
import sys
import py4j
import json

findspark.init()
os.environ['JAVA_HOME'] = '/usr/local/java/jdk-11.0.18+10'
os.environ['SPARK_HOME'] = '/usr/local/spark'
sys.path.append('/usr/local/spark/python')
sys.path.append('/usr/local/spark/python/lib/pyspark.zip')
sys.path.append('/usr/local/spark/python/lib/py4j-0.10.9.5-src.zip')
sys.path.append('/usr/local/spark/bin')
sys.path.append('/usr/local/spark/yarn/spark-3.3.2-yarn-shuffle.jar')
sys.path.append('/usr/local/spark/jars')

# Create a Spark session
spark = SparkSession.builder \
    .config("spark.driver.host", "localhost") \
    .config("spark.driver.logLevel", "DEBUG") \
    .appName('Realtime_Streamprocessing_app') \
    .getOrCreate()

# Define the schema for the data
schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("timestamp", LongType(), True),
    StructField("distance_covered", DoubleType(), True),
    StructField("engine_speed", DoubleType(), True),
    StructField("fuel_consumed", DoubleType(), True),
])

# Consume data from the Kafka topic
df = spark \
    .readStream \
    .format('kafka') \
    .option('kafka.bootstrap.servers', 'localhost:9092') \
    .option('subscribe', 'Truck-Data') \
    .option('startingOffsets', 'earliest') \
    .option('failOnDataLoss', 'false') \
    .option('includeTimestamp', 'true') \
    .option('schema', schema) \
    .load()

# Define the schema for the data
schema = StructType([
    StructField('id', IntegerType(), True),
    StructField('timestamp', StringType(), True),
    StructField('distance_covered', DoubleType(), True),
    StructField('engine_speed', DoubleType(), True),
    StructField('fuel_consumed', DoubleType(), True),
])

# Convert the value column from binary to string, and parse the fields
parsed_df = df.selectExpr('CAST(value AS STRING)').select(from_json('value', schema).alias('data'), 'timestamp')

# Flatten the struct column and rename the fields
transformed_df = parsed_df.selectExpr(
    'data.id AS id',
    'timestamp',
    'data.distance_covered AS distance_covered',
    'data.engine_speed AS engine_speed',
    'data.fuel_consumed AS fuel_consumed'
)

# Now, we write our transformed data to a MySQL table
transformed_df.writeStream \
    .format("jdbc") \
    .option("url", "jdbc:mysql://MySQL_Container:3306/KAFKA_DB") \
    .option("dbtable", "SPARK_TABLE_TRANSFORMED") \
    .option("user", "root") \
    .option("password", "root") \
    .option("checkpointLocation", "/tmp/checkpoints") \
    .option("driver", "com.mysql.cj.jdbc.Driver") \
    .start()

# Submit my file to spark-submit

# spark.sparkContext.addPyFile('./spark_streamer.py')
# spark.sparkContext.submit(MainClass='spark_streamer.py')


"""empty message

Revision ID: 251afb20e176
Revises:
Create Date: 2023-10-09 00:19:06.848257

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '251afb20e176'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('users',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('username', sa.String(), nullable=False),
    sa.Column('email', sa.String(), nullable=False),
    sa.Column('password', sa.String(), nullable=False),
    sa.Column('contacts', sa.String(), nullable=True),
    sa.Column('created_at', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
    sa.Column('updated_at', sa.DateTime(), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('email'),
    sa.UniqueConstraint('username')
    )
    op.create_table('businesses',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(), nullable=True),
    sa.Column('category', sa.String(), nullable=True),
    sa.Column('sub_category', sa.String(), nullable=True),
    sa.Column('owner_id', sa.Integer(), nullable=False),
    sa.Column('hours_open', sa.String(), nullable=True),
    sa.Column('contacts', sa.String(), nullable=True),
    sa.Column('poster', sa.String(), nullable=True),
    sa.Column('location', sa.String(), nullable=True),
    sa.Column('created_at', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
    sa.Column('updated_at', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['owner_id'], ['users.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('businessimgs',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('businessimgurl', sa.String(), nullable=False),
    sa.Column('business_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['business_id'], ['businesses.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('products',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=200), nullable=True),
    sa.Column('description', sa.String(), nullable=True),
    sa.Column('price', sa.Float(), nullable=True),
    sa.Column('poster', sa.String(), nullable=True),
    sa.Column('business_id', sa.Integer(), nullable=True),
    sa.Column('created_at', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
    sa.Column('updated_at', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['business_id'], ['businesses.id'], ),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('name')
    )
    op.create_table('reviews',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('user_id', sa.Integer(), nullable=False),
    sa.Column('business_id', sa.Integer(), nullable=False),
    sa.Column('comment', sa.String(), nullable=False),
    sa.Column('rating', sa.Float(), nullable=True),
    sa.Column('created_at', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
    sa.Column('updated_at', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['business_id'], ['businesses.id'], ),
    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('productimgs',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('product_id', sa.Integer(), nullable=False),
    sa.Column('productimgurl', sa.String(), nullable=True),
    sa.ForeignKeyConstraint(['product_id'], ['products.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('productimgs')
    op.drop_table('reviews')
    op.drop_table('products')
    op.drop_table('businessimgs')
    op.drop_table('businesses')
    op.drop_table('users')
    # ### end Alembic commands ###


from selenium import webdriver

def get_links(company_name):
	url = "https://www.google.com/"
	browser = webdriver.PhantomJS()
	browser.get(url)
	search_box = browser.find_element_by_name("q")
	search_box.send_keys(company_name)
	search_box.submit()
	try:
		links = browser.find_elements_by_xpath("//ol[class='web_regular_results']//h3//a")
	except:
		links = browser.find_elements_by_xpath("//h3//a")
	results = []
	for link in links:
		href = link.get_attribute("href")
		print(href)
		results.append(href)
	browser.close()
	return results

get_links("dog")


import itertools

VALID_PROTOCOLS = ['tcp', 'udp', 'sctp']

class InvalidProtocol(Exception):
    def __init__(self, invalid_protocols):
        self.args = invalid_protocols

    def __str__(self):
        return f"Invalid Protocol: {self.args}"

class OvnLoadBalancer:
    def __init__(self, lb_name, nbctl, vips=None, protocols=VALID_PROTOCOLS):
        '''
        Create load balancers with optional vips.
        lb_name: String used as basis for load balancer name.
        nbctl: Connection used for ovn-nbctl commands
        vips: Optional dictionary mapping VIPs to a list of backend IPs.
        protocols: List of protocols to use when creating Load Balancers.
        '''
        self.nbctl = nbctl
        self.protocols = [
            prot for prot in protocols if prot in VALID_PROTOCOLS
        ]
        if len(self.protocols) == 0:
            raise InvalidProtocol(protocols)
        self.name = lb_name
        self.vips = {}
        self.lbs = []
        for protocol in self.protocols:
            self.lbs.append(self.nbctl.create_lb(self.name, protocol))
        if vips:
            self.add_vips(vips)

    def add_vip(self, vip, vport, backends, backend_port, version):
        self.add_vips(
            OvnLoadBalancer.get_vip_map(
                vip, vport, backends, backend_port, version
            )
        )

    def add_vips(self, vips):
        '''
        Add VIPs to a load balancer.
        vips: Dictionary with key being a VIP string, and value being a list of
        backend IP address strings. It's perfectly acceptable for the VIP to
        have no backends.
        '''
        MAX_VIPS_IN_BATCH = 500
        for i in range(0, len(vips), MAX_VIPS_IN_BATCH):
            updated_vips = {}
            for vip, backends in itertools.islice(
                vips.items(), i, i + MAX_VIPS_IN_BATCH
            ):
                cur_backends = self.vips.setdefault(vip, [])
                if backends:
                    cur_backends.extend(backends)
                updated_vips[vip] = cur_backends

            for lb in self.lbs:
                self.nbctl.lb_set_vips(lb, updated_vips)

    def clear_vips(self):
        '''
        Clear all VIPs from the load balancer.
        '''
        self.vips.clear()
        for lb in self.lbs:
            self.nbctl.lb_clear_vips(lb)

    def add_backends_to_vip(self, backends, vips=None):
        '''
        Add backends to existing load balancer VIPs.
        backends: A list of IP addresses to add as backends to VIPs.
        vips: An iterable of VIPs to which backends should be added. If this is
        'None' then the backends are added to all VIPs.
        '''
        for cur_vip, cur_backends in self.vips.items():
            if not vips or cur_vip in vips:
                cur_backends.extend(backends)

        for lb in self.lbs:
            self.nbctl.lb_set_vips(lb, self.vips)

    def add_to_routers(self, routers):
        for lb in self.lbs:
            self.nbctl.lb_add_to_routers(lb, routers)

    def add_to_switches(self, switches):
        for lb in self.lbs:
            self.nbctl.lb_add_to_switches(lb, switches)

    def remove_from_routers(self, routers):
        for lb in self.lbs:
            self.nbctl.lb_remove_from_routers(lb, routers)

    def remove_from_switches(self, switches):
        for lb in self.lbs:
            self.nbctl.lb_remove_from_switches(lb, switches)

    @staticmethod
    def get_vip_map(vip, vport, backends, backend_port, version):
        if version == 6:
            return {
                f'[{vip}]:{vport}': [
                    f'[{b.ip6}]:{backend_port}' for b in backends
                ]
            }
        else:
            return {
                f'{vip}:{vport}': [f'{b.ip}:{backend_port}' for b in backends]
            }

class OvnLoadBalancerGroup:
    def __init__(self, group_name, nbctl):
        self.nbctl = nbctl
        self.name = group_name
        self.lbg = self.nbctl.create_lbg(self.name)

    def add_lb(self, ovn_lb):
        for lb in ovn_lb.lbs:
            self.nbctl.lbg_add_lb(self.lbg, lb)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 19 10:13:38 2019

@author: parkerglenn
"""
from sklearn.decomposition import PCA
import matplotlib.pylab as plt
import seaborn as sns
from sklearn.decomposition import TruncatedSVD
from matplotlib import pyplot
import mpld3
from mpld3 import display
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans
import pylab

###############################################################################
##################VISUALIZATION################################################
###############################################################################

#Date Distribution
out = pd.cut(df["joined"], bins = [20000000,20020000,20040000,20060000,20080000,20100000,20120000,20140000,20160000,20180000], include_lowest = True)
ax = out.value_counts(sort = False).plot.bar(rot = 0, color = "b", figsize = (20,10))
ax.set_ylim(bottom = 0, top = 450)
ax.set_xticklabels(x for x in ["2000 to 2002","2002 to 2004","2004 to 2006","2006 to 2008","2008 to 2010","2010 to 2012","2012 to 2014","2014 to 2016","2016 to 2018"])
plt.xlabel("Date Range", fontsize = 18)
plt.ylabel("Frequency", fontsize = 18)
plt.title("Date Distribution", fontsize = 25)
for i in ax.patches:
    ax.text(i.get_x() + .10, 5, str(i.get_height()), fontsize = 20, color = "black")

#Distance Heatmap
dist = 1 - cosine_similarity(matrix)

cmap = pyplot.cm.cubehelix
dimensions = (20,20)
fig, ax = pyplot.subplots(figsize=dimensions)
sns.heatmap(dist, vmin = 0, vmax = 1, cmap = cmap).set_title("Tfidf Distances Between Articles", fontsize = 15)

"""
Notice hot spot around the 625:630 line.
Those article titles:
['Dem Debate Blogging #1',
 'Dem Debate Blogging #2',
 'Dem Debate Blogging #3',
 'Dem Debate Blogging #4',
 'Dem Debate Blogging #5',
 'Dem Debate Blogging #6']

Around 90:160: large circle of relatively similar articles.
Reason: Iowa Caucuses.
"""

#Tfidf Matrix, in 2D SVD scatterplot
svd = TruncatedSVD(n_components=2).fit(matrix)
data2D = svd.transform(matrix)
plt.title("Truncated SVD, 2 Components")
colors = rng.rand(1000)
plt.scatter(data2D[:,0], data2D[:,1], marker = "o", c = colors, cmap = "YlGnBu", s = 10)

######With clusters assigned as colors########
data2D = svd.transform(matrix)
kmeans = KMeans(n_clusters = 520)
kmeans.fit(data2D)
y_kmeans = kmeans.predict(data2D)
y_pred = kmeans.labels_.tolist()

success(kmeans,y_pred,matrix)

articles = {"title": titles, "date": new_df["date"], "cluster": y_pred, "content": new_df["content"], "event": events[:1000]}
frame = pd.DataFrame(articles, index = [y_pred] , columns = ['title', 'date', 'cluster', 'content', "event"])
frame['cluster'].value_counts()

"""Creates scalable points for cluster centers found within y_true.
The size of the center is dependent on how many events occur withing that specific cluster. """

centers = kmeans.cluster_centers_
fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
threshold = 4
for cluster, center in enumerate(centers):
    cluster+=1
    # Only maps cluster if it has more than "threshold" events in it
    if cluster in y_true and len(frame.loc[cluster]["event"].values.tolist()) > threshold:
        #Gets event name that the cluster center represents
        event = events[y_true.index(cluster)]
        #s scaled based on number of events in cluster
        ax.plot(center[0], center[1], markersize = float(len(frame.loc[cluster]["event"].values.tolist())),  marker = "o");
        plt.annotate(event, (center[0],center[1]))
ax.set_title('Cluster Centers with Predominant Event', size=14)
plt.show()

mpld3.show(fig)

# mpld3.save_html(fig, "Cluster_Centers.html")

"""Zoomed out plot with colors"""
plt.title("Truncated SVD with Colored Clusters")
plt.scatter(data2D[:, 0], data2D[:, 1], c=y_kmeans, cmap = "tab20", s=30)
"""Example plot, zoomed in to visualize cluster centers"""

fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
ax.plot(data2D[:, 0], data2D[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('Truncated SVD with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    print(i)
    plt.annotate(txt + ", " + str(y_pred[i]), (data2D[:, 0][i], data2D[:, 1][i]))
mpld3.show(fig)

centers = kmeans.cluster_centers_
plt.scatter(centers[:, 0], centers[:, 1], c='black', s=30, alpha=0.5, marker = 'x');
pyplot.axis(ymax = 0, ymin =-.25 , xmax = .2, xmin = .14)
plt.title("Truncated SVD with Cluster Centers")
pyplot.axis(ymax = 0, ymin =-.25 , xmax = .2, xmin = .14)
plt.scatter(data2D[:, 0], data2D[:, 1], c=y_kmeans, cmap = "tab20", s=30)

######Interactive ScatterPlot with SVD######
svd = TruncatedSVD(n_components=2).fit(matrix)
data2D = svd.transform(matrix)

fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
ax.plot(data2D[:, 0], data2D[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('Truncated SVD with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    print(i)
    plt.annotate(txt + ", " + str(y_pred[i]), (data2D[:, 0][i], data2D[:, 1][i]))
mpld3.show(fig)
#mpld3.save_html(fig, "Truncated_SVD_D3.html")

#####Interactive ScatterPlot with Dense Matrix and PCA######
"""Probably not the method to use. SVD seems better since it takes the sparse matrix "tfidf_matrix" directly."""
x = tfidf_matrix.todense()
coords = PCA(n_components=2).fit_transform(x)
fig, ax = plt.subplots(figsize = (14,8))

np.random.seed(0)
ax.plot(coords[:, 0], coords[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('PCA with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    plt.annotate(txt, (coords[:, 0][i], coords[:, 1][i]))
mpld3.show(fig)

"""Dendogram Making"""
fig = pylab.figure(figsize=(100,70))
children = hac.children_
distance = np.arange(children.shape[0])
no_of_observations = np.arange(2, children.shape[0]+2)
linkage_matrix = np.column_stack([children, distance, no_of_observations]).astype(float)
dendrogram(linkage_matrix, labels = (events), truncate_mode = "level", leaf_font_size = 8)
fig.show

"""ScatterPlot"""
coords = PCA(n_components=2).fit_transform(dense_matrix)
fig, ax = plt.subplots(figsize = (14,8))
np.random.seed(0)
ax.plot(coords[:, 0], coords[:, 1],
        'or', ms=10, alpha=0.2)
ax.set_title('Truncated SVD with Cluster Assignments', size=14)
ax.grid(color='lightgray', alpha=0.7)
for i, txt in enumerate(events):
    print(i)
    plt.annotate(txt + ", " + str(y_pred[i]), (coords[:, 0][i], coords[:, 1][i]))
mpld3.show(fig)

#############################################################
################Hyperparameter Testing#######################
#############################################################

hyper_params = pd.read_csv("/Users/parkerglenn/Desktop/DataScience/Article_Clustering/csv/Hyperparam_testing/Hyper_Params.csv")
hyper_params = hyper_params.reset_index()

f_scores = [x for x in hyper_params["f_score"]]
s_scores = [x for x in hyper_params["s_score"]]
person_rate = [x for x in hyper_params["person_rate"]]
ents_rate = [x for x in hyper_params["ents_rate"]]

"""F1 Score distribution across hyperparameters
Best F1 Score: ents_rate = 1.63, person_rate = 2.57, f_score = .927 on 412 articles. BUT s_score is a measly .0788.
"""
f_data = pd.DataFrame({"Person Rate": [round(x,2) for x in person_rate], "Entity Rate":[round(x,2) for x in ents_rate], "Z": hyper_params.f_score})
f_data_pivoted = f_data.pivot("Person Rate","Entity Rate","Z")
ax1 = sns.heatmap(f_data_pivoted, cmap = plt.cm.hot)
cbar = ax1.collections[0].colorbar
cbar.set_label('F1 Score', labelpad=15)
ax1.invert_yaxis()
plt.show()

"""Silhouette Score distribution across hyperparameters
More direct correlation here than in F1 Score. Most notably, as the rates increase, so does S Score. This is due to
the fact that the rating of words in articles becomes more radical; articles that are different from each other, i.e. share no
entities, are now much more distant than before. Even more specifically, when the Entity Rate and Person Rate are dissimilar, the
S Score is the highest. Again, the same radical rating phenomena: By limiting the amount of weighted features, those that are weighted
make the article more of an outlier than before."""

s_data = pd.DataFrame({"Person Weighting": [round(x,2) for x in person_rate], "Non-Person Weighting":[round(x,2) for x in ents_rate], "Z": hyper_params.s_score})
s_data_pivoted = s_data.pivot("Person Weighting","Non-Person Weighting","Z")
ax2 = sns.heatmap(s_data_pivoted, cmap = plt.cm.hot)
cbar = ax2.collections[0].colorbar
cbar.set_label('Silhouette Score', labelpad=15)
ax2.invert_yaxis()
plt.show()

"""Cumulative Total
Highest: ents_rate = 6.368, person_rate = 2.263, f_score = 0.922, s_score = 0.093

"""
sf_data = pd.DataFrame({"Person Weighting": [round(x,2) for x in person_rate], "Entity Weighting":[round(x,2) for x in ents_rate], "Z": hyper_params.s_score + hyper_params.f_score})
sf_data_pivoted = sf_data.pivot("Person Weighting","Entity Weighting","Z")
ax3 = sns.heatmap(sf_data_pivoted, cmap = plt.cm.hot)
cbar = ax3.collections[0].colorbar
cbar.set_label('Composite Score', labelpad=15)
ax3.invert_yaxis()
plt.show()

"""Little thing to find best cumulative score and rates that achieved it."""
hightot = 0
for tup in enumerate(hyper_params["f_score"]):
    tot = tup[1] + hyper_params.loc[tup[0]]["s_score"]
    if hightot < tot:
        hightot = tot
        best_scores = (hyper_params.loc[tup[0]]["ents_rate"],hyper_params.loc[tup[0]]["person_rate"])
best_scores


"""Check command."""
from __future__ import annotations

import sys

import click
from result import Ok

from pyrgo.cli.cmds._utils import inform_and_run_program
from pyrgo.command_exec import PythonCommandExec
from pyrgo.conf import PyrgoConf

@click.command("check")
@click.option(
    "-t",
    "--timeout",
    "timeout",
    default=360,
    type=click.IntRange(min=60),
)
@click.option(
    "--add-noqa",
    "add_noqa",
    is_flag=True,
    default=False,
    show_default=True,
    type=click.BOOL,
)
@click.option(
    "--ignore-noqa",
    "ignore_noqa",
    is_flag=True,
    default=False,
    show_default=True,
    type=click.BOOL,
)
def check(*, timeout: int, add_noqa: bool, ignore_noqa: bool) -> None:
    """Check code with `mypy` and `ruff`."""
    configuration = PyrgoConf.new()
    ruff_command = PythonCommandExec.new(program="ruff")
    mypy_command = PythonCommandExec.new(program="mypy.dmypy").add_args(
        args=["run", "--timeout", str(timeout), "--"],
    )

    if add_noqa:
        ruff_command.add_args(args=["--add-noqa"])
    if ignore_noqa:
        ruff_command.add_args(args=["--ignore-noqa"])

    for command in [ruff_command, mypy_command]:
        command.add_args(args=configuration.relevant_paths)

    program_execution = inform_and_run_program(commands=[ruff_command, mypy_command])
    if not isinstance(program_execution, Ok):
        sys.exit(1)

    sys.exit(0)


######################################################
#
# Author: Davide Colombo
# Date: 30/12/21 15:01
# Description: INSERT HERE THE DESCRIPTION
#
######################################################
from datetime import datetime
from typing import Set, Dict, List
from airquality.database.adapter import DatabaseAdapter
from airquality.datamodel.fromdb import GeoareaLocationDM, OpenweathermapKeyDM, \
    SensorInfoDM, SensorLocationDM, SensorApiParamDM

class DatabaseGateway(object):
    def __init__(self, database_adapt: DatabaseAdapter):
        self._database_adapt = database_adapt

    def execute(self, query: str):
        self._database_adapt.execute(query)

# =========== SELECT ID QUERIES
    def _safe_fetch_id(self, query: str):
        row = self._database_adapt.fetchone(query)
        return 1 if row[0] is None else row[0] + 1

    def query_max_sensor_id_plus_one(self) -> int:
        return self._safe_fetch_id(
            query="SELECT MAX(id) FROM level0_raw.sensor;"
        )

    def query_max_mobile_packet_id_plus_one(self) -> int:
        return self._safe_fetch_id(
            query="SELECT MAX(packet_id) FROM level0_raw.mobile_measurement;"
        )

    def query_max_station_packet_id_plus_one(self) -> int:
        return self._safe_fetch_id(
            query="SELECT MAX(packet_id) FROM level0_raw.station_measurement;"
        )

# =========== SELECT SET QUERIES
    def query_poscodes_of_country(self, country_code: str) -> Set[str]:
        rows = self._database_adapt.fetchall(
            query=f"SELECT postal_code FROM level0_raw.geographical_area WHERE country_code = '{country_code}';"
        )
        return {row[0] for row in rows}

    def query_sensor_names_of_type(self, sensor_type: str) -> Set[str]:
        rows = self._database_adapt.fetchall(
            query=f"SELECT sensor_name FROM level0_raw.sensor WHERE sensor_type ILIKE '%{sensor_type}%';"
        )
        return {row[0] for row in rows}

# =========== SELECT MAPPING QUERIES
    def query_measure_param_owned_by(self, owner: str) -> Dict[str, int]:
        query = f"SELECT id, param_code FROM level0_raw.measure_param WHERE param_owner ILIKE '%{owner}%';"
        rows = self._database_adapt.fetchall(query=query)
        if len(rows) == 0:
            raise ValueError(f"Table 'level0_raw.measure_param' doesn't contain sensors owned by = '{owner}'")
        return {code: ident for ident, code in rows}

# =========== SELECT LIST QUERIES
    def query_sensor_apiparam_of_type(self, sensor_type: str) -> List[SensorApiParamDM]:
        query = "SELECT a.sensor_id, a.ch_key, a.ch_id, a.ch_name, a.last_acquisition " \
                "FROM level0_raw.sensor_api_param AS a INNER JOIN level0_raw.sensor AS s " \
                f"ON s.id = a.sensor_id WHERE s.sensor_type ILIKE '%{sensor_type}%';"
        rows = self._database_adapt.fetchall(query=query)
        if len(rows) == 0:
            raise ValueError(f"Table 'level0_raw.sensor_api_param' doesn't contain sensors of type = '{sensor_type}'")
        return [SensorApiParamDM(sid=sid, key=key, id=ident, ch=name, last=last) for sid, key, ident, name, last in rows]

    def query_openweathermap_keys(self) -> List[OpenweathermapKeyDM]:
        query = "SELECT key_value, done_req_min, max_req_min FROM level0_raw.openweathermap_key;"
        rows = self._database_adapt.fetchall(query=query)
        if len(rows) == 0:
            raise ValueError(f"Table 'level0_raw.openweathermap_key' is empty.")
        return [OpenweathermapKeyDM(key=key_val, n_done=done_r, n_max=max_r) for key_val, done_r, max_r in rows]

    def query_weather_conditions(self) -> Dict[str, int]:
        query = "SELECT id, code, icon FROM level0_raw.weather_condition;"
        rows = self._database_adapt.fetchall(query=query)
        if len(rows) == 0:
            raise ValueError(f"Table 'level0_raw.weather_condition' is empty.")
        return {f"{code}_{icon}": id_ for id_, code, icon in rows}

# =========== SELECT SINGLE ROW QUERIES
    def query_last_acquisition_of(self, sensor_id: int, ch_name: str) -> datetime:
        query = "SELECT last_acquisition FROM level0_raw.sensor_api_param " \
                f"WHERE sensor_id = {sensor_id} AND ch_name = '{ch_name}';"
        row = self._database_adapt.fetchone(query=query)
        if row is None:
            raise ValueError(f"Cannot found records corresponding to sensor_id = '{sensor_id}' and "
                             f"channel_name = '{ch_name}' in 'level0_raw.sensor_api_param' table.")
        return row[0]

    def query_place_location(self, country_code: str, place_name: str) -> GeoareaLocationDM:
        query = "SELECT id, ST_X(geom), ST_Y(geom) FROM level0_raw.geographical_area " \
                f"WHERE country_code = '{country_code}' AND place_name = '{place_name}';"
        row = self._database_adapt.fetchone(query=query)
        if row is None:
            raise ValueError(f"Cannot found a record corresponding to country_code = '{country_code}' and "
                             f"place_name = '{place_name}' in 'level0_raw.geographical_area' table")
        return GeoareaLocationDM(id=row[0], longitude=row[1], latitude=row[2])

    def query_fixed_sensor_unique_info(self, sensor_id: int):
        query = "SELECT s.id, s.sensor_name, ST_X(l.geom), ST_Y(l.geom) FROM level0_raw.sensor AS s " \
                "INNER JOIN level0_raw.sensor_at_location AS l ON s.id = l.sensor_id " \
                f"WHERE l.sensor_id = {sensor_id} ORDER BY l.valid_from DESC;"
        row = self._database_adapt.fetchone(query=query)
        if row is None:
            raise ValueError(f"Cannot found record corresponding to sensor_id = "
                             f"'{sensor_id}' in 'level0_raw.sensor_at_location' table")
        return SensorInfoDM(sensor_id=row[0], sensor_name=row[1], sensor_lng=row[2], sensor_lat=row[3])

    def query_mobile_sensor_unique_info(self, sensor_id: int):
        query = f"SELECT id, sensor_name FROM level0_raw.sensor WHERE id = {sensor_id};"
        row = self._database_adapt.fetchone(query=query)
        if row is None:
            raise ValueError(f"Cannot found record corresponding to sensor_id = "
                             f"'{sensor_id}' in 'level0_raw.sensor' table.")
        return SensorInfoDM(sensor_id=row[0], sensor_name=row[1])

    def query_purpleair_sensor_location(self, sensor_index: int) -> SensorLocationDM:
        query = "SELECT l.sensor_id, ST_X(geom), ST_Y(geom) FROM level0_raw.sensor_at_location AS l INNER JOIN " \
                "level0_raw.sensor AS s ON s.id = l.sensor_id WHERE s.sensor_type ILIKE '%purpleair%' AND " \
                f"sensor_name ILIKE '%{sensor_index}%' AND valid_to IS NULL;"
        row = self._database_adapt.fetchone(query=query)
        if row is None:
            raise ValueError(f"Cannot found a record corresponding to sensor_type = 'purpleair' which name "
                             f"contains sensor_index = '{sensor_index}' in 'level0_raw.sensor' table.")
        return SensorLocationDM(sensor_id=row[0], longitude=row[1], latitude=row[2])

    def query_hourly_forecast_records(self) -> List:
        return self._database_adapt.fetchall(query="SELECT * FROM level0_raw.hourly_forecast;")

    def query_daily_forecast_records(self) -> List:
        return self._database_adapt.fetchall(query="SELECT * FROM level0_raw.daily_forecast;")

    def exists_weather_alert_of(self, alert, geoarea_id: int) -> bool:
        row = self._database_adapt.fetchone(
            query=f"SELECT id FROM level0_raw.weather_alert WHERE geoarea_id = {geoarea_id} "
                  f"AND alert_event = '{alert.event}' AND alert_begin = '{alert.begin}';"
        )
        return False if row is None else True


import os

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

from data import predict_input_fn, pred_labels
from model import model_func

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

mnist_autoecnoder = tf.estimator.Estimator(model_fn=model_func,
        params={'n_encode': 3, 'n_outs': 784}, model_dir='./autoencoder')

imgs = []

preds = mnist_autoecnoder.predict(input_fn=predict_input_fn)
for i in preds:
    imgs.append(i['decoder_out'])

imgs = np.asarray(imgs)

indices = np.random.randint(pred_labels.shape[0], size=100)

plt.figure(figsize=(10, 10))
for i, index in enumerate(indices):
    plt.subplot(10, 10, i+1)
    plt.imshow(imgs[index].reshape(28, 28), cmap='gray')
    plt.axis('off')
plt.tight_layout()
plt.subplots_adjust(wspace=0, hspace=0)
plt.show()


#
# This file contains important functions for calculations and converting dates.
#

import matplotlib.dates as mdates
import numpy as np

# converts bytes
def bytedate2num(fmt):
    def converter(byte):
        return mdates.strpdate2num(fmt)(byte.decode("utf-8"))
    return converter

# convert the date to a numeric number
def date2num(df,fmt):
    date_converter = bytedate2num(fmt)
    date = np.loadtxt(df, converters={0: date_converter})
    return date

# calculates the lim
def lims(df,lim,percent):
    lim = lim
    if lim == "max":
        lim = np.max(df)
        lim = lim*percent

    elif lim == "min":
        lim = np.min(df)
        lim = lim*percent

    else:
        lim = float(lim)

    return lim  # return the lim

# calculates the moving average
def movingaverage(values,window):

    weights = (np.repeat(1.0,window))/window

    smas = np.convolve(values,weights,"valid")
    return smas  # return the average

# calculates the RSI
def rsiFunc(prices,n=14):
    deltas = np.diff(prices)
    seed = deltas[:n+1]
    down = -seed[seed<0].sum()/n
    up = seed[seed>=0].sum()/n
    rs = up/down
    rsi = np.zeros_like(prices)
    rsi[:n] = 100. - 100./(1.+rs)

    for i in range(n,len(prices)):
        delta= deltas[i-1]
        if delta > 0:
            upval = delta
            downval = 0.
        else:
            upval=0.
            downval = -delta

        up = (up*(n-1)+upval)/n
        down = (down*(n-1)+downval)/n

        rs = up/down
        rsi[i]= 100.-100./(1.+rs)

    return rsi  # return the RSI


from django.urls import path
from rest_framework_nested import routers

from accounting_report.api.viewsets import AccountingReportViewSet
from bank_guarantee.api.viewsets import BGRequestsViewSet

from clients.api.viewsets import AgentInstructionsDocumentsViewSet

from clients.api.views import SetContractStateView, CompanyContactsView, \
    ClientStatisticView
from clients.api.viewsets import (
    ClientsViewSet, AgentsViewSet, AgentSettingsViewSet, AgentSettingsDocumentsViewSet,
    AgentInformationViewSet, CompanyViewSet, CompanyInfoViewSet,
    BankRejectionReasonTemplateViewSet, BankBlackListViewSet
)

router = routers.SimpleRouter()
router.register(r'clients', ClientsViewSet, basename='clients')
router.register(r'agents', AgentsViewSet, basename='agents')
router.register(r'accounting_report', AccountingReportViewSet,
                basename='accounting_report')

router.register(r'settings', AgentSettingsViewSet, basename='agent_settings')

router.register(r'settings/documents', AgentSettingsDocumentsViewSet,
                basename='agent_settings_documents')

router.register(r'settings/documents_info', AgentSettingsDocumentsViewSet,
                basename='agent_settings_documents')

router.register(r'agent_information', AgentInformationViewSet,
                basename='agent_information')

router.register(r'company', CompanyViewSet, basename='company')
router.register(r'requests/bank_guarantee', BGRequestsViewSet,
                basename='bank_guarantee')
router.register(r'company_info', CompanyInfoViewSet, basename='company_info')

router.register(r'agent_instructions_documents', AgentInstructionsDocumentsViewSet,
                basename='agent_instructions_documents')

router.register(r'bank_rejection_reason_templates',
                BankRejectionReasonTemplateViewSet,
                basename='bank_rejection_reason_template')

router.register(r'bank_black_list',
                BankBlackListViewSet,
                basename='bank_black_list')

urlpatterns = [
                  path('set_contract_state/<int:pk>/', SetContractStateView.as_view(),
                       name='set_contract_state'),
                  path('set_contract_state/', SetContractStateView.as_view(),
                       name='set_contract_state'),
                  path('company_contacts', CompanyContactsView.as_view(),
                       name='company_contacts'),
                  path('clients/statistic', ClientStatisticView.as_view(),
                       name='company_contacts'),
              ] + router.urls


"""
Setup Module to setup Python Handlers (Git Handlers) for the Git Plugin.
"""
import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name='jupyterlab_git',
    version='0.5.0',
    author='Git Intern Team - Noah Stapp, Jenna Landy, Alena Mueller; AWS - Jaipreet Singh, Neelam Gehlot; Quansight - Saul Shanabrook',
    description="A server extension for JupyterLab's git extension",
    long_description=long_description,
    long_description_content_type="text/markdown",
    packages=setuptools.find_packages(),
    install_requires=[
        'notebook',
        'psutil'
    ],
    package_data={'jupyterlab_git': ['*']},
)


import numpy as np
import pandas as pd
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.neighbors import KNeighborsClassifier
import random
from sklearn.metrics import classification_report
from sklearn.neural_network import MLPClassifier

filename = '/home/ucesxc0/Scratch/output/second_training_images_1/cnn_tf2_1/merged_image_for_training'
np.random.seed(7)
random.seed(7)

df = pd.read_csv(filename)
X = np.array(df.loc[:, df.columns !='Label'])
Y = np.array(df['Label'])

#split data into train and test
X_train, X_test, y_train, y_test = train_test_split(X,Y, test_size=0.2, random_state=7)

# KNN Grid Search
KNN = KNeighborsClassifier()
parameters = {'n_neighbors': [3, 5, 10, 15, 20, 25, 30, 35, 40]}
clf = GridSearchCV(estimator=KNN, param_grid=parameters, cv=5)
fit = clf.fit(X_train, y_train)
print('optimal parameter value: ', fit.best_params_)
Prediction_KNN = fit.best_estimator_.predict(X_test)
Accuracy_KNN = len(np.where(Prediction_KNN == y_test)[0]) * 1. / len(y_test)
print('Accuracy: ', Accuracy_KNN)
print(classification_report(y_test, Prediction_KNN, digits=3))

# SVM Grid Search
SVM = SVC()
parameters = {'C': [0.5, 1, 4, 7, 10, 13, 16, 20]}
clf = GridSearchCV(estimator=SVM, param_grid=parameters, cv=5)
fit = clf.fit(X_train, y_train)
print('optimal parameter value: ', fit.best_params_)
Prediction_SVM = fit.best_estimator_.predict(X_test)
Accuracy_SVM = len(np.where(Prediction_SVM == y_test)[0]) * 1. / len(y_test)
print('Accuracy: ', Accuracy_SVM)
print(classification_report(y_test, Prediction_SVM, digits=3))

# Decision Tree Grid Search
DT = DecisionTreeClassifier()
parameters = {'max_depth': [1, 5, 10, 15, 20, 25, 30, 35, 40]}
clf = GridSearchCV(estimator=DT, param_grid=parameters, cv=5)
fit = clf.fit(X_train, y_train)
print('optimal parameter value: ', fit.best_params_)
Prediction_DT = fit.best_estimator_.predict(X_test)
Accuracy_DecisionTree = len(np.where(Prediction_DT == y_test)[0]) * 1. / len(y_test)
print('Accuracy: ', Accuracy_DecisionTree)
print(classification_report(y_test, Prediction_DT, digits=3))


from django.urls import path
from django.contrib.auth import views as auth_views
from . import views

urlpatterns = [
    path('login/', auth_views.LoginView.as_view() , name='login'),
    path('logout/', auth_views.LogoutView.as_view() , name='logout'),
    path('register/', views.register , name='register'),
    path('difviews/' , views.difviews , name='difviews'),
    path('<int:id>/' , views.order_detail , name='order_detail'),
]


import re

from genshi.builder import Markup, tag
from genshi.filters.transform import Transformer
from trac.config import BoolOption
from trac.core import Component, ExtensionPoint, implements
from trac.mimeview.api import Context
from trac.resource import Resource, render_resource_link, get_resource_url
from trac.util.compat import sorted
from trac.web.api import IRequestFilter, ITemplateStreamFilter
from trac.web.chrome import add_stylesheet
from trac.wiki.api import IWikiChangeListener, IWikiPageManipulator, \
                          IWikiSyntaxProvider
from trac.wiki.model import WikiPage
from trac.wiki.web_ui import WikiModule

from tractags.api import DefaultTagProvider, TagSystem, _
from tractags.macros import TagTemplateProvider

class WikiTagProvider(DefaultTagProvider):
    """Tag provider for Trac wiki."""

    realm = 'wiki'

    exclude_templates = BoolOption('tags', 'query_exclude_wiki_templates',
        default=True,
        doc="Whether tagged wiki page templates should be queried.")

    first_head = re.compile('=\s+([^=\n]*)={0,1}')

    def check_permission(self, perm, action):
        map = {'view': 'WIKI_VIEW', 'modify': 'WIKI_MODIFY'}
        return super(WikiTagProvider, self).check_permission(perm, action) \
            and map[action] in perm

    def get_tagged_resources(self, req, tags, filter=None):
        if self.exclude_templates:
            db =  self.env.get_db_cnx()
            like_templates = ''.join(
                ["'", db.like_escape(WikiModule.PAGE_TEMPLATES_PREFIX), "%%'"])
            filter = (' '.join(['name NOT', db.like() % like_templates]),)
        return super(WikiTagProvider, self).get_tagged_resources(req, tags,
                                                                 filter)

    def describe_tagged_resource(self, req, resource):
        if not self.check_permission(req.perm(resource), 'view'):
            return ''
        page = WikiPage(self.env, resource.id)
        if page.exists:
            ret = self.first_head.search(page.text)
            return ret and ret.group(1) or ''
        return ''

class WikiTagInterface(TagTemplateProvider):
    """Implement the user interface for tagging Wiki pages."""
    implements(IRequestFilter, ITemplateStreamFilter,
               IWikiChangeListener, IWikiPageManipulator)

    # IRequestFilter methods
    def pre_process_request(self, req, handler):
        return handler

    def post_process_request(self, req, template, data, content_type):
        if req.method == 'GET' and req.path_info.startswith('/wiki/') and \
                req.args.get('action') == 'edit' and \
                req.args.get('template') and 'tags' not in req.args:
            # Retrieve template resource to be queried for tags.
            template_page = WikiPage(self.env,''.join(
                                     [WikiModule.PAGE_TEMPLATES_PREFIX,
                                      req.args.get('template')]))
            if template_page and template_page.exists and \
                    'TAGS_VIEW' in req.perm(template_page.resource):
                ts = TagSystem(self.env)
                tags = sorted(ts.get_tags(req, template_page.resource))
                # Prepare tags as content for the editor field.
                tags_str = ' '.join(tags)
                self.env.log.debug("Tags retrieved from template: '%s'" \
                                   % unicode(tags_str).encode('utf-8'))
                # DEVEL: More arguments need to be propagated here?
                req.redirect(req.href(req.path_info,
                                      action='edit', tags=tags_str,
                                      template=req.args.get('template')))
        return (template, data, content_type)

    # ITemplateStreamFilter methods
    def filter_stream(self, req, method, filename, stream, data):
        page_name = req.args.get('page', 'WikiStart')
        resource = Resource('wiki', page_name)
        if filename == 'wiki_view.html' and 'TAGS_VIEW' in req.perm(resource):
            return self._wiki_view(req, stream)
        elif filename == 'wiki_edit.html' and 'TAGS_MODIFY' in req.perm(resource):
            return self._wiki_edit(req, stream)
        return stream

    # IWikiPageManipulator methods
    def prepare_wiki_page(self, req, page, fields):
        pass

    def validate_wiki_page(self, req, page):
        # If we're saving the wiki page, and can modify tags, do so
        if req and 'TAGS_MODIFY' in req.perm(page.resource) \
                and req.path_info.startswith('/wiki') and 'save' in req.args:
            page_modified = req.args.get('text') != page.old_text or \
                    page.readonly != int('readonly' in req.args)
            # Always save tags if the page has been otherwise modified
            if page_modified:
                self._update_tags(req, page)
            elif page.version > 0:
                # If the page hasn't been otherwise modified, save the tags
                # and redirect so we don't get the "page has not been modified"
                # warning
                if self._update_tags(req, page):
                    req.redirect(get_resource_url(self.env, page.resource, req.href, version=None))
        return []

    # IWikiChangeListener methods
    def wiki_page_added(self, page):
        pass

    def wiki_page_changed(self, page, version, t, comment, author, ipnr):
        pass

    def wiki_page_renamed(self, page, old_name):
        """Called when a page has been renamed (since Trac 0.12)."""
        new_resource = Resource('wiki', page.name)
        old_resource = Resource('wiki', old_name)
        self.log.debug("Moving tags from %s to %s",
                       old_resource.id, new_resource.id)
        tag_system = TagSystem(self.env)
        # XXX Ugh. Hopefully this will be sufficient to fool any endpoints.
        from trac.test import Mock, MockPerm
        req = Mock(authname='anonymous', perm=MockPerm())
        tag_system.reparent_tags(req, old_resource, new_resource)

    def wiki_page_deleted(self, page):
        tag_system = TagSystem(self.env)
        # XXX Ugh. Hopefully this will be sufficient to fool any endpoints.
        from trac.test import Mock, MockPerm
        req = Mock(authname='anonymous', perm=MockPerm())
        tag_system.delete_tags(req, page.resource)

    def wiki_page_version_deleted(self, page):
        pass

    # Internal methods
    def _page_tags(self, req):
        pagename = req.args.get('page', 'WikiStart')

        tag_system = TagSystem(self.env)
        resource = Resource('wiki', pagename)
        tags = sorted(tag_system.get_tags(req, resource))
        return tags

    def _wiki_view(self, req, stream):
        add_stylesheet(req, 'tags/css/tractags.css')
        tags = self._page_tags(req)
        if not tags:
            return stream
        tag_system = TagSystem(self.env)
        li = []
        for tag_ in tags:
            resource = Resource('tag', tag_)
            anchor = render_resource_link(self.env,
                Context.from_request(req, resource), resource)
            anchor = anchor(rel='tag')
            li.append(tag.li(anchor, ' '))

        # TRANSLATOR: Header label text for tag list at wiki page bottom.
        insert = tag.ul(class_='tags')(tag.li(_("Tags"), class_='header'), li)
        return stream | Transformer('//div[contains(@class,"wikipage")]').after(insert)

    def _update_tags(self, req, page):
        tag_system = TagSystem(self.env)
        newtags = tag_system.split_into_tags(req.args.get('tags', ''))
        oldtags = tag_system.get_tags(req, page.resource)

        if oldtags != newtags:
            tag_system.set_tags(req, page.resource, newtags)
            return True
        return False

    def _wiki_edit(self, req, stream):
        # TRANSLATOR: Label text for link to '/tags'.
        link = tag.a(_("view all tags"), href=req.href.tags())
        # TRANSLATOR: ... (view all tags)
        insert = tag(Markup(_("Tag under: (%(tags_link)s)", tags_link=link)))
        insert(
            tag.br(),
            tag.input(id='tags', type='text', name='tags', size='50',
                value=req.args.get('tags', ' '.join(self._page_tags(req))))
        )
        insert = tag.div(tag.label(insert), class_='field')
        return stream | Transformer('//div[@id="changeinfo1"]').append(insert)

class TagWikiSyntaxProvider(Component):
    """Provide tag:<expr> links."""

    implements(IWikiSyntaxProvider)

    # IWikiSyntaxProvider methods
    def get_wiki_syntax(self):
        yield (r'''\[tag(?:ged)?:(?P<tlpexpr>'.*'|".*"|\S+)\s*(?P<tlptitle>[^\]]+)?\]''',
               lambda f, n, m: self._format_tagged(f,
                                    m.group('tlpexpr'),
                                    m.group('tlptitle')))
        yield (r'''(?P<tagsyn>tag(?:ged)?):(?P<texpr>(?:'.*?'|".*?"|\S)+)''',
               lambda f, n, m: self._format_tagged(f,
                                    m.group('texpr'),
                                    '%s:%s' % (m.group('tagsyn'), m.group('texpr'))))

    def get_link_resolvers(self):
        return []

    def _format_tagged(self, formatter, target, label):
        RE = re.compile(r'^(\\[\'"]*\'|\\[\'"]*\"|"|\')(.*)(\1)')
        iter_max = 5
        iter_run = 0
        for iter_run in range(0, iter_max):
            # Reduce outer quotations.
            ctarget = RE.sub(r'\2', target)
            iter_run += 1
            if ctarget == target or iter_run > iter_max:
                self.env.log.debug(' ,'.join([target, ctarget, str(iter_run)]))
                break
            target = ctarget
        if label:
            href = formatter.context.href
            url = get_resource_url(self.env, Resource('tag', target), href)
            return tag.a(label, href=url)
        return render_resource_link(self.env, formatter.context,
                                    Resource('tag', target))


import app_config

from main.models import EducationCenter
import csv
from csv import reader
import sys
from django.contrib.gis.geos import GEOSGeometry
WORKING_DIR = app_config.proj_path + '/util_scripts/'

def update_locations(filename):
    with open(f"{WORKING_DIR}{filename}",encoding='ISO-8859-1') as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',', quotechar='"')
        next(csv_reader)
        for row in csv_reader:
            print("Updating geom for center {0}".format( row[1] ))
            wkt_point = "POINT ({0} {1})".format(row[2], row[3])
            point = GEOSGeometry(wkt_point)
            center = EducationCenter.objects.get(pk=(int(row[0])))
            center.location = point
            center.save()
            print("Center updated")

def main():
    args = sys.argv[1:]
    filename = args[0]
    update_locations(filename)

if __name__ == '__main__':
    main()


import numpy as np
from keras import Sequential
from keras.datasets import mnist
from keras.layers import Dense, Activation
from keras.optimizers import SGD
from keras.utils import np_utils

# to reproduce results, you probably knew that already...
np.random.seed(1671)

# building the network
NB_EPOCH = 200
BATCH_SIZE = 128
VERBOSE = 1
# number of possible outputs == number of digits, i.e. |[0-9]|
NB_CLASSES = 10
OPTIMIZER = SGD()
N_HIDDEN = 128
# how much of training set is saved for validation, in this case 20%?
VALIDATION_SPLIT = 0.2

# data is shuffled and split between training and validation sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784
# 784 being the number of input features, one per pixel of MNIST image
RESHAPED = 784

# side note: the book doesn't follow PEP 8 at all. Their comment spacing is all over the place. Drives me nuts.
X_train = X_train.reshape(60000, RESHAPED)
X_test = X_test.reshape(10000, RESHAPED)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

# normalize
X_train /= 255
X_test /= 255

print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, NB_CLASSES)
Y_test = np_utils.to_categorical(y_test, NB_CLASSES)

# final layer is softmax activation function
# 10 output classes, one for each digit
model = Sequential()
model.add(Dense(NB_CLASSES, input_shape=(RESHAPED,)))
model.add(Activation('softmax'))
model.summary()

# compile.
# using categorical cross-entropy objective function, as that apparently makes the most sense with softmax
# and accuracy as a metric
model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])

# train.
# using NB_EPOCHS epochs (i.e. exposures to training set) and BATCH_SIZE batch size (i.e. number of training cases
# observed before model updates weights)
history = model.fit(X_train,
                    Y_train,
                    batch_size=BATCH_SIZE,
                    epochs=NB_EPOCH,
                    verbose=VERBOSE,
                    validation_split=VALIDATION_SPLIT)

# evaluate on the reserved validation set
score = model.evaluate(X_test, Y_test, verbose=VERBOSE)
print("Test score: ", score[0])
print("Test accuracy ", score[1])

# after run results:
# Test score:  0.27738585292994977
# Test accuracy  0.9227


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#    @Author : 李将
#
#              _____               ______
#     ____====  ]OO|_n_n__][.      |    |]
#    [________]_|__|________)<
#     oo    oo  'oo OOOO-| oo\_   ~o~~~o~'
# +--+--+--+--+--+--+--+--+--+--+--+--+--+
#    @Time : 2022/9/2 14:40
#    @FIle： client.py
#    @Software: PyCharm

from google.protobuf import json_format

from grpc.pb2 import hello_billbil_pb2 as pb2

request = pb2.HelloDeWeiReq()
request.name = 'boby'
request.age = 11

res_str = request.SerializeToString()  # 序列化 变成二进制

print('res_str', res_str)

request2 = pb2.HelloDeweiRsp()

request2.ParseFromString(res_str)  # 反序列化，将二进制转化成对象

res_json = json_format.MessageToJson(request2)  # 设置为json
print(res_json)