import sys
import fs
import sqlite3

from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMainWindow, QWidget, QTableWidget, QTableWidgetItem, QLabel, QApplication, QMessageBox

def word_alg(word, part):
    ending = ""
    suffix = ""
    prefix = ""
    gender = "Нет"
    form = "Нет"
    do_break = False
    for ltr in word[::-1]:
        ending += ltr
        for morpheme in ex.language.morphemes:
            if ending[::-1] == morpheme.body and morpheme._type[:6] == "ending" and morpheme.part == part:
                ending = ending[::-1]
                if morpheme.part in ("noun", "adjective"):
                    gender = ex.language.endings_gender[ending][:4]
                    form = ex.language.endings_gender[ending][5:]
                do_break = True
                break
        if do_break:
            break
    do_break = False
    if ending == word[::-1]:
        ending = "Нулевое"
        end_gap = 0
    else:
        end_gap = len(ending)
    for ltr in word[-1 - end_gap::-1]:
        suffix += ltr
        for morpheme in ex.language.morphemes:
            if suffix[::-1] == morpheme.body and (morpheme._type == "suffix" or morpheme._type == "all")\
                    and morpheme.part == part:
                suffix = suffix[::-1]
                do_break = True
                break
        if do_break:
            break
    if suffix == word[-1 - end_gap::-1]:
        suffix = "Нет"
        suf_gap = 0
    else:
        suf_gap = len(suffix)
    do_break = False
    for ltr in word:
        prefix += ltr
        for morpheme in ex.language.morphemes:
            if prefix == morpheme.body and (morpheme._type == "prefix" or morpheme._type == "all"):
                do_break = True
                break
        if do_break:
            break
    if prefix == word:
        prefix = "Нет"
        pre_gap = 0
    else:
        pre_gap = len(prefix)
    root = word[pre_gap:len(word) - suf_gap - end_gap]
    ipa_word = "["
    for letter in word.lower():
        for k, v in ex.language.alphabet.items():
            if k == letter:
                ipa_word += v
    return [root, prefix, suffix, ending, ipa_word + "]", form, gender]

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.setEnabled(True)
        MainWindow.resize(800, 673)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setEnabled(True)
        self.centralwidget.setObjectName("centralwidget")
        self.lineEdit_2 = QtWidgets.QLineEdit(self.centralwidget)
        self.lineEdit_2.setGeometry(QtCore.QRect(10, 20, 113, 20))
        self.lineEdit_2.setObjectName("lineEdit_2")
        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        self.label_2.setGeometry(QtCore.QRect(40, 0, 41, 16))
        self.label_2.setObjectName("label_2")
        self.tableWidget = QtWidgets.QTableWidget(self.centralwidget)
        self.tableWidget.setEnabled(True)
        self.tableWidget.setGeometry(QtCore.QRect(10, 50, 481, 581))
        self.tableWidget.setObjectName("tableWidget")
        self.tableWidget.setColumnCount(0)
        self.tableWidget.setRowCount(0)
        self.label_12 = QtWidgets.QLabel(self.centralwidget)
        self.label_12.setGeometry(QtCore.QRect(520, 10, 221, 31))
        font = QtGui.QFont()
        font.setPointSize(18)
        self.label_12.setFont(font)
        self.label_12.setObjectName("label_12")
        self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_2.setGeometry(QtCore.QRect(130, 10, 91, 31))
        self.pushButton_2.setObjectName("pushButton_2")
        self.label_18 = QtWidgets.QLabel(self.centralwidget)
        self.label_18.setGeometry(QtCore.QRect(230, 10, 221, 31))
        self.label_18.setText("")
        self.label_18.setObjectName("label_18")
        self.layoutWidget = QtWidgets.QWidget(self.centralwidget)
        self.layoutWidget.setGeometry(QtCore.QRect(510, 50, 271, 61))
        self.layoutWidget.setObjectName("layoutWidget")
        self.horizontalLayout = QtWidgets.QHBoxLayout(self.layoutWidget)
        self.horizontalLayout.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.pushButton_3 = QtWidgets.QPushButton(self.layoutWidget)
        self.pushButton_3.setObjectName("pushButton_3")
        self.horizontalLayout.addWidget(self.pushButton_3)
        self.pushButton_4 = QtWidgets.QPushButton(self.layoutWidget)
        self.pushButton_4.setObjectName("pushButton_4")
        self.horizontalLayout.addWidget(self.pushButton_4)
        self.layoutWidget1 = QtWidgets.QWidget(self.centralwidget)
        self.layoutWidget1.setGeometry(QtCore.QRect(510, 110, 271, 61))
        self.layoutWidget1.setObjectName("layoutWidget1")
        self.horizontalLayout_2 = QtWidgets.QHBoxLayout(self.layoutWidget1)
        self.horizontalLayout_2.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout_2.setObjectName("horizontalLayout_2")
        self.pushButton_5 = QtWidgets.QPushButton(self.layoutWidget1)
        self.pushButton_5.setObjectName("pushButton_5")
        self.horizontalLayout_2.addWidget(self.pushButton_5)
        self.pushButton = QtWidgets.QPushButton(self.layoutWidget1)
        self.pushButton.setObjectName("pushButton")
        self.horizontalLayout_2.addWidget(self.pushButton)
        self.pushButton_8 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_8.setGeometry(QtCore.QRect(600, 570, 75, 41))
        self.pushButton_8.setStyleSheet("background-color: rgb(255, 63, 5);")
        self.pushButton_8.setObjectName("pushButton_8")
        self.layoutWidget2 = QtWidgets.QWidget(self.centralwidget)
        self.layoutWidget2.setGeometry(QtCore.QRect(510, 170, 271, 61))
        self.layoutWidget2.setObjectName("layoutWidget2")
        self.horizontalLayout_3 = QtWidgets.QHBoxLayout(self.layoutWidget2)
        self.horizontalLayout_3.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout_3.setObjectName("horizontalLayout_3")
        self.pushButton_6 = QtWidgets.QPushButton(self.layoutWidget2)
        self.pushButton_6.setObjectName("pushButton_6")
        self.horizontalLayout_3.addWidget(self.pushButton_6)
        self.pushButton_10 = QtWidgets.QPushButton(self.layoutWidget2)
        self.pushButton_10.setObjectName("pushButton_10")
        self.horizontalLayout_3.addWidget(self.pushButton_10)
        self.label_5 = QtWidgets.QLabel(self.centralwidget)
        self.label_5.setGeometry(QtCore.QRect(590, 240, 101, 31))
        font = QtGui.QFont()
        font.setPointSize(18)
        self.label_5.setFont(font)
        self.label_5.setObjectName("label_5")
        self.widget = QtWidgets.QWidget(self.centralwidget)
        self.widget.setGeometry(QtCore.QRect(510, 280, 271, 25))
        self.widget.setObjectName("widget")
        self.horizontalLayout_4 = QtWidgets.QHBoxLayout(self.widget)
        self.horizontalLayout_4.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout_4.setObjectName("horizontalLayout_4")
        self.pushButton_7 = QtWidgets.QPushButton(self.widget)
        self.pushButton_7.setObjectName("pushButton_7")
        self.horizontalLayout_4.addWidget(self.pushButton_7)
        self.pushButton_9 = QtWidgets.QPushButton(self.widget)
        self.pushButton_9.setObjectName("pushButton_9")
        self.horizontalLayout_4.addWidget(self.pushButton_9)
        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 800, 21))
        self.menubar.setObjectName("menubar")
        self.menuTable = QtWidgets.QMenu(self.menubar)
        self.menuTable.setObjectName("menuTable")
        self.menu = QtWidgets.QMenu(self.menubar)
        self.menu.setObjectName("menu")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)
        self.action = QtWidgets.QAction(MainWindow)
        self.action.setObjectName("action")
        self.action_2 = QtWidgets.QAction(MainWindow)
        self.action_2.setObjectName("action_2")
        self.action_id = QtWidgets.QAction(MainWindow)
        self.action_id.setObjectName("action_id")
        self.action_4 = QtWidgets.QAction(MainWindow)
        self.action_4.setObjectName("action_4")
        self.action_6 = QtWidgets.QAction(MainWindow)
        self.action_6.setObjectName("action_6")
        self.action_7 = QtWidgets.QAction(MainWindow)
        self.action_7.setObjectName("action_7")
        self.action_8 = QtWidgets.QAction(MainWindow)
        self.action_8.setObjectName("action_8")
        self.action_10 = QtWidgets.QAction(MainWindow)
        self.action_10.setObjectName("action_10")
        self.action_3 = QtWidgets.QAction(MainWindow)
        self.action_3.setObjectName("action_3")
        self.action_5 = QtWidgets.QAction(MainWindow)
        self.action_5.setObjectName("action_5")
        self.menuTable.addAction(self.action)
        self.menuTable.addAction(self.action_2)
        self.menuTable.addSeparator()
        self.menuTable.addAction(self.action_id)
        self.menuTable.addAction(self.action_6)
        self.menuTable.addAction(self.action_7)
        self.menuTable.addAction(self.action_5)
        self.menuTable.addAction(self.action_4)
        self.menuTable.addSeparator()
        self.menuTable.addAction(self.action_10)
        self.menu.addAction(self.action_8)
        self.menubar.addAction(self.menuTable.menuAction())
        self.menubar.addAction(self.menu.menuAction())

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))
        self.label_2.setText(_translate("MainWindow", "Таблица"))
        self.label_12.setText(_translate("MainWindow", "Работа с данными"))
        self.pushButton_2.setText(_translate("MainWindow", "Найти/обновить"))
        self.pushButton_3.setText(_translate("MainWindow", "Добавить строку"))
        self.pushButton_4.setText(_translate("MainWindow", "Сохранить"))
        self.pushButton_5.setText(_translate("MainWindow", "Сортировать id"))
        self.pushButton.setText(_translate("MainWindow", "Удалить выбранную строку"))
        self.pushButton_8.setText(_translate("MainWindow", "Выйти"))
        self.pushButton_6.setText(_translate("MainWindow", "Выгрузить таблицу"))
        self.pushButton_10.setText(_translate("MainWindow", "Загрузить данные языка"))
        self.label_5.setText(_translate("MainWindow", "Функции"))
        self.pushButton_7.setText(_translate("MainWindow", "Морфологический анализ"))
        self.pushButton_9.setText(_translate("MainWindow", "Анализ текста"))
        self.menuTable.setTitle(_translate("MainWindow", "Таблица"))
        self.menu.setTitle(_translate("MainWindow", "Функции"))
        self.action.setText(_translate("MainWindow", "Загрузить таблицу"))
        self.action.setShortcut(_translate("MainWindow", "Ctrl+O"))
        self.action_2.setText(_translate("MainWindow", "Выгрузить таблицу"))
        self.action_2.setShortcut(_translate("MainWindow", "Ctrl+D"))
        self.action_id.setText(_translate("MainWindow", "Сортировать id"))
        self.action_id.setShortcut(_translate("MainWindow", "Shift+S"))
        self.action_4.setText(_translate("MainWindow", "Сохранить"))
        self.action_4.setShortcut(_translate("MainWindow", "Ctrl+S"))
        self.action_6.setText(_translate("MainWindow", "Добавить строку"))
        self.action_6.setShortcut(_translate("MainWindow", "Ctrl+Shift+A"))
        self.action_7.setText(_translate("MainWindow", "Удалить выбранную строку"))
        self.action_7.setShortcut(_translate("MainWindow", "Ctrl+Shift+D"))
        self.action_8.setText(_translate("MainWindow", "Морфологический анализ"))
        self.action_8.setShortcut(_translate("MainWindow", "Ctrl+M"))
        self.action_10.setText(_translate("MainWindow", "Выход"))
        self.action_3.setText(_translate("MainWindow", "Обновить таблицу"))
        self.action_3.setShortcut(_translate("MainWindow", "Ctrl+R"))
        self.action_5.setText(_translate("MainWindow", "Загрузить данные языка"))
        self.action_5.setShortcut(_translate("MainWindow", "Ctrl+L"))

class Alg_Form(object):
    def setupUi(self, Form):
        Form.setObjectName("Form")
        Form.resize(324, 330)
        self.verticalLayout = QtWidgets.QVBoxLayout(Form)
        self.verticalLayout.setObjectName("verticalLayout")
        self.label_5 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(18)
        self.label_5.setFont(font)
        self.label_5.setObjectName("label_5")
        self.verticalLayout.addWidget(self.label_5)
        self.horizontalLayout_7 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_7.setObjectName("horizontalLayout_7")
        self.label_6 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_6.setFont(font)
        self.label_6.setObjectName("label_6")
        self.horizontalLayout_7.addWidget(self.label_6)
        self.lineEdit_3 = QtWidgets.QLineEdit(Form)
        self.lineEdit_3.setObjectName("lineEdit_3")
        self.horizontalLayout_7.addWidget(self.lineEdit_3)
        self.verticalLayout.addLayout(self.horizontalLayout_7)
        self.horizontalLayout_9 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_9.setObjectName("horizontalLayout_9")
        self.label_2 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_2.setFont(font)
        self.label_2.setObjectName("label_2")
        self.horizontalLayout_9.addWidget(self.label_2)
        self.lineEdit_8 = QtWidgets.QLineEdit(Form)
        self.lineEdit_8.setEnabled(True)
        self.lineEdit_8.setObjectName("lineEdit_8")
        self.horizontalLayout_9.addWidget(self.lineEdit_8)
        self.verticalLayout.addLayout(self.horizontalLayout_9)
        self.horizontalLayout_6 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_6.setObjectName("horizontalLayout_6")
        self.label_7 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_7.setFont(font)
        self.label_7.setObjectName("label_7")
        self.horizontalLayout_6.addWidget(self.label_7)
        self.lineEdit_4 = QtWidgets.QLineEdit(Form)
        self.lineEdit_4.setEnabled(False)
        self.lineEdit_4.setObjectName("lineEdit_4")
        self.horizontalLayout_6.addWidget(self.lineEdit_4)
        self.verticalLayout.addLayout(self.horizontalLayout_6)
        self.horizontalLayout_5 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_5.setObjectName("horizontalLayout_5")
        self.label_8 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_8.setFont(font)
        self.label_8.setObjectName("label_8")
        self.horizontalLayout_5.addWidget(self.label_8)
        self.lineEdit_5 = QtWidgets.QLineEdit(Form)
        self.lineEdit_5.setEnabled(False)
        self.lineEdit_5.setObjectName("lineEdit_5")
        self.horizontalLayout_5.addWidget(self.lineEdit_5)
        self.verticalLayout.addLayout(self.horizontalLayout_5)
        self.horizontalLayout_4 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_4.setObjectName("horizontalLayout_4")
        self.label_9 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_9.setFont(font)
        self.label_9.setObjectName("label_9")
        self.horizontalLayout_4.addWidget(self.label_9)
        self.lineEdit_6 = QtWidgets.QLineEdit(Form)
        self.lineEdit_6.setEnabled(False)
        self.lineEdit_6.setObjectName("lineEdit_6")
        self.horizontalLayout_4.addWidget(self.lineEdit_6)
        self.verticalLayout.addLayout(self.horizontalLayout_4)
        self.horizontalLayout_3 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_3.setObjectName("horizontalLayout_3")
        self.label_10 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_10.setFont(font)
        self.label_10.setObjectName("label_10")
        self.horizontalLayout_3.addWidget(self.label_10)
        self.lineEdit_7 = QtWidgets.QLineEdit(Form)
        self.lineEdit_7.setEnabled(False)
        self.lineEdit_7.setObjectName("lineEdit_7")
        self.horizontalLayout_3.addWidget(self.lineEdit_7)
        self.verticalLayout.addLayout(self.horizontalLayout_3)
        self.horizontalLayout_2 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_2.setObjectName("horizontalLayout_2")
        self.label_11 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_11.setFont(font)
        self.label_11.setObjectName("label_11")
        self.horizontalLayout_2.addWidget(self.label_11)
        self.lineEdit_2 = QtWidgets.QLineEdit(Form)
        self.lineEdit_2.setEnabled(False)
        self.lineEdit_2.setObjectName("lineEdit_2")
        self.horizontalLayout_2.addWidget(self.lineEdit_2)
        self.verticalLayout.addLayout(self.horizontalLayout_2)
        self.horizontalLayout_10 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_10.setObjectName("horizontalLayout_10")
        self.label_3 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_3.setFont(font)
        self.label_3.setObjectName("label_3")
        self.horizontalLayout_10.addWidget(self.label_3)
        self.lineEdit_9 = QtWidgets.QLineEdit(Form)
        self.lineEdit_9.setEnabled(False)
        self.lineEdit_9.setObjectName("lineEdit_9")
        self.horizontalLayout_10.addWidget(self.lineEdit_9)
        self.verticalLayout.addLayout(self.horizontalLayout_10)
        self.horizontalLayout_11 = QtWidgets.QHBoxLayout()
        self.horizontalLayout_11.setObjectName("horizontalLayout_11")
        self.label_4 = QtWidgets.QLabel(Form)
        font = QtGui.QFont()
        font.setPointSize(12)
        self.label_4.setFont(font)
        self.label_4.setObjectName("label_4")
        self.horizontalLayout_11.addWidget(self.label_4)
        self.lineEdit_10 = QtWidgets.QLineEdit(Form)
        self.lineEdit_10.setEnabled(False)
        self.lineEdit_10.setObjectName("lineEdit_10")
        self.horizontalLayout_11.addWidget(self.lineEdit_10)
        self.verticalLayout.addLayout(self.horizontalLayout_11)
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.pushButton_9 = QtWidgets.QPushButton(Form)
        self.pushButton_9.setObjectName("pushButton_9")
        self.horizontalLayout.addWidget(self.pushButton_9)
        self.pushButton_7 = QtWidgets.QPushButton(Form)
        self.pushButton_7.setObjectName("pushButton_7")
        self.horizontalLayout.addWidget(self.pushButton_7)
        self.verticalLayout.addLayout(self.horizontalLayout)

        self.retranslateUi(Form)
        QtCore.QMetaObject.connectSlotsByName(Form)

    def retranslateUi(self, Form):
        _translate = QtCore.QCoreApplication.translate
        Form.setWindowTitle(_translate("Form", "Form"))
        self.label_5.setText(_translate("Form", "Морфологический анализ"))
        self.label_6.setText(_translate("Form", "Слово"))
        self.label_2.setText(_translate("Form", "Часть речи"))
        self.label_7.setText(_translate("Form", "Корень"))
        self.label_8.setText(_translate("Form", "Приставка"))
        self.label_9.setText(_translate("Form", "Окончание"))
        self.label_10.setText(_translate("Form", "Суффикс"))
        self.label_11.setText(_translate("Form", "МФА"))
        self.label_3.setText(_translate("Form", "Число"))
        self.label_4.setText(_translate("Form", "Род"))
        self.pushButton_9.setText(_translate("Form", "Применить"))
        self.pushButton_7.setText(_translate("Form", "Загрузить в язык"))

class Text_Form(object):
    def setupUi(self, Form):
        Form.setObjectName("Form")
        Form.resize(478, 243)
        self.label = QtWidgets.QLabel(Form)
        self.label.setGeometry(QtCore.QRect(10, 10, 461, 31))
        font = QtGui.QFont()
        font.setPointSize(18)
        self.label.setFont(font)
        self.label.setObjectName("label")
        self.lineEdit = QtWidgets.QLineEdit(Form)
        self.lineEdit.setGeometry(QtCore.QRect(10, 50, 451, 111))
        self.lineEdit.setAlignment(QtCore.Qt.AlignLeading|QtCore.Qt.AlignLeft|QtCore.Qt.AlignTop)
        self.lineEdit.setObjectName("lineEdit")
        self.label_2 = QtWidgets.QLabel(Form)
        self.label_2.setGeometry(QtCore.QRect(10, 200, 121, 21))
        self.label_2.setObjectName("label_2")
        self.pushButton = QtWidgets.QPushButton(Form)
        self.pushButton.setGeometry(QtCore.QRect(170, 170, 111, 23))
        self.pushButton.setObjectName("pushButton")

        self.retranslateUi(Form)
        QtCore.QMetaObject.connectSlotsByName(Form)

    def retranslateUi(self, Form):
        _translate = QtCore.QCoreApplication.translate
        Form.setWindowTitle(_translate("Form", "Form"))
        self.label.setText(_translate("Form", "                   Анализ текста."))
        self.label_2.setText(_translate("Form", "Количество слов:"))
        self.pushButton.setText(_translate("Form", "Провести анализ"))

class Change:
    def __init__(self, uid, body, period):
        self.uid = uid
        self.body = body
        self.period = period

class Morpheme:
    def __init__(self, body, _type, part):
        self.body = body
        self._type = _type
        self.part = part

class Language:
    def __init__(self, name):
        self.name = name
        self.roots = []
        self.morphemes = []
        self.morphemes_bodies = []
        self.prefixes = []
        self.suffixes = []
        self.endings_gender = {}
        self.changes = []
        self.alphabet = {}

    def addMorpheme(self, morpheme, _type, part):
        if Morpheme(morpheme, _type, part) not in self.morphemes:
            self.morphemes.append(Morpheme(morpheme, _type, part))
        if _type[:6] == "ending" and len(_type) > 6:
            self.endings_gender[morpheme] = _type[7:11] + ":" + _type[12:]

    def add_change(self, change):
        if Change(change[0], change[1], change[2]) not in self.changes:
            self.changes.append(Change(change[0], change[1], change[2]))

    def add_letter(self, letter):
        if letter[1] not in self.alphabet.keys():
            self.alphabet[letter[1]] = letter[2]

class FirstSep(fs.Ui_MainWindow, QMainWindow):
    def __init__(self):
        super().__init__()
        self.setupUi(self)

class SecondSep(Ui_MainWindow, QMainWindow):
    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.base_connection = sqlite3.connect("DataBases\\Test.sqlite")
        self.pushButton.clicked.connect(self.deleteRow)
        self.pushButton_2.clicked.connect(self.getTable)
        self.pushButton_3.clicked.connect(lambda: self.addRow(1, [self.tableWidget.rowCount() + 1]))
        self.pushButton_4.clicked.connect(self.save)
        self.pushButton_5.clicked.connect(self.sort_byId)
        self.pushButton_6.clicked.connect(self.unload_table)
        self.pushButton_7.clicked.connect(self.show_alg)
        self.pushButton_8.clicked.connect(self.quit)
        self.pushButton_10.clicked.connect(self.loadLanguage)
        self.action.triggered.connect(self.getTable)
        self.action_2.triggered.connect(self.unload_table)
        self.action_id.triggered.connect(self.sort_byId)
        self.action_6.triggered.connect(self.addRow)
        self.action_7.triggered.connect(self.deleteRow)
        self.action_4.triggered.connect(self.save)
        self.action_5.triggered.connect(self.loadLanguage)
        self.action_8.triggered.connect(self.show_alg)
        self.action_10.triggered.connect(self.quit)
        self.tableWidget.itemChanged.connect(self.change_item)
        self.modified = {}
        self.titles = []
        self.base_name = ""
        self.table_name = ""
        self.adding_row = False
        self.deleting_row = False
        self.new_table = False
        self.algorithm = None
        self.text = None
        self.language = None

    def setLanguage(self):
        self.language = Language("Test")

    def loadLanguage(self):
        self.setLanguage()
        cursor = self.base_connection.cursor()
        select_alphabet = cursor.execute("SELECT * FROM Alphabet")
        for ltr in select_alphabet:
            self.language.add_letter(ltr)
        select_morphemes = cursor.execute("SELECT body, type, part FROM Morphemes")
        for morpheme in select_morphemes:
            self.language.addMorpheme(morpheme[0], morpheme[1], morpheme[2])

    def getTable(self):
        self.setLanguage()
        self.table_name = self.lineEdit_2.text()
        if not self.table_name:
            self.label_18.setText("Вы ничего не ввели в поле")
            return -1
        cursor = self.base_connection.cursor()
        cursor.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='" + self.table_name + "'")
        if cursor.fetchone()[0] == 0:
            self.label_18.setText("Таблицы с таким названием нет")
            return -1
        self.label_18.setText("")
        select_table = cursor.execute("SELECT * FROM " + self.table_name)
        self.new_table = True
        self.tableWidget.setRowCount(0)
        columns_names = list(map(lambda x: x[0], cursor.description))
        self.tableWidget.setColumnCount(len(columns_names))
        self.tableWidget.setHorizontalHeaderLabels(columns_names)
        for e in select_table:
            self.tableWidget.itemChanged.connect(self.change_item)
            self.addRow(len(columns_names), e)
        self.new_table = False
        self.titles = [description[0] for description in cursor.description]

    def addRow(self, column_count, row_cells):
        if self.base_connection is None:
            QMessageBox.warning(self, 'Ошибка', "Таблица не открыта", QMessageBox.Ok)
            return -1
        self.adding_row = True
        row_count = self.tableWidget.rowCount()
        self.tableWidget.insertRow(row_count)
        for e in range(column_count):
            self.tableWidget.setItem(row_count, e, QTableWidgetItem(str(row_cells[e])))
        self.adding_row = False

    def change_item(self, item):
        if self.titles and not self.new_table:
            if self.adding_row:
                self.modified["addRow" + item.text()] = item.text()
            elif self.deleting_row:
                self.modified["delRow"] = item.text()
            else:
                self.modified[self.titles[item.column()] + ":" + str(item.row() + 1)] = item.text()

    def save(self):
        if not self.lineEdit_2.text():
            QMessageBox.warning(self, 'Ошибка', "Таблица не открыта", QMessageBox.Ok)
            return -1
        if self.modified:
            valid = QMessageBox.question(
                self, 'Сохранение', "Вы уверены, что хотите сохранить изменения?",
                QMessageBox.Yes, QMessageBox.No)
            if valid == QMessageBox.Yes:
                print(self.modified)
                cursor = self.base_connection.cursor()
                titles_str = " ("
                titles_str += ",".join(i for i in self.titles)
                titles_str += ") "
                for i, v in self.modified.items():
                    if i[:6] == "addRow":
                        values = "VALUES(" + v + ","
                        for title in range(len(self.titles) - 2):
                            values += "'',"
                        values += "'')"
                        cursor.execute("INSERT INTO " + self.table_name + titles_str + values)
                    elif i == "delRow":
                        cursor.execute("DELETE FROM " + self.table_name + " WHERE id = ?", (v,))
                    else:
                        cursor.execute("UPDATE " + self.table_name + " SET '" + i[:i.find(":")] +
                                       "' = ? WHERE id = ?", (v, i[i.find(":") + 1:]))
                    self.base_connection.commit()
                self.modified = {}
        else:
            QMessageBox.warning(self, 'Ошибка', "Изменений не обнаружено", QMessageBox.Ok)
            return -1

    def deleteRow(self):
        if self.tableWidget.currentRow() == -1:
            QMessageBox.warning(self, 'Ошибка', "Вы не выбрали строку", QMessageBox.Ok)
            return -1
        if self.titles:
            deleting = QMessageBox.question(
                self, 'Удаление', "Вы уверены, что хотите удалить выбранную строку?",
                QMessageBox.Yes, QMessageBox.No)
            if deleting == QMessageBox.Yes:
                self.deleting_row = True
                current_row = self.tableWidget.currentRow()
                self.modified["delRow"] = self.tableWidget.item(current_row, 0).text()
                self.tableWidget.removeRow(self.tableWidget.currentRow())
                self.deleting_row = False
        else:
            QMessageBox.warning(self, 'Ошибка', "Таблица не открыта", QMessageBox.Ok)
            return -1

    def sort_byId(self):
        for row in range(self.tableWidget.rowCount()):
            self.modified["id:" + self.tableWidget.item(row, 0).text()] = str(row + 1)
            self.tableWidget.setItem(row, 0, QTableWidgetItem(str(row + 1)))

    def unload_table(self):
        if self.base_connection is None:
            QMessageBox.warning(self, 'Ошибка', "Таблица не открыта", QMessageBox.Ok)
            return -1
        self.tableWidget.clear()
        self.tableWidget.setRowCount(0)
        self.tableWidget.setColumnCount(0)
        self.table_name = ""
        self.base_name = ""

    def quit(self):
        does_quit = QMessageBox.question(
            self, 'Выход', "Вы уверены, что хотите выйти?",
            QMessageBox.Yes, QMessageBox.No)
        if does_quit == QMessageBox.Yes:
            if self.base_connection:
                self.base_connection.close()
                self.base_connection = None
            sys.exit(app.exec())

class Russian(SecondSep):
    def __init__(self):
        super().__init__()
        self.pushButton_7.clicked.connect(self.show_alg)
        self.pushButton_9.clicked.connect(self.show_text)
        self.base_connection = sqlite3.connect("DataBases\\Russian.sqlite")
        self.loadLanguage()

    def setLanguage(self):
        self.language = Language("Russian")

    def loadLanguage(self):
        self.setLanguage()
        cursor = self.base_connection.cursor()
        select_morphemes = cursor.execute("SELECT body, type, part FROM Morphemes")
        for morpheme in select_morphemes:
            self.language.addMorpheme(morpheme[0], morpheme[1], morpheme[2])
        select_alphabet = cursor.execute("SELECT * FROM Alphabet")
        for ltr in select_alphabet:
            self.language.add_letter(ltr)

    def check_if_letter(self, string):
        for ltr in self.language.alphabet.keys():
            if ltr in string:
                return True
        return False

    def word_alg(self, word, part):
        ending = ""
        suffix = ""
        prefix = ""
        gender = "Нет"
        form = "Нет"
        do_break = False
        for ltr in word[::-1]:
            ending += ltr
            for morpheme in self.language.morphemes:
                if ending[::-1] == morpheme.body and morpheme._type[:6] == "ending" and morpheme.part == part:
                    ending = ending[::-1]
                    if morpheme.part in ("noun", "adjective"):
                        gender = self.endings_gender[ending][:4]
                        form = self.endings_gender[ending][5:]
                    do_break = True
                    break
            if do_break:
                break
        do_break = False
        if ending == word[::-1]:
            ending = "Нулевое"
            end_gap = 0
        else:
            end_gap = len(ending)
        for ltr in word[-1 - end_gap::-1]:
            suffix += ltr
            for morpheme in self.language.morphemes:
                if suffix[::-1] == morpheme.body and morpheme._type == "suffix" and morpheme.part == part:
                    suffix = suffix[::-1]
                    do_break = True
                    break
            if do_break:
                break
        if suffix == word[-1 - end_gap::-1]:
            suffix = "Нет"
            suf_gap = 0
        else:
            suf_gap = len(suffix)
        do_break = False
        for ltr in word:
            prefix += ltr
            for morpheme in self.language.morphemes:
                if prefix == morpheme.body and morpheme._type == "prefix":
                    do_break = True
                    break
            if do_break:
                break
        if prefix == word:
            prefix = "Нет"
            pre_gap = 0
        else:
            pre_gap = len(prefix)
        root = word[pre_gap:len(word) - suf_gap - end_gap]
        ipa_word = "["
        for letter in word.lower():
            for k, v in self.alphabet.items():
                if k == letter:
                    ipa_word += v
        return [root, prefix, suffix, ending, ipa_word + "]", form, gender]

    def show_alg(self):
        if not self.algorithm:
            self.algorithm = Algorithm()
        self.algorithm.show()

    def show_text(self):
        if not self.text:
            self.text = Text()
        self.text.show()

class Algorithm(Alg_Form, QWidget):
    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.pushButton_7.clicked.connect(self.load_to_lang)
        self.pushButton_9.clicked.connect(self.apply)
        self.language = None

    def load_to_lang(self):
        connection = sqlite3.connect("DataBases\\Russian.sqlite")
        cursor = connection.cursor()
        w_id = len(cursor.execute("SELECT * FROM Words").fetchall()) + 1
        word = self.lineEdit_3.text()
        part = self.lineEdit_8.text()
        root = self.lineEdit_4.text()
        prefix = self.lineEdit_5.text()
        ending = self.lineEdit_6.text()
        suffix = self.lineEdit_7.text()
        ipa = self.lineEdit_2.text()
        form = self.lineEdit_9.text()
        gender = self.lineEdit_10.text()
        cursor.execute("INSERT INTO Words(id,word,partofspeech,ipa,root,prefix,suffix,ending,gender,form) VALUES"
                       " (?,?,?,?,?,?,?,?,?,?)", (w_id, word, part, ipa, root, prefix, suffix, ending, gender,
                                                  form)).fetchall()
        connection.commit()
        self.lineEdit_4.setDisabled(True)
        self.lineEdit_5.setDisabled(True)
        self.lineEdit_6.setDisabled(True)
        self.lineEdit_7.setDisabled(True)
        self.lineEdit_2.setDisabled(True)
        self.lineEdit_9.setDisabled(True)
        self.lineEdit_10.setDisabled(True)

    def apply(self):
        part = self.lineEdit_8.text()
        word = self.lineEdit_3.text()
        morphemes = word_alg(word, part)
        self.lineEdit_4.setText(morphemes[0])
        self.lineEdit_5.setText(morphemes[1])
        self.lineEdit_6.setText(morphemes[3])
        self.lineEdit_7.setText(morphemes[2])
        self.lineEdit_2.setText(morphemes[4])
        self.lineEdit_9.setText(morphemes[5])
        self.lineEdit_10.setText(morphemes[6])
        self.lineEdit_4.setDisabled(False)
        self.lineEdit_5.setDisabled(False)
        self.lineEdit_6.setDisabled(False)
        self.lineEdit_7.setDisabled(False)
        self.lineEdit_2.setDisabled(False)
        self.lineEdit_9.setDisabled(False)
        self.lineEdit_10.setDisabled(False)

class Text(Text_Form, QWidget):
    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.pushButton.clicked.connect(self.text_analysis)

    def text_analysis(self):
        text = self.lineEdit.text().split()
        words_count = 0
        for word in text:
            if ex.check_if_letter(word):
                words_count += 1
        self.label_2.setText("Количество слов: " + str(words_count))

sys._excepthook = sys.excepthook

def exception_hook(exctype, value, traceback):
    sys._excepthook(exctype, value, traceback)
    sys.exit(1)

languages = []
sys.excepthook = exception_hook
if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = Russian()
    ex.show()
    sys.exit(app.exec())


import datetime
import random
from HW_module_4_task3 import text_decomposition_for_correctness as normalize
import os

class PrintMessage:
    def __init__(self, message):
        self.message = message

    # create a method which inserts the formatted text into text file
    def print_message(self):
        ptf = open("Module6_paste.txt", "a")      # open file to add
        print(self.message, file=ptf)
        ptf.close()                            # close file

class News:
    def __init__(self, news_msg, location):
        self.news_msg = news_msg               # news_info
        self.location = location               # location where it has happened

    # create a method which creates a news (text, city, time)
    def news_message(self):
        message = f'News ----------------------------\n{self.news_msg}\n{self.location},' \
                  f'{datetime.datetime.now().strftime("%d/%m/%y %H:%M")}\n' \
                  f'---------------------------------\n\n'
        PrintMessage(normalize(message)).print_message()

    def message_list(self):
        d = []
        d.append('News ----------------------------\n')
        d.append(self.news_msg + "\n")
        d.append(self.location + ', ' + datetime.datetime.now().strftime("%d/%m/%y %H:%M") + "\n")
        d.append('---------------------------------\n\n')
        return d

class Advertising:
    def __init__(self, adv_message, actual_until=None):
        self.adv_message = adv_message
        self.actual_until = actual_until

    # create a method which creates an advertisement (text, expiration day, how many days left)
    def advertising(self):
        message = f'Private Ad -----------------------\n{self.adv_message}\n' \
                  f'Actual until: {self.actual_until},' \
                  f'{(datetime.datetime.strptime(self.actual_until, "%d/%m/%y") - datetime.datetime.now()).days}days left\n' \
                  f'----------------------------------\n\n'
        PrintMessage(normalize(message)).print_message()

    def message_list(self):
        d = []
        d.append('Private Ad -----------------------\n')
        d.append(self.adv_message + "\n")
        d.append("Actual until: " + self.actual_until + ', '
                 + str((datetime.datetime.strptime(self.actual_until, "%d/%m/%y") - datetime.datetime.now()).days)
                 + " days left" + "\n")
        d.append('----------------------------------\n\n')
        return d

class Guess:
    def __init__(self, guessing):
        self.guessing = guessing

    # create an interactive method with indicating possibility using random elements from list
    def ask_future(self):
        test_list = ['It is certain', 'It is decidedly so', 'Without a doubt', 'Yes – definitely', 'As I see it, yes',
                        'Most likely', 'Ask again later', 'Better not tell you now', 'Cannot predict now',
                        'Don\'t count on it', 'My reply is no', 'My sources say no', 'Very doubtful']
        random_test_list = random.randint(0, len(test_list) - 1)
        answer = test_list[random_test_list]
        prediction = f'Ask me about your future? ----------------\n' \
                     f'Your question - "{self.guessing}",\n' \
                     f'Witch\'s answer will be - {answer}\n' \
                     f'----------------------------------\n\n'
        PrintMessage(normalize(prediction)).print_message()
        return answer

    def message_list(self):
        answer = self.ask_future()
        d = []
        d.append('Ask me about your future? ----------------\n')
        d.append('Your question - ' + self.guessing + "\n")
        d.append('Witch answer will be - ' + answer + '\n')
        d.append('----------------------------------\n\n')
        return d

class FromAnotherSource:

    # create a method which makes another directories and reads a file from that directory OR use the default one
    def read_file(self):
        original_path = os.path.dirname(os.path.realpath(__file__))
        print(f'Your default directory is "{os.getcwd()}"')
        is_true = True
        while is_true:
            answer = int(input(
                f'If you want to ingest your file from default directory - enter 1, if you want to change it - enter 2: '))
            if answer == 1:
                while True:
                    try:
                        file_name = input('Enter the file name with its format: ')
                        with open(file_name, "r", encoding='utf-8') as source:
                            f_contents = source.readlines()
                        print('Okay, such file exists')
                        path_for_remove = str(file_name)
                        is_true = False
                        break
                    except FileNotFoundError:
                        print('No file with such name. Try again')
                    except PermissionError:
                        print('No file with such name. Try again')
            elif answer == 2:
                while True:
                    try:
                        change_path = input('Enter the file path: ')
                        if not os.path.isdir(change_path):
                            os.mkdir(change_path)
                        os.chdir(change_path)
                        print(f'Now you directory is "{os.getcwd()}"')
                        break
                    except SyntaxError:
                        print('You made a mistake. Try again')
                    except FileNotFoundError:
                        print('You made a mistake. Try again')
                while True:
                    try:
                        file_name = input('Enter the file name with its format: ')
                        with open(file_name, "r", encoding='utf-8') as source:
                            f_contents = source.readlines()
                        print('Such file exists')
                        path_for_remove = os.path.join(str(change_path), str(file_name))
                        os.chdir(original_path)
                        is_true = False
                        break
                    except FileNotFoundError:
                        print('No file with such name. Try again')
                    except PermissionError:
                        print('No file with such name. Try again')
            else:
                print('Try again here')
        num_records = int(input("Enter records num:\n"))
        count = 0
        print('f_contest - ', f_contents)
        for i, line in enumerate(f_contents):
            if line == '\n':
                count += 1
                if count == 2 * num_records:
                    f_contents_new = f_contents[:i + 1]
                    break

        separator = '\n'      # creating list of lists
        f_contents_new_fix = []
        sublist = []
        for item in f_contents_new:
            if item == separator:
                f_contents_new_fix.append(sublist)
                sublist = []
            else:
                sublist.append(item)
        f_contents_new_fix.append(sublist)
        return f_contents_new, f_contents_new_fix, path_for_remove

# ask a user what data he wants to print and then call a class and insert the data into file using inserting method
if __name__ == "__main__":
    while True:
        print('Please enter your choice:', '1 - News', '2 - Private Ad', '3 - Divination','4 - Add data from another file', '5 - Nothing', sep='\n')
        flag = input('Choose the appropriate number: ')
        if flag == '1':
            news = News(input('Please enter news text\n'),
                        input('Please enter location\n'))
            news_mess = news
            news_mess.news_message()
        elif flag == '2':
            advng = Advertising(input('Please enter advertisement text\n'),
                                input('Please enter expire date in the format dd/mm/yy\n'))
            adv_message = advng
            adv_message.advertising()
        elif flag == '3':
            guessing = Guess(input('Ask me about your near future\n'))
            question_divination = guessing
            question_divination.ask_future()
        elif flag == '4':
            f_contents_new, path_for_remove = FromAnotherSource().read_file()
            with open("Module6_paste.txt", "a", encoding='utf-8') as file:
                file.writelines(f_contents_new)
            print(f'This file {path_for_remove} will be removed now\n')
            os.remove(path_for_remove)
        elif flag == '5':
            print('That is all.')
            break
        else:
            print('Try again')


import os
import csv
import pickle as pkl
from sklearn.model_selection import train_test_split

def import_ribo_dataset(
    ribo_data_dirpath: str, train_size: float = 0.7, random_state: int = 42
):
    with open(os.path.join(ribo_data_dirpath, "raw_counts.csv"), "r") as read_obj:
        counts = list(csv.reader(read_obj))
    with open(os.path.join(ribo_data_dirpath, "raw_sequences.csv"), "r") as read_obj:
        sequences = list(csv.reader(read_obj))

    return train_test_split(
        sequences, counts, train_size=train_size, random_state=random_state
    )

def import_v_data():
    with open(
        os.path.join(os.environ["V_DATA_DIRPATH"], "CTRL_train_feats.pkl"), "rb"
    ) as fname:
        train = pkl.load(fname)
    with open(
        os.path.join(os.environ["V_DATA_DIRPATH"], "CTRL_val_feats.pkl"), "rb"
    ) as fname:
        test = pkl.load(fname)

    seq_train, seq_test, counts_train, counts_test = [], [], [], []
    for _, val in train.items():
        seq_train.append(val[0])
        counts_train.append(val[1])

    for _, val in test.items():
        seq_test.append(val[0])
        counts_test.append(val[1])

    return seq_train, seq_test, counts_train, counts_test


from casadi import *
from acados_template import AcadosModel, AcadosOcp, AcadosOcpSolver
import numpy as np
import scipy.linalg
import os
from ament_index_python.packages import get_package_share_directory

def ev_track_ode_model(mass, camera_transform, desired_depth, desired_area):

    model_name = 'ev_drone_track'

    # constants
    m = mass
    g = 9.81 # gravity constant [m/s^2]
    cRb = camera_transform  # fixed transformation between drone's body frame to camera frame

    # constants related to image features
    z = desired_depth
    a = desired_area

    # set up states & controls
    xn = SX.sym('xn')      # visual features (chosen from image moments)
    yn = SX.sym('yn')
    an = SX.sym('an')
    qw = SX.sym('qw')      # quaternion
    qx = SX.sym('qx')
    qy = SX.sym('qy')
    qz = SX.sym('qz')
    vx = SX.sym('vx')      # body-frame linear velocity
    vy = SX.sym('vy')
    vz = SX.sym('vz')

    x = vertcat(xn, yn, an, qw, qx, qy, qz, vx, vy, vz)

    # controls
    f      = SX.sym('f')       # thrust
    omegax = SX.sym('omegax')  # angular rates
    omegay = SX.sym('omegay')
    omegaz = SX.sym('omegaz')

    u = vertcat(f, omegax, omegay, omegaz)

    # xdot
    xn_dot = SX.sym('xn_dot')
    yn_dot = SX.sym('yn_dot')
    an_dot = SX.sym('an_dot')
    qw_dot = SX.sym('qw_dot')
    qx_dot = SX.sym('qx_dot')
    qy_dot = SX.sym('qy_dot')
    qz_dot = SX.sym('qz_dot')
    vx_dot = SX.sym('vx_dot')
    vy_dot = SX.sym('vy_dot')
    vz_dot = SX.sym('vz_dot')

    xdot = vertcat(xn_dot, yn_dot, an_dot, qw_dot, qx_dot, qy_dot, qz_dot, vx_dot, vy_dot, vz_dot)

    # algebraic variables
    # z = None

    # parameters
    p = []

    # dynamics
    f_expl = vertcat(
        -1*(cRb[0,0]*vx + cRb[0,1]*vy + cRb[0,2]*vz),
        -1*(cRb[1,0]*vx + cRb[1,1]*vy + cRb[1,2]*vz),
        -1*(cRb[2,0]*vx + cRb[2,1]*vy + cRb[2,2]*vz),
        0.5*(-qx*omegax - qy*omegay - qz*omegaz ), # quaternion derivatives: q_dot = 1/2*q[*]omega, [*]: quaternion multiplication
        0.5*( qw*omegax + qy*omegaz - qz*omegay ),
        0.5*( qw*omegay + qz*omegax - qx*omegaz ),
        0.5*( qw*omegaz + qx*omegay - qy*omegax ),
        -2*(qx*qz - qw*qy)*g,                       # acceleration: a = f/m - R^T*[0; 0; g]
        -2*(qy*qz + qw*qx)*g,
        f/m - (1 - 2*(qx*qx + qy*qy))*g
    )

    f_impl = xdot - f_expl

    model = AcadosModel()

    model.f_impl_expr = f_impl
    model.f_expl_expr = f_expl
    model.x = x
    model.xdot = xdot
    model.u = u
    # model.z = z
    model.p = p
    model.name = model_name

    return model

def visual_tracking_ocp_settings(mass, camera_transform, desired_depth, desired_area, intrinsic_param, Tf, N, Q, R, x0, max_thrust, max_omega_xy, max_omega_z, max_v):

    # create ocp object
    ocp = AcadosOcp()

    # set model
    model = ev_track_ode_model(mass, camera_transform, desired_depth, desired_area)
    ocp.model = model

    # set dimensions
    nx = model.x.size()[0]
    nu = model.u.size()[0]
    ny = nx + nu
    ny_e = nx

    ocp.dims.N = N

    # set cost module
    ocp.cost.cost_type = 'LINEAR_LS'
    ocp.cost.cost_type_e = 'LINEAR_LS'

    cost_Q = np.diag(Q)
    cost_R = np.diag(R)
    ocp.cost.W = scipy.linalg.block_diag(cost_Q, cost_R)
    ocp.cost.W_e = cost_Q

    ocp.cost.Vx = np.zeros((ny, nx))   # state
    ocp.cost.Vx[:nx,:nx] = np.eye(nx)

    ocp.cost.Vu = np.zeros((ny, nu))   # controls
    ocp.cost.Vu[-4:,-4:] = np.eye(nu)

    ocp.cost.Vx_e = np.eye(nx)         # terminal state

    ocp.cost.yref  = np.zeros((ny, ))
    ocp.cost.yref_e = np.zeros((ny_e, ))

    # set constraints
    ocp.constraints.x0 = x0  # initial constraints

    ocp.constraints.lbu = np.array([0.1*max_thrust, -max_omega_xy, -max_omega_xy, -max_omega_z])
    ocp.constraints.ubu = np.array([0.9*max_thrust, max_omega_xy, max_omega_xy, max_omega_z])
    ocp.constraints.idxbu = np.array([0, 1, 2, 3]) # control constraints

    ocp.constraints.lbx = np.array([0, 0, -max_v, -max_v, -max_v])
    ocp.constraints.ubx = np.array([intrinsic_param.x_lim, intrinsic_param.y_lim, max_v, max_v, max_v])
    ocp.constraints.idxbx = np.array([0, 1, 7, 8, 9]) # state constraints

    # set QP solver and integration
    ocp.solver_options.tf = Tf
    ocp.solver_options.qp_solver = "PARTIAL_CONDENSING_HPIPM"
    ocp.solver_options.nlp_solver_type = "SQP_RTI"
    ocp.solver_options.hessian_approx = "GAUSS_NEWTON"
    ocp.solver_options.integrator_type = "ERK"
    ocp.solver_options.sim_method_num_stages = 4
    ocp.solver_options.sim_method_num_steps = 3
    ocp.solver_options.nlp_solver_max_iter = 200
    ocp.solver_options.tol = 1e-4

    # set acados export directory & model file
    ocp.code_export_directory = os.path.join(get_package_share_directory('visual_mpc'), 'acados_code')
    model_file = os.path.join(
        os.path.relpath(get_package_share_directory('visual_mpc'), os.curdir),
        model.name + "_" + "acados_ocp.json"
    )
    # create solver
    acados_solver = AcadosOcpSolver(ocp, json_file=model_file)

    return acados_solver


from .utils import set_env
# avoid numpy taking up more than one thread
with set_env(MKL_NUM_THREADS='1',
             NUMEXPR_NUM_THREADS='1',
             OMP_NUM_THREADS='1'):
    import numpy as np

# thanks to Francis Song for this function
# source: http://www.nervouscomputer.com/hfs/cmdscale-in-python/
def cmdscale(D):
    """Classical multidimensional scaling (MDS)

    Args:
        D (numpy.array)
            Symmetric distance matrix (n, n)

    Returns:
        Y (numpy.array)
            Configuration matrix (n, p). Each column represents a dimension. Only the
            p dimensions corresponding to positive eigenvalues of B are returned.
            Note that each dimension is only determined up to an overall sign,
            corresponding to a reflection.
        e (numpy.array)
            Eigenvalues of B (n, 1)
    """
    # Number of points
    n = len(D)

    # Centering matrix
    H = np.eye(n) - np.ones((n, n))/n

    # YY^T
    B = -H.dot(D**2).dot(H)/2

    # Diagonalize
    evals, evecs = np.linalg.eigh(B)

    # Sort by eigenvalue in descending order
    idx = np.argsort(evals)[::-1]
    evals = evals[idx]
    evecs = evecs[:, idx]

    # Compute the coordinates using positive-eigenvalued components only
    w, = np.where(evals > 0)
    L = np.diag(np.sqrt(evals[w]))
    V = evecs[:, w]
    Y = V.dot(L)

    return Y, evals[evals > 0]


import json
import tkinter as tk
from tkinter import ttk, font, messagebox
from pprint import pprint
import argparse
import subprocess

debug=0
simulation=0

class LabMenu(tk.Frame):
    def __init__(self, parent):
        """
        Initialize the Lab Selection window.
        :param parent: The parent Tkinter window
        """
        self.parent = parent
        self.labconf = self.get_conf(args.conf)

        self.parent.title("Lab Menu")
        self.numlabs = len(self.labconf["labs"])
        self.radioGroup = ttk.LabelFrame(self.parent, text="Lab Options")
        self.radioGroup.grid(row=0, columnspan=4)

        self.labSelection = tk.IntVar()

        lab_radio = []

        for num in range(self.numlabs):
            labname = 'Lab {}:  {}'.format(self.labconf["labs"][num]["id"], self.labconf["labs"][num]["title"])
            if debug:
                print("Labname is %s, num is %s" % (labname, num))
            lab_radio.append(tk.Radiobutton(self.radioGroup, text=labname, variable=self.labSelection, value=num, bg="gray90")
                                 .grid(sticky='w', row=num+1, column=0))

        self.buttonGroup = ttk.LabelFrame(self.parent, text="Buttons").grid(row=10, columnspan=4)
        self.lab_ok_button = tk.Button(self.buttonGroup, text="Ok", command=self.do_selection)
        self.lab_ok_button.grid(row=11, column=1)
        self.lab_cancel_button = tk.Button(self.buttonGroup, text="Quit", command=self.ask_quit)
        self.lab_cancel_button.grid(row=11, column=2)

    def get_conf(self, conf_file):
        """
        Get the Lab configuration details from the external configuration file.
        :param conf_file: A string of the filepath to the configuration file
        :return: conf: A hierarchical dictionary structure (with arrays) of the configuration
        """
        with open(conf_file) as file:
            conf=json.load(file)
            if debug:
                pprint(conf)
            return conf

    def ask_quit(self):
        """
        Do a graceful exit by asking the user if they would like to quit.
        :return: None
        """
        if tk.messagebox.askyesno("Quit", "Are you sure you want to quit now?"):
            root.destroy()

    def do_selection(self):
        """
        Execute on the chosen Lab Selection and fire off the appropriate scripts.
        :return: None
        """
        if debug:
            print ("Selected array entry %s" % self.labSelection.get())
        print (self.labconf["labs"][self.labSelection.get()]["id"])
        print (self.labconf["labs"][self.labSelection.get()]["title"])
        try:
            print (self.labconf["labs"][self.labSelection.get()]["scripts"])
            for num in range(len(self.labconf["labs"][self.labSelection.get()]["scripts"])):
                try:
                    result = subprocess.run(self.labconf["labs"][self.labSelection.get()]["scripts"][num], shell=True)
                    print (result)
                except subprocess.CalledProcessError as e:
                    print (e.returncode)
                    print (e.stderr)

        except KeyError:
            if debug:
                print ("No script array provided")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Show GoldenGate Lab Menu")
    parser.add_argument("--debug", help="Enable debugging output for troubleshooting", action='store_true')
    parser.add_argument("--simulation", help="Enable simulation mode which provides command execution output but not execution", action='store_true')
    parser.add_argument("--conf", help="Specify path for lab menu configuration")
    args = parser.parse_args()

    if args.debug:
        debug=1

    if args.simulation:
        simulation=1

    root = tk.Tk()
    LabMenu(root)
    root.mainloop()


from pyqtgraph.Qt import QtGui, QtCore
import numpy as np
import pyqtgraph as pg

app = QtGui.QApplication([]) # turn on a qt app
win = pg.GraphicsWindow(title="Basic plotting examples") # add a window to the app to hold the plot
win.resize(1000,600)
win.setWindowTitle('pyqtgraph example: Plotting')

# Enable antialiasing for prettier plots
pg.setConfigOptions(antialias=True)

p6 = win.addPlot(title="Updating plot")
curve = p6.plot(pen='y')
data = np.random.normal(size=(10,1000))
ptr = 0
def update():
    global curve, data, ptr, p6
    curve.setData(data[ptr%10])
    if ptr == 0:
        p6.enableAutoRange('xy', False)  ## stop auto-scaling after the first data set is plotted
    ptr += 1
timer = QtCore.QTimer()
timer.timeout.connect(update)
timer.start(50)

## Start Qt event loop unless running in interactive mode or using pyside.
if __name__ == '__main__':
    QtGui.QApplication.instance().exec_()


import imageio.v2 as imageio
import os

# Create an empty list to store the images
images = []

# Load all the PNGs in the folder
for file_name in sorted(os.listdir('/g/data/er8/lightning/georgia/Bourke_radar')):
    if file_name.endswith('.png'):
        file_path = os.path.join('/g/data/er8/lightning/georgia/Bourke_radar', file_name)
        images.append(imageio.imread(file_path))

# Save the images as a GIF
imageio.mimsave('Bourke.gif', images, fps=2)


from aiohttp import HttpVersion
from aiohttp.http_parser import RawRequestMessage
from multidict import CIMultiDict, CIMultiDictProxy
from yarl import URL

from .lambda_request_type import AwsLambdaRequest

def build_message(event: AwsLambdaRequest) -> RawRequestMessage:
    context = event['requestContext']
    http = context['http']
    version = http['protocol'].lstrip('HTTP/').split('.')
    http_version = HttpVersion(int(version[0]), int(version[1]))
    url = URL.build(scheme='http',
                    host=context['domainName'],
                    path=http['path'],
                    query_string=event['rawQueryString'])
    headers_ = CIMultiDict(event['headers'])

    return RawRequestMessage(
        method=http['method'].upper(),
        path=http['path'],
        version=http_version,
        headers=CIMultiDictProxy(headers_),
        raw_headers=tuple((k.encode(), v.encode()) for k, v in event['headers'].items()),
        should_close=False,
        compression=None,
        upgrade=False,
        chunked=False,
        url=url,
    )


import argparse
from difflib import SequenceMatcher
from itertools import cycle
from itertools import islice
import json
import numpy as np
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import sys

# Parse the JSON log and extract patterns
# Usage:
#    python extract_patterns_json.py json_file requested_ranks
#                                    requested_execution_time [options]
#    Options: -o output file (default: output.json)
#             -v verbose on (default: False)
#             -i individual patterns on (defaut: False)
# The individual patterns options allows to repeat patterns that occur only
# once (for which interpolation will fail).

individual_patterns = False
verbose = False

def string_distance(a, b):
    return SequenceMatcher(None, a, b).ratio()

def roundrobin(iterables):
    """Yields an item from each iterable, alternating between them.

        >>> list(roundrobin(['ABC', 'D', 'EF']))
        ['A', 'D', 'E', 'B', 'F', 'C']

    This function produces the same output as :func:`interleave_longest`, but
    may perform better for some inputs (in particular when the number of
    iterables is small).

    """
    # Recipe credited to George Sakkis
    pending = len(iterables)
    nexts = cycle(iter(it).__next__ for it in iterables)
    while pending:
        try:
            for next in nexts:
                yield next()
        except StopIteration:
            pending -= 1
            nexts = cycle(islice(nexts, pending))

# Populate the log with a start and end flag to force a min of two clusters
def add_edges(events_time, events_type):
    min_ts = min([min(events_time[rank]) for rank in events_time])
    max_ts = max([max(events_time[rank]) for rank in events_time])
    for rank in events_time:
        if max_ts - events_time[rank][0] < events_time[rank][0] - min_ts:
            # Populate with a START flag at the beginning of the execution
            events_time[rank].insert(0, min_ts)
            events_type[rank].insert(0, ".DEBUGSTART")
        if max_ts - events_time[rank][-1] > events_time[rank][-1] - min_ts:
            # Populate with an END flag at the end of the execution
            events_time[rank].append(max_ts)
            events_type[rank].append(".DEBUGEND")
    return {rank: [(i - min_ts) / 1000000 for i in events_time[rank]]
            for rank in events_time}, events_type

# Read the json file and keep events timestamp and type
def read_json_data(file_name):
    inf = open(file_name, "r")
    events_time = {}
    events_type = {}
    json_data = json.load(inf)
    for entry in json_data:
        if entry["name"][0] == ".":
            continue
        if (entry["pid"], entry["tid"]) not in events_time:
            events_time[(entry["pid"], entry["tid"])] = []
            events_type[(entry["pid"], entry["tid"])] = []
        events_time[(entry["pid"], entry["tid"])].append(entry["ts"])
        events_type[(entry["pid"], entry["tid"])].append(entry["name"])
    inf.close()
    return add_edges(events_time, events_type)

# update labels based on the new cluster split
def create_labels(labels, split_index):
    new_label = max(labels) + 1
    current_label = labels[split_index]
    for i in range(split_index+1, len(labels)):
        if labels[i] != current_label:
            break
        labels[i] = new_label
    return labels

# Criterias to terminate the clustering before reaching individual
# events. Currently, the max delay between events needs to be over 3 seconds
def cannot_split_clusters(labels, max_delay):
    if len(labels) == max(labels) - 1:
        return False
    if max_delay > 3:
        return False
    return True

# Hierarchical clustering for the events series
def find_clusters(events_time, end_marker=False):
    events_delay = [(events_time[i] - events_time[i-1])
                    for i in range(1,len(events_time))]
    labels = [0] * len(events_time)
    # if end marker exists
    if end_marker: # the first split is to seprate the end event from the rest
        labels[-1] = 1
    best_clusters = [0, labels]
    if len(labels) == 1:
        return labels
    while len(events_delay) > 0:
        # Split clusters on the largerst delay between events
        max_delay = max([events_delay[i] for i in range(len(events_delay))
                         if labels[i]==labels[i+1]])
        split_index = next(i for i in range(len(events_delay))
                       if events_delay[i]==max_delay)
        if verbose:
            print("[dbg] Split between timestamp", events_time[split_index],
              events_time[split_index+1])
        # if the clusters start computing the silhouette, update the value for
        # the current selection (unless this is the first split)
        if max_delay <= 60 and best_clusters[0] == 0 and max(labels) > 0:
            silhouette_avg = silhouette_score(events_time.reshape(-1, 1), labels)
            best_clusters = [silhouette_avg, labels[:]]
            if verbose:
                print("[dbg] Store current cluster with a score of",
                      silhouette_avg)
        labels = create_labels(labels, split_index)
        if cannot_split_clusters(labels, max_delay):
            break

        # Compute the silhouette score which gives a score for the density and
        # separation of the formed clusters (only if the delay between events
        # is less than a minute, otherwise save the clusters)
        silhouette_avg = 0
        if max_delay <= 60:
            silhouette_avg = silhouette_score(events_time.reshape(-1, 1), labels)
        if verbose:
            print("[dbg] Clustering score", silhouette_avg)
        if max_delay <= 60 and silhouette_avg > best_clusters[0]:
            if verbose:
                print("[dbg] BEST cluster", best_clusters[0], silhouette_avg)
            best_clusters = [silhouette_avg, labels[:]]
    if best_clusters[0] == 0:
            best_clusters = [silhouette_avg, labels[:]]
    if verbose:
        print("[dbg] Total clusters", max(best_clusters[1]) + 1)
    return best_clusters[1]

# unorder and only count (not file information)
def get_count_pattern(cluster_edges, events_type, pattern_list):
    optype_cluster = [events_type[cluster_edges[i-1]:cluster_edges[i]]
                      for i in range(1, len(cluster_edges))]
    count_pattern = [{optype : cluster.count(optype) for optype in set(cluster)} for cluster in optype_cluster]
    # example count pattern [{'fprintf': 7, 'fwrite': 12}, {'fprintf': 10, 'fwrite': 14}]
    log = []
    cnt = len(pattern_list.keys())
    for pattern in count_pattern:
        # patterns have {operation: count}
        pattern_string = [key+":"+str(pattern[key]) for key in pattern]
        pattern_string.sort()
        pattern_string = "-".join(pattern_string)
        # update the list of all existing patterns
        if pattern_string not in pattern_list:
            pattern_list[pattern_string] = cnt
            cnt += 1
        log.append(pattern_list[pattern_string])
    return pattern_list, log

# Extract all the event patterns describing patterns and their frequency
def find_patterns(labels, events_type, events_time, pattern_list, rank):
    patterns = {}
    cluster_edges = [0] + [i+1 for i in range(len(labels) - 1)
                           if labels[i] != labels[i+1]] + [len(labels) + 1]
    time_clusters = {cluster_edges[i-1] : events_time[cluster_edges[i-1]:cluster_edges[i]]
                     for i in range(1, len(cluster_edges))}
    time_log = [(min(time_clusters[i]), max(time_clusters[i])) for i in time_clusters]
    pattern_list, log = get_count_pattern(cluster_edges, events_type, pattern_list)
    # Add timing information over the log: (pattern_id, time_start, time_end)
    log = [(log[i], time_log[i][0], time_log[i][1], rank) for i in range(len(log))]
    if verbose:
        print("[dbg] Total patterns:", len(pattern_list.keys()), pattern_list)
        print("[dbg] The current TAU log:", [(i[1],i[2]) for i in log])
        print("[dbg] List of patterns:", [i[0] for i in log])
    return pattern_list, log

# Extract the start clusters based on patterns across all the ranks
# The log contains a list of (pattern_id, time_start, time_end)
def extract_start_clusters(log, min_ts, max_ts):
    start_idx = min_ts
    for rank in log:
        log_patterns = [i[0] for i in log[rank]]
        # if there are patterns occuring more than once
        if len(log_patterns) > len(set(log_patterns)):
            # find the first pattern that occurs more than once
            idx = next(i for i in range(1, len(log_patterns))
                       if log_patterns.count(log_patterns[i]) > 1)
            idx -= 1
        else:
            # otherwise take all events that are closer to beginning if any
            idx = [i for i in range(len(log_patterns))
                   if log[rank][i][2] - min_ts < max_ts - log[rank][i][2]]
            if len(idx) > 0:
                idx = idx[0]
            else:
                idx = -1
        if idx >= 0:
            # make sure the begin section does not exceed 1/3 of execution
            limit = min_ts + (max_ts - min_ts) * 0.3
            idx = [i for i in range(0, idx + 1) if log[rank][i][2] < limit]
            if len(idx) > 0:
                idx = max(idx)
                start_idx = max(start_idx, log[rank][idx][2])
    return start_idx

# Extract the end clusters based on patterns across all the ranks
# The log contains a list of (pattern_id, time_start, time_end)
def extract_end_clusters(log, min_ts, max_ts):
    # the end section will always include the "program end" entry
    end_idx = max_ts
    for rank in log:
        log_patterns = [i[0] for i in log[rank]]
        # if not all patterns are occuring only once
        if len(log_patterns) > len(set(log_patterns)):
            # find the last pattern that occurs more than once
            idx = next(i for i in reversed(range(1, len(log_patterns)))
                       if log_patterns.count(log_patterns[i]) > 1)
            idx += 1
        else: # otherwise take all events that are closer to the end if any
            idx = [i for i in range(len(log_patterns))
                   if log[rank][i][1] - min_ts > max_ts - log[rank][i][1]]
            if len(idx) > 0:
                idx = idx[0]
            else:
                idx = -1
        if idx >= 0 and idx <len(log[rank]):
            # make sure the end section does not exceed 1/3 of execution
            limit = max_ts - (max_ts - min_ts) * 0.3
            idx = [i for i in range(idx, len(log_patterns)) if log[rank][i][1] > limit]
            if len(idx) > 0:
                idx = min(idx)
                end_idx = min(end_idx, log[rank][idx][1])
    return end_idx

def repeat_comp_pattern(pattern, total_exec, req_exec,
                        ts_series, ts_series_end, rank):
    log = []
    # we cannot use interpolation so we just repeat the pattern
    if individual_patterns:
        if verbose:
            print("[dbg] Repeat for cluster pattern", pattern, total_exec)
        cnt = 1
        while True:
            # add entries total_exec apart
            ts = [i + cnt * total_exec for i in ts_series]
            log += [(pattern, ts_series[cnt % len(ts_series)],
                     ts_series_end[cnt % len(ts_series)], rank, i)
                    for i in ts if i <= req_exec]
            if max(ts) > req_exec:
                break
            cnt += 1
    return log

def interpolate_comp_pattern(log, total_exec, req_exec, rank, degree=1):
    if total_exec > req_exec:
        # cut all the events that exceed the requested execution time
        return [i for i in log if i[2] < req_exec]
    # if the requested time is larger than the existing log, extrapolate
    pattern_ids = set([i[0] for i in log])
    for pattern in pattern_ids:
        ts_series = [i[1] for i in log if i[0]==pattern]
        ts_series_end = [i[2] for i in log if i[0]==pattern]
        if len(ts_series) < 5:
            log += repeat_comp_pattern(pattern, total_exec, req_exec,
                                       ts_series, ts_series_end, rank)
        else:
            try:
                #The use of degree=1 signifies a linear fit.
                fit = np.polyfit(range(len(ts_series)), ts_series, degree)
                if verbose:
                    print("[dbg] Fit for cluster pattern", pattern, fit)
                np.seterr(invalid='ignore')
                line = np.poly1d(fit)
                cnt = len(ts_series)
                while True:
                    ts = line(cnt)
                    if ts > req_exec:
                        break
                    log += [(pattern, ts_series[cnt % len(ts_series)],
                             ts_series_end[cnt % len(ts_series)], rank, ts)]
                    cnt += 1
            except:
                log += repeat_comp_pattern(pattern, total_exec, req_exec,
                                           ts_series, ts_series_end, rank)
    return log

# Decrease or increase the total execution time of the log
def update_exec_pattern(log, start_ts, end_ts, req_exec, degree=1):
    req_log = {rank:[] for rank in log}
    total_exec = int(max([max([i[2] for i in log[rank]]) for rank in log]))
    shift = req_exec - total_exec
    if verbose:
        print("[dbg] Shift by:", shift)
    for rank in log:
        # keep the start pattern
        req_log[rank] += [i for i in log[rank] if i[2] <= start_ts]
        # interpolate the middle pattern
        if verbose:
            print("[dbg] Generate extended log for rank", rank)
        req_log[rank] += interpolate_comp_pattern(
                [i for i in log[rank] if i[2] > start_ts and i[1] < end_ts],
                end_ts - start_ts, req_exec - (total_exec - end_ts), rank, degree=degree)
        # shift the end pattern to the end of the request time
        req_log[rank] += [(i[0], i[1], i[2], rank, i[1]+shift)
                          for i in log[rank] if i[1] >= end_ts]
    return req_log

# extract clusters of ranks that have the exact same behavior
def get_rank_exact_match(log):
    rank_pattern = {}
    for rank in log:
        pattern = "".join([str(i[0]) for i in log[rank]])
        if pattern not in rank_pattern:
            rank_pattern[pattern] = []
        rank_pattern[pattern].append(rank)
    return rank_pattern

# return clusters of ranks with the same behavior
# Example output: [[0], [1, 2, 3, 4], [5, 6, 7, 8, 9]]
def get_rank_pattern(log):
    rank_pattern = get_rank_exact_match(log)
    words = np.array(list(rank_pattern.keys())).reshape(-1, 1)
    best_clusters = (0, list(range(len(words))))
    for true_k in range(2, len(words)):
        model = KMeans(n_clusters=true_k, init='k-means++',
                       max_iter=100, n_init=1)
        cluster = model.fit(words)
        if len(cluster.labels_)==1 or len(cluster.labels_)==len(words):
            silhouette_avg = 0
        else:
            silhouette_avg = silhouette_score(words, cluster.labels_)
        if silhouette_avg > best_clusters[0]:
            best_clusters = (silhouette_avg, cluster.labels_)
    # parse the clusters and create groups of rank ids
    rank_clusters = []
    words = [i[0] for i in words]
    for cluster_id in range(max(best_clusters[1]) + 1):
        temp = [rank_pattern[words[i]] for i in range(len(best_clusters[1]))
                if best_clusters[1][i] == cluster_id]
        rank_clusters.append([])
        for i in temp:
            rank_clusters[-1] += i
    return rank_clusters

# decreate or increase the number of ranks in the log
def update_rank_pattern(log, total_ranks, req_ranks, variability=0):
    total_ranks = len(set(i[0] for i in log))
    if req_ranks < total_ranks:
        # delete the extra ranks from the log
        for rank in range(req_ranks, total_ranks):
            if verbose:
                print("[dbg] Delete", rank)
            # delete all entries (tids) with pid = rank
            del_list = [i for i in log if i[0]==rank]
            for i in del_list:
                del log[rank]
    else:
        # find clusters of ranks with the same bahavior: [[0], [1, 2], [3]]
        rank_cluster = get_rank_pattern(log)
        # repeat the ranks that do not have individual behaviors
        rank_pattern = [i for i in rank_cluster if len(i) > 1]
        rank_pattern = list(roundrobin(rank_pattern))
        if len(rank_pattern) == 0: # if all have individual patterns
            rank_pattern = list(range(total_ranks)) # repeat all ranks
        # add rank behavior in order based on the patterns found
        cnt = 0
        for rank in range(total_ranks, req_ranks):
            if verbose:
                print("[dbg] Rank %d will have the same pattern as rank %d" %(
                    rank[0], rank_pattern[rank % len(rank_pattern)]))
                if variability > 0:
                    print("[dbg]  - added variability between (-%2.1f and %2.1f)" %(
                        variability, variability))
            # the log needs to contain all the threads for the chosen rank
            target_rank = rank_pattern[rank % len(rank_pattern)]
            tid_list = [i[1] for i in log if i[0]==target_rank]
            for tid in tid_list:
                log[(rank, tid)] = log[(target_rank, tid)][:]
                if variability > 0:
                    # add noise to the timestamp of the pattern, either to the original
                    # timestamp or the interpolated value
                    noise = np.random.uniform(-variability, variability,
                                              len(log[(rank, tid)]))
                    noise = [log[(rank, tid)][i][1]+noise[i]
                             if len(log[(rank, tid)][i])==4
                             else log[(rank, tid)][i][4]+noise[i]
                             for i in range(len(log[(rank, tid)]))]
                    # add the updated value to the new log
                    log[(rank, tid)] = [(log[(rank, tid)][i][0], log[(rank, tid)][i][1],
                                         log[(rank, tid)][i][2], log[(rank, tid)][i][3],
                                         max(0, noise[i]))
                                 for i in range(len(log[(rank, tid)]))]
    return log

# Read the json file and keep a dictionary with strings
# related to each timestamp
def read_time_to_string(file_name):
    inf = open(file_name, "r")
    time_to_string = {}
    json_data = json.load(inf)
    min_ts = min([entry["ts"] for entry in json_data])
    for entry in json_data:
        if (entry["pid"], entry["tid"]) not in time_to_string:
            time_to_string[(entry["pid"], entry["tid"])] = {}
        time_to_string[(entry["pid"], entry["tid"])][(entry["ts"]-min_ts)/1000000] = entry
    inf.close()
    return time_to_string

# Create a JSON file from the log based on the input JSON
# log is a dict {rank: [(pattern_id, initial_ts_start, initial_ts_end, initial_rank)]}
# with the possibility of having another element at the end with the new timestamp
def from_pattern_create_log(log, input_file, output_file):
    max_exec = 0
    time_to_string = read_time_to_string(input_file)
    data = []
    for rank in log:
        if verbose:
            print("[dbg] Writing log for rank %s to the output file" %(rank[0]))
        for pattern in log[rank]:
            # get all the entries occuring between the timestamps
            create_events = [i for i in time_to_string[pattern[3]]
                             if i <= pattern[2] and i >= pattern[1]]
            delay = 0
            if len(pattern)>4:
                max_exec = max(max_exec, pattern[4])
                delay = pattern[4] - pattern[1]
            else:
                max_exec = max(max_exec, pattern[2])
            event_list = [time_to_string[pattern[3]][i] for i in create_events]
            for event in event_list:
                temp_event = {i:event[i] for i in event}
                temp_event["pid"] = rank[0]
                temp_event["tid"] = rank[1]
                temp_event["ts"] = int(temp_event["ts"] + delay * 1000000)
                data.append(temp_event)
    print("Creating a TAU log file with %s ranks and %s execution time" %(
        len(log.keys()), max_exec))
    with open(output_file, 'w') as f:
        f.write(
        '[' +
        ',\n'.join(json.dumps(i) for i in data) +
        ']\n')
    print("DONE. Output json file stored in", output_file)
    return 0

def parse_input_argument():
    global individual_patterns, verbose
    parser = argparse.ArgumentParser()
    parser.add_argument('infile', help='Input TAU json file')
    parser.add_argument('req_ranks', type=int, help='Number of requested ranks')
    parser.add_argument('req_exec', type=int,
                        help='Requested execution time in seconds')
    parser.add_argument("-v", "--verbose", action='store_true',
                        help="Print debug information")
    parser.add_argument("-i", "--interpolate", action='store_false',
                        help="Repeat only log patterns that can be interpolated")
    parser.add_argument("-o", "--output", default="output.json",
                        help='Output file to store the new JSON log')
    parser.add_argument("-d", "--degree", type=int, default=1,
                        help='Interpolation polynomial degree. Default: linear')
    parser.add_argument("-r", "--rankvar", type=int, default=0,
                        help='Variability among rank duplication. Adds a random noise' \
                              ' with a random distribution with a center of 0 and' \
                              ' length of value (in seconds). Default: 0 (no variability)')
    parser.add_argument("-s", "--stats", action='store_true',
                        help='Only show stats about the TAU log, do not create' \
                             ' a new one. The requested time and ranks will be ignored')
    args = parser.parse_args()
    if args.interpolate:
        individual_patterns = True
    if args.verbose:
        verbose = True
    return args

if __name__ == '__main__':
    args = parse_input_argument()
    req_ranks = args.req_ranks
    req_exec = args.req_exec
    labels = {}
    # read the json TAU file
    events_time, events_type = read_json_data(args.infile)
    min_ts = min([min(events_time[rank]) for rank in events_time])
    max_ts = max([max(events_time[rank]) for rank in events_time])
    print("Number of ranks %d" %(len(set([i[0] for i in events_time.keys()]))))
    print("Execution time %d seconds" %(max_ts - min_ts))
    if args.stats:
        print("The -stats flag was provided, exiting without creating the output log")
        exit()
    pattern_list = {}
    log = {}
    for rank in events_time:
        if verbose:
            print("[dbg] Extract clusters for rank", rank)
        labels[rank] = find_clusters(
                np.array(events_time[rank]),
                end_marker=(events_type[rank][-1]=="program exit"))
        pattern_list, log[rank] = find_patterns(labels[rank], events_type[rank],
                                          events_time[rank], pattern_list, rank)
    start_ts = extract_start_clusters(log, min_ts, max_ts)
    end_ts = extract_end_clusters(log, min_ts, max_ts)
    if verbose:
        print("[dbg] Start position", start_ts)
        print("[dbg] End position", end_ts)

    if (max_ts - min_ts) != req_exec:
        log = update_exec_pattern(log, start_ts, end_ts, req_exec,
                                  degree=args.degree)
    if len(events_time.keys()) != req_ranks:
        log = update_rank_pattern(log, len(events_time.keys()), req_ranks,
                                  variability=args.rankvar)

    from_pattern_create_log(log, args.infile, args.output)