'''
This program is meant to simulate a General Adversarial Network, where the classifier
is used to improve the generator. Mainly, I'm trying to find an ideal diversity value
to use during generation such that the classifier predicts the most false positives.
This is only a simulation of a GAN because neither the classifier or the generator
are being updated after each pass, rather I'm just using two models to find the optimal
value for one variable.

Author: Alex Fantine

The Classifier object has access to the following variables and methods:
self.model
self.test_x, self.test_y (for testing purposes only, since new data should be provided)
self.tokenizer
self.max_index (for denormalization)

self.generate_error_report(predicted, test_x, test_y, tokenizer, max_index, false_pos_only)
where predicted is the result from the model, and x is the input values and y are the labels
and false_pos_only is set to true if only printing the false positives

returns false_pos, false_neg (only false_pos if false_pos_only is true)
where each element is a list containing [predicted prob, actual value, string headline]
'''

from Classifier import Classifier
from Generator import Generator
from keras.preprocessing.sequence import pad_sequences
import random
import numpy as np
from collections import defaultdict

'''
This method generates 1,000 headlines, where every ten, the input word changes.

generator is the generator object to be used
diversity vals is the array of diversities for each word, will print error if
number of diversities does not match the number of words in the generation.

returns a list of tuples, where each is a headline string and its repeat value
'''
def generate_headlines(generator, diversity_vals, num_headlines, verbose = 0):
    headlines = []
    for i in range(0, num_headlines):
        if verbose == 1:
            print('iteration ' , i+1, '/', num_headlines)
        rando_int = random.randint(0,len(generator.tokenizer.word_index.items()))

        for word, index in generator.tokenizer.word_index.items():
            if index == rando_int:
                rando_word = word
                break
        #num_words, seed_text, diversity, experiments=1
        generated_headline = generator.generate_headline(15, rando_word, diversity_vals)

        headlines.append(generated_headline)
    return headlines

#prints all of the headlines with repeat values
def print_headlines(generated_headlines):
    for line in generated_headlines:
        print(line[0], ': ', line[1])

#returns an array of just headline strings, useful for creating input for the classifier
def clean_headlines(generated_headlines):
    cleaned_headlines = []
    for line in generated_headlines:
        cleaned_headlines.append(line[0])
    return cleaned_headlines

#converts headlines to integer arrays so they can be read by the classifier
def preprocess_headlines(classifier, x):
    new_x = []
    for line in x:
        token_list = classifier.tokenizer.texts_to_sequences([line])[0]
        new_x.append(token_list)

    inp_x = np.array(pad_sequences(new_x, maxlen=classifier.max_seq_len))
    return inp_x

#prints the probability predictions in the parameter list and the sentence, used for false_pos
def print_prediction_info(values):
    for item in values:
        print('the classifier said ', item[0], ' but the actual was ', item[1])
        print(item[2])
        print("")

'''
Perform one step of the GAN simulation, which includes:
Pick a random float in the given range and create a list of that float for each words diversity
Generate 100 headlines with the generator and the given diversity
Create input data using generated headlines for the classifier
Write the report from the classifier, which includes the compilation of all false postiives

Return: the chosen diversity float, the list of false positives
'''
def perform_gan_step(generator, classifier):
    div_list = []
    random.seed()
    div_val = random.uniform(.5, .9)
    #div_val = 1.626816675757897
    print("")
    print('diversity value: ', div_val)
    for i in range(0, 16):
        div_list.append(div_val)

    #generate 100 headlines with set diversity
    generated_headlines = generate_headlines(generator, div_list, 100, verbose = 0)

    x = clean_headlines(generated_headlines)
    y = [0 for line in range(len(x))]

    X = preprocess_headlines(classifier, x)
    random.shuffle(X)

    predicted = classifier.model.predict_proba(X, verbose=0)

    false_pos = classifier.generate_error_report(predicted, generated_headlines, \
    y, classifier.tokenizer, True)

    return div_val, false_pos

'''
Tests a number of different diversities equal to the parameter num_iter
Prints the false positive list if print_false_pos== True
'''
def test_one_diversity(generator, classifier, num_iter=10, print_false_pos = False):
    output_file = open('diversity_experiment.txt', 'a')
    #num_iter = 10
    print("", end='\n', file=output_file)
    for i in range(0,num_iter):
        print("iteration ", i+1, '/', num_iter)
        diversity, false_pos = perform_gan_step(generator, classifier)

        if print_false_pos:
            print_prediction_info(false_pos)

def get_best_diversities(num_max):
    #preprocess the data to remove newlines
    output_file = open('diversity_experiment.txt', 'r')
    raw_data = ''.join([line for line in output_file])
    raw_data.strip("")
    data = raw_data.split('\n')
    data = [item for item in data if item is not ""]

    #create dictionary of num false pos and diversity vals
    val_dict = defaultdict(list)
    for i in range(len(data)):
        vals = data[i].split(' :  ')
        val_dict[vals[1]].append(vals[0])

    #sort keys by maxval
    keys = list(val_dict.keys())
    int_keys = [int(key) for key in keys]
    int_keys.sort(reverse = True)

    #take the averages of the diversities up to the max number of false pos
    total_averages = []
    for i in range(0,num_max):
        print(int_keys[i], ': ', val_dict[str(int_keys[i])])
        val_dict[str(int_keys[i])] = [float(val) for val in val_dict[str(int_keys[i])]]
        average =  sum(val_dict[str(int_keys[i])])/len(val_dict[str(int_keys[i])])
        print('average of vals: ', average)
        total_averages.append(average)
    print("")
    print('total average: ', sum(total_averages)/len(total_averages))

if __name__ == '__main__':

    generator = Generator()
    classifier = Classifier()
    #random.seed(5) #setting a seed for control when generating sentences of varying diversities
    test_one_diversity(generator, classifier, 5, True)


import gpflow
import numpy as np
from scipy import integrate, stats
from matplotlib import pyplot as plt
from exp.bayesian_quadrature.derivative_trick.kernels import RBF_dd
from exp.bayesian_quadrature.derivative_trick.synthetic_functions import gaussian_pdf
from exp.bayesian_quadrature.derivative_trick.utils \
    import predict_integral, plot_gp_vs_truth, plot_gaussian

START = -10
END = 10
N_obs = 50
N_true = 1000
N_test = 1000
N_test_int = 100
X_TEST_SINGLE = 0.5
f = gaussian_pdf

# Get integrand truth
X_true = np.linspace(START, END, N_true).reshape((-1, 1))
Y_true = f(X_true)

# Get observations
X_obs = np.linspace(START, END, N_obs).reshape((-1, 1))
Y_obs = f(X_obs)

# Build model
k = RBF_dd()
m = gpflow.models.GPR(X_obs, Y_obs, kern=k, mean_function=None)
print(m)

# Train model
opt = gpflow.train.ScipyOptimizer()
m.likelihood.variance = 0.0001
m.likelihood.variance.trainable = False
opt.minimize(m)
print(m)

# Get integrand predictions
X_test = np.linspace(START - 10, END + 10, N_test).reshape((-1, 1))
means, variances = m.predict_f(X_test)

# Get integral truth
integral_truth = integrate.quad(f, START, END)[0]

# Get integral predictions
X_test_int = np.linspace(START, END, N_test_int).reshape((-1, 1))
integral_means, integral_vars = predict_integral(m, X_test_int, X_obs, Y_obs)

# Perform plotting
plt.subplot(1, 3, 1)
plot_gp_vs_truth(X_true, Y_true, X_obs, Y_obs, X_test,
                 means, variances, 'Integrand Fit')

plt.subplot(1, 3, 2)
Y_int_true = stats.norm.cdf(X_true)
integral_vars_diag = np.diag(integral_vars).reshape((-1, 1))
plot_gp_vs_truth(X_true, Y_int_true, None, None, X_test_int,
                 integral_means, integral_vars_diag, 'Integral Fit')

plt.subplot(1, 3, 3)
X_test_int_single = np.array([START, X_TEST_SINGLE], dtype=float).reshape((-1, 1))
integral_truth_single = integrate.quad(f, START, X_TEST_SINGLE)[0]
integral_means_single, integral_vars_single = predict_integral(m, X_test_int_single, X_obs, Y_obs)
approx_int_means_single = float(integral_means_single[1])
approx_int_vars_single = np.diag(integral_vars_single)[1]
plot_gaussian(integral_truth_single, approx_int_means_single,
              approx_int_vars_single, label='X=3.0')

plt.show()


from flask import Flask, request, jsonify
import requests

app = Flask(__name__)

@app.route('/')
def index():
    return 'Hello, world!'

@app.route('/student/<regno>')
def get_student_status(regno):
    r = requests.get('http://db:5432/student/' + regno)
    if r.status_code == 200:
        return jsonify(r.json())
    else:
        return 'Error: Could not retrieve student data.'

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


import tkinter as tk
from collections import Counter
from datetime import datetime, date
from tkinter import ttk, scrolledtext
from tkinter.ttk import Frame, Radiobutton, Label, Scrollbar, Button, OptionMenu, Entry
from tkinter import Spinbox, IntVar
from tkinter import *
import matplotlib
matplotlib.use("TkAgg")
import quandl
import matplotlib.pyplot as plt
from matplotlib.backends.backend_tkagg import (FigureCanvasTkAgg,
                                               NavigationToolbar2TkAgg)
from matplotlib.figure import Figure
import math
import pandas as pd
from pandas_datareader import data as web
from pandas_datareader import _utils as err

class ViewFrame(tk.Frame):
    def __init__(self, master=None, company_data=None):
        self.master = master
        self.master.title("View frame")
        tk.Frame.__init__(self, master, background="#0000ff")

        text_frame = tk.Frame(master, background="#000000")
        graph_frame = tk.Frame(master, background="#000000")

        self.text_prompt = self.create_text_widget(text_frame)

        (scrolled_frame, self.scrolled_canvas) = self.create_graph_widget(graph_frame)

        self.plot_frames = []
        self.populate(company_data, scrolled_frame)
        self.layout()

        text_frame.pack(side=tk.TOP, fill="x", expand=False)
        graph_frame.pack(side=tk.BOTTOM, fill="both", expand=True)

    def create_graph_widget(self, parent):
        canvas = tk.Canvas(parent, borderwidth=0, background="#000000")
        frame = tk.Frame(canvas, background="#000000")
        vsb = tk.Scrollbar(parent, orient="vertical", command=canvas.yview)
        canvas.configure(yscrollcommand=vsb.set)

        vsb.pack(side="right", fill="y", expand=0)
        canvas.pack(side="left", fill="both", expand=1)
        canvas.create_window((4,4), window=frame, anchor="nw",
                                  tags="frame")

        frame.bind("<Configure>", self.onFrameConfigure)
        canvas.bind("<Configure>", self.onFrameConfigure)
        return frame, canvas

    def create_text_widget(self, parent):
        text_prompt = scrolledtext.ScrolledText(master=parent,
                                             wrap=tk.WORD,
                                             width=20,
                                             height=10,
                                             background="#000000",
                                             foreground="#00ff00")
        text_prompt.pack(padx=0, pady=10, fill=tk.BOTH, expand=True)
        return text_prompt

    def populate(self, company_data, scrolled_frame):
        self.plot_frames = []
        for i, company in enumerate(company_data):
            p_frame = self.build_plot_frame(scrolled_frame, company)
            self.plot_frames.append(p_frame)

    def layout(self):
        if self.plot_frames:
            for widget in self.plot_frames:
                widget.grid_remove()
            items_per_row = int(self.scrolled_canvas.winfo_width() / self.plot_frames[0].winfo_width())
            if items_per_row < 1:
                items_per_row = 1
            for i, widget in enumerate(self.plot_frames):
                widget.grid(row=int(i/items_per_row), column=i%items_per_row, sticky=S+E, padx=(0, 5), pady=(0, 5))

    def onFrameConfigure(self, event):
        '''Reset the scroll region to encompass the inner frame'''
        self.scrolled_canvas.configure(scrollregion=self.scrolled_canvas.bbox("all"))
        self.layout()

    def build_plot_frame(self, parent_container, company_data):
        """ Given a company dictionary, creates a plot on a frame.
        """
        p_frame = Frame(parent_container, relief=GROOVE, borderwidth=2, background="#000000")
        # Create a list of datetime.
        dates = []
        for sighting in company_data["sightings"]:
            dates.append(datetime.strptime(sighting["datetime"][:10], "%Y-%m-%d"))
        # Create the plot

        dates = list(Counter(dates).items())

        fig = plt.figure(figsize=(5, 5), dpi=100)
        axis = fig.add_subplot(111)

        # Populate the tkinter frame
        label = tk.Label(p_frame, text=company_data["name"], font=("Monaco", 24), background="#000000", foreground="#00ff00")
        label.pack(pady=10,padx=10)

        stock_data = None
        if "tickers" in company_data.keys():
            stock_data = self.get_stock_data(dates, company_data["tickers"][0])

        if stock_data:
            new_list = []
            for (a, b) in dates:
                d_val = 0
                for (c, d) in stock_data:
                    if a == c:
                        d_val = d
                        break
                new_list.append((a, b, d_val))
            for (c, d) in stock_data:
                b_val = 0
                for (a, b) in dates:
                    if a == c:
                        b_val = b
                        break
                if not b_val:
                    new_list.append((c, b_val, d))
            new_list = sorted(new_list, key=lambda x: x[0])

            x, y_1, y_2 = zip(*new_list)

            axis2 = axis.twinx()
            axis.bar(x, y_1, color='green')
            axis2.bar(x, y_2, color='blue')
            axis2.tick_params('y', colors='r')
        else:
            x_1, y_1 = zip(*dates)
            axis.bar(x_1, y_1, color='green')

        axis.set_facecolor((0, 0, 0))
        fig.autofmt_xdate()
        axis.spines['top'].set_color('green')
        axis.spines['right'].set_color('green')
        axis.spines['bottom'].set_color('green')
        axis.spines['left'].set_color('green')
        axis.xaxis.label.set_color('green')
        axis.tick_params(axis='x', colors='green')
        axis.tick_params(axis='y', colors='green')
        fig.patch.set_facecolor('black')

        #hide the toolbar
        fig.canvas.toolbar.pack_forget()
        axis.set_clip_on(False)

        canvas = FigureCanvasTkAgg(fig, p_frame)
        canvas.show()
        canvas.get_tk_widget().pack(side=tk.BOTTOM, fill=tk.BOTH, expand=True)

        #toolbar = NavigationToolbar2TkAgg(canvas, p_frame)
        #toolbar.update()
        canvas._tkcanvas.pack(side=tk.TOP, fill=tk.BOTH, expand=True)
        return p_frame

    def check_callback(self):
        print("check_callback")

    def get_stock_data(self, dates, symbol):
        """ Get the stock quotes between the dates.
        """
        #query_list = ['WIKI' + '/' + symbol + '.' + str(k) for k in range(1, 13)]
        print(type(dates[0][0]))
        print(dates[0][0])

        print(dates[-1][0])
        start = datetime(dates[0][0].year, dates[0][0].month, dates[0][0].day)
        end = datetime(dates[-1][0].year, dates[-1][0].month, dates[-1][0].day)
        print(start)
        #start = datetime.datetime(2016,1,1)
        end = date.today()
        try:
            stock_data = web.DataReader("HM-B.ST", "yahoo", start, end)
        except err.RemoteDataError:
            try:
                stock_data = web.DataReader("HM-B.ST", "google", start, end)
            except err.RemoteDataError:
                return None
        if not stock_data.empty:
            stock_change = abs((stock_data["Close"] / stock_data["Open"]) -1)
            stock_change_list = []
            for key, value in stock_change.to_dict().items():
                if math.isnan(value):
                    value = 0
                stock_change_list.append((key.to_pydatetime(), value))
            stock_change_list = sorted(stock_change_list, key=lambda x: x[0])
            return stock_change_list
        else:
            return None


from django.core.exceptions import ValidationError
from django.test import TestCase

from lost_and_found.objects_posts.models import Object

class TestObject(TestCase):
    valid_name = 'object1'
    valid_image = 'http://image.com'
    valid_height = 2
    valid_weight = 1

    def test__when_width_is_less_than_max__expect_to_work(self):
        width = 5
        object_1 = Object(
            name=self.valid_name,
            image=self.valid_image,
            width=width,
            height=self.valid_height,
            weight=self.valid_weight
        )
        object_1.full_clean()
        object_1.save()

    def test__when_width_is_equal_to_max__expect_to_work(self):
        width = 600
        object_1 = Object(
            name=self.valid_name,
            image=self.valid_image,
            width=width,
            height=self.valid_height,
            weight=self.valid_weight
        )
        object_1.full_clean()
        object_1.save()

    def test__when_width_is_more_than_max__expect_to_raise_error(self):
        width = 605
        object_1 = Object(
            name=self.valid_name,
            image=self.valid_image,
            width=width,
            height=self.valid_height,
            weight=self.valid_weight
        )

        with self.assertRaises(ValidationError) as ex:
            object_1.full_clean()
            object_1.save()
        self.assertIsNotNone(ex.exception)


import argparse
import logging
import multiprocessing
import serial
import time
import socket
import types

from .client import Client
from modbus_tk.modbus_tcp import TcpServer
from modbus_tk.modbus_rtu import RtuServer
from modbus_tk.hooks import install_hook

# logger = logging.getLogger(__name__)

###########################################################################################
#   MONKEY PATCH ALERT
#
#   To allow regression tests to quickly start-up/shutdown their Modbus slaves,
#   we are monkey patching two methods in modbus_tk.modbus_tcp.TcpServer.  These changes
#   shutdown the socket as quickly as possible and re-use the socket port even if it is
#   in TIME_WAIT.  Both of these one line changes are REQUIRED to avoid the dreaded
#   "Address already in use" error.
#
###########################################################################################

def _do_init(self):
    """initialize server"""
    self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    if self._timeout_in_sec:
        self._sock.settimeout(self._timeout_in_sec)
    # BIND to port even if it is still in TIME_WAIT
    self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    self._sock.setblocking(0)
    self._sock.bind(self._sa)
    self._sock.listen(10)
    self._sockets.append(self._sock)

def _do_exit(self):
    """clean the server tasks"""
    # close the sockets
    for sock in self._sockets:
        try:
            # Immediately shut down connection.
            sock.shutdown(socket.SHUT_RDWR)
            sock.close()
            self._sockets.remove(sock)
        except Exception as msg:
            raise Exception("Error while closing socket, Exception occurred: %s", msg)
    self._sock.close()
    self._sock = None

###########################################################################################
#  The two methods above are patched the TcpServer instance that is created by Server
#  below. See Server.__init__ method.
###########################################################################################

class Server:
    """Base Modbus Server.

    Basic Modbus Server that implements both RTU and TCP. For setting up an RTU server, make sure 'port' is configured
    to 0.  Basic hooks are set up here, so that when subclassing this class, only minimal functions (namely update)
    need to be overwritten.

    A Server can have multiple slaves, although TCP protocol only allows for one (RTU allows for multiple slaves).
    In this instance, the slave id should be set to 1.

    This class is mostly used for testing to simulate a modbus server that we would see in the field.
    """
    def __init__(self, address='127.0.0.1', port=502, baud=19200, timeout_in_sec=1, databank=None):
        """Initialize Base Server and install modbus hooks

        Has two important instance variables:
            _server - an instance of a Modbus RTU or TCP server.
            _slaves - dictionary of ModbusClient slaves keyed by slave ID. (For TCP, this should be 1)

        :param address: Address of the Server (Not used in RTU). Defaults to 127.0.0.1
        :param port: Port of the Server (0 for RTU, defaults to 502 for TCP). For RTU, this MUST be set to 0.
        :param baud: Baudrate (Not used in TCP). Defaults to 19200
        :param timeout_in_sec: Timeout (Not used in RTU). Defaults to 1
        :param databank: (Not used in RTU)
        """

        if port == 0:
            server_serial = serial.Serial(
                address,
                baudrate=baud,
                bytesize=8,
                parity=serial.PARITY_EVEN,
                stopbits=1,
                xonxoff=0)
            self._server = RtuServer(server_serial)
        else:
            self._server = TcpServer(address=address, port=port, timeout_in_sec=timeout_in_sec, databank=databank)
            #  Begin Monkey Patch
            self._server._do_init = types.MethodType(_do_init, self._server)
            self._server._do_exit = types.MethodType(_do_exit, self._server)
            # End Monkey Patch

        self._slaves = dict()

        # Set up basic modbus hooks. These by default just log when they are getting called. To handle requests,
        #  subclasses of Server should overwrite the 'update' method, which is called by 'after'.
        self.install_hook('modbus.Server.before_handle_request', 'before')
        self.install_hook('modbus.Server.after_handle_request', 'after')
        self.install_hook('modbus.Slave.handle_write_multiple_registers_request', 'write_register')
        self.install_hook('modbus.Slave.handle_write_single_register_request', 'write_register')
        self.install_hook('modbus.Slave.handle_read_holding_registers_request', 'read_register')
        self.install_hook('modbus.Slave.handle_write_multiple_coils_request', 'write_coils')
        self.install_hook('modbus.Slave.handle_write_single_coil_request', 'write_coils')
        # self.install_hook('modbus.Databank.on_error', 'print_error')

    def define_slave(self, slave_id, client_class, unsigned=True):
        """Add a Modbus Client Slave.

        :param slave_id: Slave ID. This must be 1 for Modbus TCP.
        :param client_class: Subclass of modbus.Client. This class holds all the register definitions.
        :param unsigned: Boolean indicating whether or not values are signed or unsigned.
        :return: Slave instance.
        """
        self._slaves[slave_id] = client_class
        slave = self._server.add_slave(slave_id, unsigned=unsigned)
        slave.klass = client_class
        if not issubclass(client_class, Client):
            raise TypeError("client_class must be subclass of {0}".format(Client))

        for request in client_class().requests():
            # logger.info("Adding [%s], %s, %s, %s", request.name, request.table, request.address, request.count)
            slave.add_block(request.name, request.table, request.address, request.count)
        return slave

    @classmethod
    def install_hook(cls, hook_name, method):
        """Helper method for installing hooks. This is necessary to ensure child methods are called if implemented."""
        install_hook(hook_name, getattr(cls, method))

    def set_values(self, slave_id, field, values):
        slave = self._server.get_slave(slave_id)
        slave_class = self._slaves[slave_id]
        request = slave_class().get_request(field)
        slave.set_values(request.name, field.address, values)

    ##################
    # Server Methods #
    ##################

    def start(self):
        self._server.start()

    def stop(self):
        self._server.stop()

    def is_alive(self):
        return self._server._thread.is_alive()

    def set_verbose(self, on=True):
        self._server.set_verbose(on)

    ######################
    # Basic Modbus Hooks #
    ######################

    @staticmethod
    def before(args):
        server, request = args
        # logger.debug("Before: {0}-{1}".format(server, request))

    @classmethod
    def after(cls, args):
        server, response = args
        # logger.debug("After: {0}-{1}".format(server, response))
        cls.update(args)

    @staticmethod
    def update(args):
        pass

    @staticmethod
    def write_register(args):
        slave, request_pdu = args
        # logger.debug("Writing: {0}-{1}".format(slave, request_pdu))

    @staticmethod
    def read_register(args):
        slave, request_pdu = args
        # logger.debug("Reading: {0}-{1}".format(slave, request_pdu))

    @staticmethod
    def write_coils(args):
        slave, request_pdu = args
        # logger.debug("Writing Coils: {0}-{1}".format(slave, request_pdu))

    # @staticmethod
    # def print_error(data_bank, exception, pdu):
    #     pass

class ServerProcess (multiprocessing.Process):
    """Parent process to set up, start, and run a Server Instance

    Used pretty heavily in testing.
    """
    def __init__(self, *args, **kwargs):
        """Instantiate Modbus Server Process

        kwargs
            host: Server IP address
            port: Server Port
            server_class: A subclass of Server if desired, otherwise defaults to server.
        """
        self._host = kwargs.pop('host', '127.0.0.1')
        self._port = kwargs.pop('port', 5020)

        self._server_class = kwargs.pop('server_class', Server)
        self._server = self._server_class(address=self._host, port=self._port)
        super(ServerProcess, self).__init__(*args, **kwargs)

    @property
    def server(self):
        return self._server

    def stop(self):
        logging.debug("Got SIGTERM. Stopping.")
        self._server.stop()
        super(ServerProcess, self).terminate()

    def run(self):
        # logger.debug("Starting ModbusServer: %s", __name__)
        self._server.start()

        # Only way I know of to keep the process running/waiting on the server loop.
        import signal

        def handler(signum, frame):
            logging.debug("Got SIGTERM. Stopping.")
            self._server.stop()
            super(ServerProcess, self).terminate()

        signal.signal(signal.SIGTERM, handler)

        while self._server.is_alive():
            time.sleep(1)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    parser = argparse.ArgumentParser()
    parser.add_argument("--host", type=str, default="127.0.0.1", help="Server Host/IP Address")
    parser.add_argument("--port", type=int, default=5020, help="Server Port Number")
    args = parser.parse_args()

    modbus_server = ServerProcess(host=args.host, port=args.port)
    modbus_server.start()
    modbus_server.join()


from fastapi import FastAPI
import pickle

app = FastAPI()

@app.on_event("startup")
def load_model():
    global model
    model = pickle.load(open("model_tree.pkl", "rb"))

@app.get('/')
def index():
    return {'message': 'This is the homepage of the API '}


#!/usr/bin/env python3

import taxonomy as tax
import viral_genome_classes as obj
import parser_interpro_results as do
import visualisation_alignment as view_aln
import visualisation_protein as view_prot

from os import path, listdir, rename
import gzip
import logging
from Bio import SeqIO
import sys
import csv
import re
from numpy import std, mean
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter

def getCdsObject(taxon_prot_ids, index_genome_file, interpro_domains_file, sp_treshold):

    gb_dict_iter = tax.getAllRefseqFromTaxonIdList(taxon_prot_ids, index_genome_file)

    cds_list = []
    for gb_file_dict in gb_dict_iter:
        cds_found = False
        taxon_id = gb_file_dict['taxon_id']
        gb_file = gb_file_dict['gb_file']

        genome = obj.gb_file_parser(gb_file, taxon_id, sp_treshold)
        if interpro_domains_file:  # interpro_domains_file can be set as None and the domain annotation part is not computed
            do.getMatchObject(genome, interpro_domains_file)
            do.associateDomainsWithPolyprotein(genome)
            # for segment in genome.segments:
            #     do.getDomainOverlappingInfo(segment)

        for segment in genome.segments:

            for cds in segment.cds:
                if cds.protein_id in taxon_prot_ids[taxon_id]:
                    cds.genetic_code = gb_file_dict['genetic_code']
                    cds_list.append(cds)
    assert len(cds_list) == sum([len(protein_ids) for protein_ids in taxon_prot_ids.values()])
    # if len(cds_list) != sum([len(protein_ids) for protein_ids in taxon_prot_ids.values()]):
    #     # print(len(cds_list), '!=' ,sum([len(protein_ids) for protein_ids in taxon_prot_ids.values()]))
    #     set_requested = set()
    #     for taxon_id, protein_ids in taxon_prot_ids.items():
    #         set_requested |= {f'{taxon_id}|{protein_id}' for protein_id in protein_ids}
    #
    #     set_retrieved = set()
    #
    #     for cds in cds_list:
    #         set_retrieved.add(f'{cds.segment.taxon_id}|{cds.protein_id}')
    #
    #     print(set_retrieved)
    #     print(set_requested - set_retrieved)
    #     input()
    return cds_list

def convertAlignmentToList(alignment_seq):
    """
    Take fasta sequence of the alignment so the sequence with gap symbol
    and make a list with indice of the ungap sequence and None when it is a gap
    the following sequence :
    --ABC-D--E
    would become :
    [None, None, 0, 1, 2, None, 3, None, None, 4]
    """
    gap = "-"
    list = []
    indice_seq = 0
    for i, e in enumerate(alignment_seq):
        if e != gap:

            list.append(indice_seq)
            indice_seq += 1
        else:
            list.append(None)
    return list

def findCloseSites(site, site_start_in_aln, site_end_in_aln, cds_list, window):

    size_site_in_aln = site_end_in_aln - site_start_in_aln + 1
    # site.neighboring_sites = set()

    for cds in cds_list:
        aln_part_list = cds.aln_list[site_start_in_aln -
                                     int(window/2)+1:site_start_in_aln+(size_site_in_aln-1)+int(window/2)]

        for position in aln_part_list:

            if position in cds.cleavage_site_positions:

                adj_site = cds.cleavage_site_positions[position]
                if adj_site != site:
                    site.neighboring_sites.add(adj_site)
                    adj_site.neighboring_sites.add(site)
                # print("correpsond to a CS", adj_site.taxon_id, adj_site.start_aa(cds) )
                # print(seq_part)
                # print(aln_part_list)

def groupCleavageSitesAcrossAlignment(cds_list, window):
    all_sites = set()
    annotated_cds_list = [cds for cds in cds_list if cds.polyprotein]
    # Reinitialisation of cleavage_site obj
    for cds in cds_list:
        cds.predicted_cleavage_sites = []
        for site in cds.cleavage_sites:
            site.cds_of_aln = []
            site.start_in_aln = None
            site.end_in_aln = None
            site.neighboring_sites = set()

    for i, cds in enumerate(annotated_cds_list):
        all_sites |= set(cds.cleavage_sites)

        for site in cds.cleavage_sites:
            site.cds_of_aln.append(cds)
            site_start = site.start_aa(cds) - 1  # to be in base 0 we need the -1
            site_end = site.end_aa(cds) - 1
            site_start_in_aln = cds.aln_list.index(site_start)
            site_end_in_aln = cds.aln_list.index(site_end)
            site.start_in_aln = site_start_in_aln
            site.end_in_aln = site_end_in_aln

            findCloseSites(site, site_start_in_aln, site_end_in_aln, annotated_cds_list, window)

    # make group of cleavage sites
    grouped_sites = set()
    group_list = []
    for site in all_sites:
        if site not in grouped_sites:
            new_group = set()
            groupCleavageSites(new_group, site)

            grouped_sites |= new_group  # add site that are grouped in a the set to not process them again
            group_list.append(new_group)

    # print('NB_GROUP', len(group_list))

    nb_cs_in_group = sum([len(group) for group in group_list])

    # print("len(all_sites) ",len(all_sites) )
    # print('nb_cs_in_group',nb_cs_in_group )
    assert nb_cs_in_group == len(all_sites), 'Cs grouped twice in two different groupe'
    # sort list of group by the position in aln of the first site of each group
    return sorted(group_list, key=lambda x: list(x)[0].start_in_aln)

def groupCleavageSites(group, site):
    # recursivity
    group.add(site)
    for adj_site in site.neighboring_sites - group:
        groupCleavageSites(group, adj_site)

def getStatOnGroup(group, nb_cds_annotated, absent_annotated_cds_penalty):
    mean_positions = []
    positions = []
    protein_in_group_list = []

    for i, site in enumerate(group):
        for cds in site.cds_of_aln:
            """
            The same cleavage site may attached to more than one cds
            It happends when two cds share an identical part due to a ribosomal frameshift/readthrough and
            the cleavage sites in this shared part are identical in the 2 proteins.
            The cleavage sites are not duplicated during the parsing but have more than one cds in
            their attribute 'proteins'.
            But here to compute stat on the group we consider the cleavaged site the number of time it appears in
            the group to make it less confusing
            """
            # if not cds.polyprotein:
            #     print(cds)
            #     print(site)
            #     input()
            # -1 to be in base 0 because it'll be use as a list index
            site_start = site.start_aa(cds) - 1
            site_end = site.end_aa(cds) - 1
            protein_in_group_list += site.cds_of_aln
            # pfnt(len([i for i in cds.aln_list if i is not None]))
            positions += [cds.aln_list.index(site_start), cds.aln_list.index(site_end)]
            mean_position_in_aln = sum([cds.aln_list.index(site_start),
                                        cds.aln_list.index(site_end)])/2.0
            mean_positions.append(mean_position_in_aln)

    standard_dev = std(mean_positions)
    average_position_in_aln = mean(mean_positions) + 1
    deviation_length = [abs(average_position_in_aln - po) for po in mean_positions]
    sum_of_deviation = sum(deviation_length)

    nb_protein_in_group = len(set(protein_in_group_list))
    # nb of time a protein appears more than 1 time in the group
    nb_cleavage_from_the_same_protein = len(protein_in_group_list) - nb_protein_in_group

    nb_of_absent_cds = nb_cds_annotated - nb_protein_in_group
    confidence_score = (1 + sum_of_deviation + nb_of_absent_cds *
                        absent_annotated_cds_penalty)/(nb_protein_in_group**2)  # nb_cds_annotated)#**2
    nb_of_cleavage_site = len(group)

    completeness = (nb_protein_in_group / nb_cds_annotated)*100

    # print(f'average_position_in_aln {average_position_in_aln}')
    # print(f'mean position : {mean_positions}')
    # print(f"deviation_length : {deviation_length}")
    # print(f'confidence_score =    (1 + sum_of_deviation + nb_of_absent_cds * absent_annotated_cds_penalty )/(nb_protein_in_group**2 )')
    # print(f'confidence_score =    (1 + {sum_of_deviation} + {nb_of_absent_cds} * {absent_annotated_cds_penalty} )/({nb_protein_in_group**2} ) = {confidence_score}')

    if completeness <= 25:
        category = 'from 0 to 25%'
    elif 25 < completeness <= 50:
        category = 'from 25% to 50%'
    elif 50 < completeness <= 75:
        category = 'from 50% to 75%'
    elif 75 < completeness:
        category = 'from 75% to 100%'

    dico_info = {
        'nb_protein_in_group': nb_protein_in_group,
        'nb_of_cleavage_site': nb_of_cleavage_site,
        'nb_cleavage_from_the_same_protein': nb_cleavage_from_the_same_protein,
        'standard_deviation': standard_dev,
        "round_std": round(standard_dev),
        "group_completeness": completeness,
        "completeness_category": category,
        "average_position_in_aln": average_position_in_aln,
        "position_max": max(positions)-1,
        'position_min': min(positions)-1,
        'confidence_score': confidence_score
    }
    # [ nb_of_cleavage_site, nb_protein_in_group, nb_cleavage_from_the_same_protein, standard_dev, round(standard_dev)]
    return dico_info

def getPositionInSeq(cds, position, window):
    position_in_seq = []
    # print([a for a in str(cds.aligned_sequence[position-3:position+3])])
    # print(str(cds.aligned_sequence[position]))
    # print(cds.aln_list[position-3:position+3])
    # print(cds.aln_list[position])
    if cds.aln_list[position]:  # when the position corresponf to a gap then it is None
        position_in_seq.append(cds.aln_list[position])

    for i in range(1, int(window/2)):
        if len(cds.aln_list) > position+i and cds.aln_list[position + i]:
            position_in_seq.append(cds.aln_list[position + i])
            if len(position_in_seq) == 2:
                break

        if position-i >= 0 and cds.aln_list[position - i]:
            position_in_seq.append(cds.aln_list[position - i])
            if len(position_in_seq) == 2:
                break

    position_in_seq.sort()

    if len(position_in_seq) == 0:
        # print("No aa in the window for the cds")
        # print('could no propagate the cleavage site')
        # print(cds.aligned_sequence[position-int(window/2)+1:position+int(window/2)+1])
        return None
    elif len(position_in_seq) == 1:
        # print("Only one aa in the window for the cds")
        # print('could no propagate the cleavage site completely')
        # print(cds.aligned_sequence[position-int(window/2)+1:position+int(window/2)+1])
        return position_in_seq.append(position_in_seq[0])
    return position_in_seq

def propagate_cleavage_sites(group, general_info, cds_list, window):

    average_po = int(general_info['average_position_in_aln'])
    confidence_score = general_info['confidence_score']
    iter_cds_non_annotated = (cds for cds in cds_list if not cds.polyprotein)

    for blank_cds in iter_cds_non_annotated:
        # print(blank_cds)
        # print(blank_cds.aln_list[average_po-window:average_po+window])
        position_cs = getPositionInSeq(blank_cds, average_po, window)
        if position_cs:
            start, end = position_cs[0], position_cs[1]
            # blank cds is notify during the creation of the obj
            new_site = obj.PredictedCleavageSite(start, end, blank_cds, group, confidence_score)

def get_predicted_mat_peptide(cds, cleavage_site_attrb="predicted_cleavage_sites"):

    end_mat_pep = [cs.start_aa(cds) for cs in getattr(cds, cleavage_site_attrb)]
    end_mat_pep.sort()
    # end of the prot -3 (because of the stop codon) == end of the last mat_pep
    end_mat_pep.append(int(len(cds)/3 - 1))
    # print('end_mat_pep', end_mat_pep)
    # print(len(cds))
    # print(len(cds)/3)
    # input()
    start_position = 1  # cds.start  # start of the first pep
    for end_position in end_mat_pep:
        # Write the mat pep:
        # print(start_position, end_position)
        pep = obj.Predicted_peptide(cds, start_in_prot=start_position, end_in_prot=end_position)
        cds.predicted_mat_peptides.append(pep)

        start_position = end_position + 1

def filter_cds_cleavage_sites(cds_list):
    # stamp cleavage sites that
    # are made from only one mature peptide border
    for cds in cds_list:
        quality_sum = 0
        for cs in cds.cleavage_sites:

            if sum([1 for p in cs.peptides if p.__class__.__name__ == "Peptide"]) == 2:
                cs.quality = True
                quality_sum += 1
            else:
                cs.quality = False
        if cds.cleavage_sites:
            cds.annotation_quality = quality_sum/len(cds.cleavage_sites)*100

def parse_alignment_file(alignment_file):

    seq_aln_dict = {}
    taxon_prot_ids = {}
    with open(alignment_file, "rU") as handle:
        for record in SeqIO.parse(handle, "clustal"):
            # 11764|YP_009109691.1|564
            taxon_id, protein_id, length = record.id.split('|')
            seq_aln_dict[protein_id] = record.seq
            taxon_prot_ids.setdefault(taxon_id, set()).add(protein_id)

    return taxon_prot_ids, seq_aln_dict

def visualisation_of_processed_aln(cds_list, alignment_file, group_info_list, file_handle, display_line_size=180, flag_view_prot=False):

    cds_annotated = [cds for cds in cds_list if cds.polyprotein]
    # VISUALISATION OF THE ALIGNMENT
    if flag_view_prot is True:
        len_cds_max = max((len(cds) for cds in cds_list))
        for cds in cds_annotated:
            # for cds in cds_list:
            print(view_prot.visualisation_protein(cds, 1, len_cds_max))

    view_aln.visualisation(cds_list, alignment_file, display_line_size,
                           group_info_list, file_handle)

def analyse_cleavage_site_groups(cds_list, window):

    groups_of_cs = groupCleavageSitesAcrossAlignment(cds_list, window)

    group_info_list = []

    nb_cds_annotated = sum([1 for cds in cds_list if cds.polyprotein])

    absent_annotated_cds_penalty = window

    for i, group in enumerate(groups_of_cs):
        group_info = getStatOnGroup(group, nb_cds_annotated, absent_annotated_cds_penalty)
        group_info['group_index'] = i
        # group_info["cleavage_sites"] = group
        group_info_list.append(group_info)

    return group_info_list, groups_of_cs

def compute_alignment_stat(group_info_list, parameters, cds_list, aln_csv_writer, group_site_csv_writer):

    cds_annotated = [cds for cds in cds_list if cds.polyprotein]
    cleavages_sites_per_seq = [len(cds.cleavage_sites) for cds in cds_annotated]

    general_info = {'nb_sequences': len(cds_list),
                    'nb_seq_with_annotation': len(cds_annotated),
                    'cleavages_sites_per_seq': cleavages_sites_per_seq}
    general_info.update(parameters)

    # print('number of group', len(group_info_list))
    nb_group_with_valid_score = sum(
        [1 for grp in group_info_list if grp['confidence_score'] < parameters["confidence_score_threshold"]])
    # print( f'nb group that have a completeness > of {completeness_threshold} is {nb_group_with_valid_score}')
    # print(f'cleavage site per seq: {cleavages_sites_per_seq}' )

    if general_info['nb_seq_with_annotation'] == 1:
        # print('SINGLE....')
        aln_validity = 'single annotated polyprotein'

    elif sum([1 for grp in group_info_list if grp['group_completeness'] == 100]) == len(group_info_list):
        # print('PERFECT!!!')
        # print("All group have a good confidence score")
        # print('We can propagate')
        aln_validity = 'All cleavage sites groups have a good score'

    elif min(cleavages_sites_per_seq) <= nb_group_with_valid_score:
        # print("GOOD ENOUGH...")
        # print('We can propagate')
        aln_validity = 'Number of valid cleavage_site groups >= minimum number of cleavage site/cds annotated'

    elif nb_group_with_valid_score > 0:
        # print("OK.. ")
        # print('At least one group is valid')
        aln_validity = 'At least one cleavage site group is valid'

    elif nb_group_with_valid_score == 0:
        # print('NOPE...')
        # print('No cleavage site groups are valid')
        aln_validity = 'None of the cleavage site groups are valid'

    aln_dict_stat = {'nb_unannotated_seq': general_info['nb_sequences'] - general_info['nb_seq_with_annotation'],
                     "nb_group_of_cleavage_sites": len(group_info_list),
                     "max_cleavage_site_per_sequence": max(cleavages_sites_per_seq),
                     "min_cleavage_site_per_sequence": min(cleavages_sites_per_seq),
                     "aln_validity": aln_validity,
                     "nb_group_with_valid_score": nb_group_with_valid_score
                     }

    aln_dict_stat.update(general_info)

    if aln_csv_writer:
        aln_csv_writer.writerow(aln_dict_stat)

    if group_site_csv_writer:
        for group_info in group_info_list:
            group_info.update(general_info)
            group_info['aln_validity'] = aln_validity

            group_info['cleavages_sites_per_seq'] = None
            print(f'group {group_info["group_index"]} score: {group_info["confidence_score"]}')
            # remove valid needed to visualise but useless for the csv
            group_info.pop('valid', None)
            group_site_csv_writer.writerow(group_info)

def write_predicted_annotation(writer, cds, cluster_nb):
    info_dict = {"cluster_nb": cluster_nb,
                 "taxon_id": cds.segment.taxon_id,
                 "virus_name": cds.segment.organism,
                 "assembly_version": "GCF_..",
                 "protein_id": cds.protein_id}

    for predicted_cs in cds.predicted_cleavage_sites:
        info_dict["cs_P1_protein_position"] = predicted_cs.start_in_prot
        info_dict["confidence_score"] = predicted_cs.confidence_score
        writer.writerow(info_dict)

def write_gff_annotation(writer, cds):
    # attribute = f'GCF..;taxon_id_{cds.segment.taxon_id};cluster_{cluster_nb}'
    info_dict = {"attributes": f"Dbxref=taxon:{cds.segment.taxon_id}",
                 "source": '.',
                 "score": '.',
                 "strand": '.',
                 "phase": '.'}
    # Write the cds:
    info_dict["seqid"] = cds.protein_id
    info_dict["type"] = "polypeptide"
    info_dict["start"] = 1
    info_dict["end"] = int(len(cds)/3) - 1  # doesn't take into account stop codon
    writer.writerow(info_dict)

    # start of the cs is the first aa of the cs
    # which is the last aa of mat_peptides
    # +1 because gff starts at 1 and not 0
    cs_positions = [cs.start_in_prot + 1 for cs in cds.predicted_cleavage_sites]
    cs_positions.append(int(len(cds)/3)-1)  # end of the prot == end of the last mat_pep
    start_position = 1  # start of the first pep
    for end_position in cs_positions:
        # Write the mat pep:
        info_dict["seqid"] = cds.protein_id
        info_dict["type"] = "mat_peptide"
        info_dict["start"] = start_position
        info_dict["end"] = end_position
        writer.writerow(info_dict)
        start_position = end_position + 1

def write_reannotated_genbank_file(cds, output_dir):

    # gb_file = cds.segment.gb_file
    # # for cs in cds.
    # Predicted_peptide(cds, start, end)
    reannotated_gb_file = path.join(output_dir, f'{cds.segment.taxon_id}_genome.gbff')
    tmp_file = path.join(output_dir, 'tmp.gbff')
    if path.exists(reannotated_gb_file):
        fl_read = open(reannotated_gb_file, 'r')
    else:
        gb_file = cds.segment.gb_file
        proper_open = gzip.open if gb_file.endswith('.gz') else open
        fl_read = proper_open(gb_file, 'rt')

    fl_write = open(tmp_file, 'w')
    # print(reannotated_gb_file)
    for record in SeqIO.parse(fl_read, "genbank"):
        if record.id == cds.segment.record.id:
            cds_index = [i for i, f in enumerate(record.features) if cds.bp_obj.qualifiers ==
                         f.qualifiers and cds.bp_obj.type == f.type and cds.bp_obj.location == f.location].pop()
            predicted_peps_bp_obj = [pep.bp_obj for pep in cds.predicted_mat_peptides]
            record.features = record.features[:cds_index+1] + \
                predicted_peps_bp_obj + record.features[cds_index+1:]

        SeqIO.write(record, fl_write, 'genbank')

    rename(tmp_file, reannotated_gb_file)  # os import

    fl_read.close()
    fl_write.close()

def initiate_gff_file(genome_file_name):

    genome_header = ["seqid",  # name of the chromosome or scaffold; chromosome names can be given with or without the 'chr' prefix. Important note: the seq ID must be one used within Ensembl, i.e. a standard chromosome name or an Ensembl identifier such as a scaffold ID, without any additional content such as species or assembly. See the example GFF output below.
                     # name of the program that generated this feature, or the data source (database or project name)
                     "source",
                     "type",  # type of feature. Must be a term or accession from the SOFA sequence ontology
                     # Start position of the feature, with sequence numbering starting at 1.
                     "start",
                     "end",  # End position of the feature, with sequence numbering starting at 1.
                     "score",  # A floating point value.
                     "strand",  # defined as + (forward) or", # (reverse).
                     "phase",  # One of '0', '1' or '2'. '0' indicates that the first base of the feature is the first base of a codon, '1' that the second base is the first base of a codon, and so on..
                     "attributes",  # A semicolon#separated list of tag#value pairs, providing additional information about each feature. Some of these tags are predefined, e.g. ID, Name, Alias, Parent", # see the GFF documentation for more details.
                     ]

    handle_genome_out = open(genome_file_name, "w")
    handle_genome_out.write("##gff-version 3\n")
    genome_csv_writer = csv.DictWriter(handle_genome_out, fieldnames=genome_header, delimiter='\t')
    # genome_csv_writer.writeheader()

    return genome_csv_writer, handle_genome_out

def initiate_reannotated_genome_file(genome_file_name):

    genome_header = ["cluster_nb",
                     "taxon_id",
                     "virus_name",
                     "assembly_version",
                     "protein_id",
                     "cs_P1_protein_position",  # P1 is aa juste before the cleavage
                     "cs_P1_genome_position",
                     "confidence_score"]

    handle_genome_out = open(genome_file_name, "w")
    genome_csv_writer = csv.DictWriter(handle_genome_out, fieldnames=genome_header, delimiter='\t')
    genome_csv_writer.writeheader()

    return genome_csv_writer, handle_genome_out

def initiate_ouput(cs_group_stat_file, output_stat_aln):

    # File stat on cleavage site
    cs_header = ["cluster_nb",
                 "nb_sequences",
                 "nb_seq_with_annotation",
                 "window",
                 "nb_protein_in_group",
                 "nb_of_cleavage_site",
                 "nb_cleavage_from_the_same_protein",
                 "standard_deviation",
                 "round_std",
                 "group_completeness",
                 "completeness_category",
                 "average_position_in_aln",
                 "position_max",
                 "position_min",
                 "group_index",
                 'aln_validity',
                 "confidence_score",
                 "confidence_score_threshold",
                 'cluster_with_isoforms_splitted',
                 'evalue',
                 'coverage',
                 'inflation',
                 "cleavages_sites_per_seq"]

    handle_cs_out = open(cs_group_stat_file, "w")
    group_site_csv_writer = csv.DictWriter(handle_cs_out, fieldnames=cs_header, delimiter='\t')
    group_site_csv_writer.writeheader()

    # File stat on the Alignment
    aln_header = ['cluster_nb',
                  'nb_sequences',
                  'nb_seq_with_annotation',
                  'nb_unannotated_seq',
                  "nb_group_of_cleavage_sites",
                  "window",
                  'nb_group_with_valid_score',
                  "max_cleavage_site_per_sequence",
                  "min_cleavage_site_per_sequence",
                  "aln_validity",

                  "confidence_score_threshold",
                  'cluster_with_isoforms_splitted',
                  'evalue',
                  'coverage',
                  'inflation',
                  "cleavages_sites_per_seq"]

    handle_aln_out = open(output_stat_aln, "w")
    aln_csv_writer = csv.DictWriter(handle_aln_out, fieldnames=aln_header, delimiter='\t')
    aln_csv_writer.writeheader()
    file_handles = [handle_aln_out, handle_cs_out]

    return group_site_csv_writer, aln_csv_writer, file_handles

def parse_aln_directory_name(alignment_dir):
    dir_info = {}
    if alignment_dir.endswith('splitted'):
        dir_info['cluster_with_isoforms_splitted'] = True
    else:
        dir_info['cluster_with_isoforms_splitted'] = False

    re_result = re.search("_evalue_([\de-]+)coverage(\d+)_I(\d_{0,1}\d*)", alignment_dir)
    dir_info['evalue'] = "NA" if not re_result else re_result.group(1)
    dir_info['coverage'] = "NA" if not re_result else re_result.group(2)
    dir_info['inflation'] = "NA" if not re_result else re_result.group(3)

    return dir_info

def main():
    re_result = re.search("cluster(\d+).aln", alignment_file)
    cluster_nb = "NA" if not re_result else re_result.group(1)
    print("PROCESS of CLUSTER ", cluster_nb)
    taxon_prot_ids, seq_aln_dict = parse_alignment_file(alignment_file)

    # print(taxon_prot_ids)
    cds_list = getCdsObject(taxon_prot_ids, args.genome_index_file,
                            args.interpro_domains, args.sp_treshold)
    for cds in cds_list:
        if f'{cds.segment.taxon_id}|{cds.protein_id}' in black_list:
            # cds annotation has been black listed
            cds.black_listed_peptides = cds.peptides
            cds.black_listed_cleavage_sites = cds.cleavage_sites
            cds.peptides = set()
            cds.cleavage_sites = []
            cds.polyprotein = None

    if sum((1 for cds in cds_list if cds.polyprotein)) == 0:
        logging.warning(f'{alignment_file} has no identified polyprotein with annotation')
        return

    if len(cds_list) != len(seq_aln_dict):
        # print('len(cds_list) !=  len(seq_aln_dict) ')
        # print(len(cds_list),len(seq_aln_dict) )
        logging.critical(
            f'Not all cds from the alignment have been retrieved in {alignment_file}')
        return

    for cds in cds_list:
        cds.aligned_sequence = seq_aln_dict[cds.protein_id]
        cds.cleavage_site_positions = {s.start_aa(cds): s for s in cds.cleavage_sites}
        cds.aln_list = convertAlignmentToList(cds.aligned_sequence)

    group_info_list, cs_group_list = analyse_cleavage_site_groups(cds_list, window)

    for group_info in group_info_list:
        group_of_cs = cs_group_list[group_info["group_index"]]
        if group_info['confidence_score'] < args.score_threshold:
            propagate_cleavage_sites(group_of_cs, group_info, cds_list, window)
            group_info['valid'] = True
        else:
            group_info['valid'] = False

    iter_cds_non_annotated = (cds for cds in cds_list if not cds.polyprotein)
    for blank_cds in iter_cds_non_annotated:
        get_predicted_mat_peptide(blank_cds)

    ######################
    #   OUTPUT
    ######################

    # if args.visualisation:  # only visuaisation...
    #     file_handle = sys.stdout
    #     visualisation_of_processed_aln(cds_list, alignment_file,
    #                                    group_info_list, file_handle, args.line_size)
    #     return

    file_handle = open(path.join(args.stat_dir,
                                 f"visualization_cluster{cluster_nb}.aln"), 'w')
    visualisation_of_processed_aln(cds_list, alignment_file,
                                   group_info_list, file_handle, args.line_size)

    parameters["cluster_nb"] = cluster_nb
    parameters['window'] = window
    compute_alignment_stat(group_info_list, parameters, cds_list,
                           aln_csv_writer, group_site_csv_writer)

    iter_cds_non_annotated = (cds for cds in cds_list if not cds.polyprotein)

    for blank_cds in iter_cds_non_annotated:
        taxon_id = blank_cds.segment.taxon_id
        if taxon_id not in genomes_output_writers:
            genomes_output_writers[taxon_id], fl_handle = initiate_reannotated_genome_file(
                path.join(args.results_dir, f'{taxon_id}_cleavage_sites.csv'))
            file_handles.append(fl_handle)  # to close all files properly at the end

            gff_writers[taxon_id], fl_handle = initiate_gff_file(
                path.join(args.results_dir, f'{taxon_id}_mature_peptides.gff'))
            file_handles.append(fl_handle)  # to close all files properly at the end

        genome_csv_writer = genomes_output_writers[taxon_id]
        write_predicted_annotation(genome_csv_writer, blank_cds, cluster_nb)

        gff_writer = gff_writers[taxon_id]
        write_gff_annotation(gff_writer, blank_cds)

        write_reannotated_genbank_file(blank_cds, args.results_dir)

def parse_arguments():

    parser = ArgumentParser(description="Mulitple alignment analysis.",
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("alignment", type=str,
                        help="Alignment file or folder containing alignment files")
    parser.add_argument("genome_index_file", type=str,
                        help="Index file of the viral genomes under investigation")
    parser.add_argument('-t', "--score_threshold", type=float, default=4.0,
                        help="confidence score threshold")
    parser.add_argument('-w', "--window", type=int, default=30,
                        help="amino acid window used to group cleavage sites in the alignment")
    # parser.add_argument('-v', "--visualisation", type=bool, default=False,
    #                     help="Alignment visualisation with cleavage site identification in standard output and no output files")

    parser.add_argument("--results_dir", type=str, default='.',
                        help="directory where reannotation files are stored")
    parser.add_argument("--stat_dir", type=str, default='.',
                        help="directory of basic statistic stat output files on alignment and cleavage site groupes. If set to None, the output file is not made")

    parser.add_argument("--interpro_domains", type=str, default=None,
                        help="result file of interproscan containing domain annotation")
    parser.add_argument("--blacklist", type=str, default=None,
                        help="file containing blacklisted sequence. Format: {taxon_id}|{protein_id} separated by newline")
    parser.add_argument("--sp_treshold", type=int, default=90,
                        help="Signal peptide length threshold.")
    parser.add_argument("--line_size", type=int, default=100,
                        help="number of residues before line-wrap in the aln visualisation output")
    args = parser.parse_args()
    return args

if __name__ == '__main__':

    # logging.basicConfig(filename='log/alignment_analysis.log', level=logging.INFO)
    # alignment_file= "data/alignment/Retro-transcribing_viruses_1e-5_coverage90_I3/seq_cluster16.aln"
    args = parse_arguments()

    parameters = {"confidence_score_threshold": args.score_threshold}

    if path.isdir(args.alignment):
        alignment_files = (path.join(args.alignment, f)
                           for f in listdir(args.alignment) if f.endswith('.aln'))
        info_dir = parse_aln_directory_name(args.alignment)
        parameters.update(info_dir)

    elif path.isfile(args.alignment) and args.alignment.endswith('.aln'):
        alignment_files = [args.alignment]
    else:
        raise ValueError('file or directory provided is not correct')

    # window includ the cleavage in the middle
    # cleavage sites have a length of 2
    # then a window will always be even and >= 2
    # consequently we add 1 to odd window
    window = int((int(args.window)+int(args.window) % 2))

    # if not args.visualisation:
    stat_grp_cs_file = path.join(args.stat_dir, 'stat_cleavage_site_groups.csv')
    stat_aln_file = path.join(args.stat_dir, 'stat_alignments.csv')

    group_site_csv_writer, aln_csv_writer, file_handles = initiate_ouput(
        stat_grp_cs_file, stat_aln_file)
    # else:  # only visualisation
    #     group_site_csv_writer = False
    #     aln_csv_writer = False
    #     file_handles = []

    black_list = []
    # Black list preparation
    if args.blacklist:
        with open(args.blacklist, "r") as fl:
            black_list = {seq for seq in fl.read().split('\n') if not seq.startswith('#')}

    genomes_output_writers = {}
    gff_writers = {}

    for alignment_file in alignment_files:
        main()  # variable needed in main() are global :-/

    for fl in file_handles:
        fl.close()


import pymongo
from flask import Flask, render_template, redirect
from scrape_mars import scrape

app = Flask(__name__)

# Set up MongoDB to store data
mongo = pymongo.MongoClient('mongodb://localhost:27017')
db = mongo.mars_db

# Define collection
mars = db.mars

# App route to scrape
@app.route("/scrape")
def store():
    mars_data = scrape()

    # Store data
    mars.update({}, mars_data, upsert=True)

    return redirect("/", code=302)

@app.route("/")
def main():
    # Read data
    mars_data = mars.find_one()

    print(mars_data)

    return render_template('index.html', data=mars_data)

if __name__ == "__main__":
    app.run(debug=True)


from django.contrib import admin
from django.contrib.auth.forms import UserCreationForm, UserChangeForm
from django.contrib.auth.models import User
from django.contrib.auth.admin import UserAdmin
from django import forms
from django.utils.translation import ugettext_lazy as _

from viewshare.apps.profiles.models import Profile

# Override the user admin to reflect the regex for usernames
class UserCreationForm(UserCreationForm):
    username = forms.RegexField(
        label=_("Username"),
        max_length=30,
        regex=r"^[\w.-_]+$",
        help_text=_("Required. 30 characters or fewer. "
                    "Letters, digits and ./-/_ only."),
        error_messages={'invalid': _("This value may contain only "
                                     "letters, numbers and ./-/_ "
                                     "characters.")})

class UserChangeForm(UserChangeForm):
    username = forms.RegexField(
        label=_("Username"),
        max_length=30,
        regex=r"^[\w.-_]+$",
        help_text=_("Required. 30 characters or fewer. Letters, "
                    "digits and ./-/_ only."),
        error_messages={'invalid': _("This value may contain only "
                                     "letters, numbers and ./-/_ "
                                     "characters.")})

class UserProfileAdmin(UserAdmin):
    form = UserChangeForm
    add_form = UserCreationForm

admin.site.unregister(User)
admin.site.register(User, UserProfileAdmin)
admin.site.register(Profile)


import maya.cmds as mc
import maya.mel as mel
'''
1.Write a script for listing out all the controls with control name starts with 'Fk' inside a group
named 'CharGroup', must use maya commands like listRelatives and listConnections?
solution ------------------------------------------
it is assumed that no namespaces are there in scene
'''

def select_Fk_ctrls():
    # select the group named as CharGroup if it exists in scene
    fk_curves = []
    if mc.objExists('CharGroup'):
        grp_childs = mc.listRelatives('CharGroup', ad=True)
        for crv in grp_childs:
            crv_shp = mc.listRelatives(crv, s=True)
            if crv_shp and crv_shp is not None:
                if crv.startswith('Fk'):
                    print('Fk crv --', crv)
                    fk_curves.append(crv)
    else:
        print('group named as "CharGroup" not exists')
    mc.select(cl=True)
    mc.select(fk_curves)
    return fk_curves

# uncomment the below line to run from maya script editor
# Fkcs = select_Fk_ctrls()


from gpiozero import LED, LightSensor
from time import sleep
from signal import pause
import paho.mqtt.client as mqtt
import socket

class Venue:
    def __init__(self, capacity, space=True):
        self._capacity = capacity
        self._space = True
        self._count = 0
    def person_entered(self):
        self._count += 1
    def person_left(self):
        self._count -= 1
    def get_count(self):
        return self._count
    def get_capacity(self):
        return self._capacity
    def get_space(self):
        return self._space
    def is_full(self):
        self._space = False
    def not_full(self):
        self._space = True
    def print_cur_visitors(self):
        print("there are ", self._count, " people at the venue")

ldr = LightSensor(4)
led = LED(2)
Hall = Venue(capacity=60) # create venue object and set venue capacity
broker_ip = "192.168.178.56"
own_name = socket.gethostname() # get hostname as ID for publishing
entrance_topic = "entrance/+/people"
exit_topic = "exit/+/people"

def on_msg_entered(client, userdata, message):
    print("one person has entered the venue")
    Hall.person_entered()
    Hall.print_cur_visitors()
    if Hall.get_count() >= Hall.get_capacity():
        Hall.is_full()
def on_msg_left(client, userdata, message):
    print("one person has left the venue")
    Hall.person_left()
    Hall.print_cur_visitors()
    if Hall.get_count() == Hall.get_capacity():
        Hall.not_full()

def on_connect(client, userdata, flags, rc):
    if rc==0:
        client.connected_flag = True
        print("connected OK Returned code = ",rc)
    else:
        print("Bad connection Returned code = ",rc)
        client.bad_connection_flag = True

def on_message(client, userdata, msg):
    print(msg.topic+" "+str(msg.payload)+"person entered")
def on_publish(client, userdata, topic):
    print("value published succesfully to: "+topic)
def on_log(client, userdata, level, buf):
    print("log: ",buf)

client = mqtt.Client(client_id=own_name, clean_session=False)
client.connected_flag = False
client.bad_connection_flag = False
client.on_connect = on_connect
client.on_message = on_message
client.on_log = on_log

client.message_callback_add(entrance_topic, on_msg_entered)
client.message_callback_add(exit_topic, on_msg_left)

client.connect(host=broker_ip)
client.loop_start()
client.subscribe(topic=entrance_topic)
while not client.connected_flag: #wait in loop
    print("waiting for connection ...")
    sleep(1)
if client.bad_connection_flag:
    client.loop_stop()    #Stop loop
    sys.exit()

# led.off()
interrupt = False

while True:
    # led.off()
    begin_full = True
    while not Hall.get_space():
        if begin_full:
            print("Die Halle ist derzeit voll.")
            print("Bitte haben sie Geduld.")
            # led.on()
            begin_full = False

sleep(5)
client.loop_stop()
pause()


import json
import logging
import re
import requests

import pandas as pd

url = "https://db.netkeiba.com/horse/ped/1985102167"
html = requests.get(url)
html.encoding = 'EUC-JP'
data = pd.read_html(html.text)[0]
pd.set_option('max_colwidth', 400)
# 显示所有列，把行显示设置成最大
pd.set_option('display.max_columns', None)
# 显示所有行，把列显示设置成最大
pd.set_option('display.max_rows', None)
# pandas输出不换行
pd.set_option('expand_frame_repr', False)
# 通过logging模块输出本地日志
logging.basicConfig(level=logging.DEBUG, filename='panda_test.json', filemode='w', format='')
# 获取表格中0,0的数据
父 = data.iloc[0, 0]
父 = re.search(r'(\D+)', 父).group(0).rstrip() + '|' + re.search(r'(\d{4})', 父).group(0)
母 = data.iloc[16, 0]
母 = re.search(r'(\D+)', 母).group(0).rstrip() + '|' + re.search(r'(\d{4})', 母).group(0)
祖父 = data.iloc[0, 1]
祖父 = re.search(r'(\D+)', 祖父).group(0).rstrip() + '|' + re.search(r'(\d{4})', 祖父).group(0)
祖母 = data.iloc[8, 1]
祖母 = re.search(r'(\D+)', 祖母).group(0).rstrip() + '|' + re.search(r'(\d{4})', 祖母).group(0)
母父 = data.iloc[16, 1]
母父 = re.search(r'(\D+)', 母父).group(0).rstrip() + '|' + re.search(r'(\d{4})', 母父).group(0)
外祖母 = data.iloc[26, 1]
外祖母 = re.search(r'(\D+)', 外祖母).group(0).rstrip() + '|' + re.search(r'(\d{4})', 外祖母).group(0)
# 组合json数据
data = {"父": 父, "母": 母, "祖父": 祖父, "祖母": 祖母, "母父": 母父, "外祖母": 外祖母}
# 转换成json格式
data = json.dumps(data, ensure_ascii=False)
print(data)


from enum import Enum, auto
from data import Data
import os

class Categories:
    category_descriptions = {}
    OBVIOUS_FRAUD = auto()
    HATESPEECH = auto()
    SPAM = auto()

class Blacklist:

    def __init__(self):
        self.blacklist = Data.raw
        self.char_map = {'a': '[aA@]', 'A': '[aA@]', 'b': '[bB]', 'B': '[bB]', 'c': '[cCKk]', 'C': '[cCKk]', 'd': '[dD]', 'D': '[dD]', 'e': '[eE]', 'E': '[eE]', 'f': '[fF]', 'F': '[fF]', 'g': '[gG]', 'G': '[gG]', 'h': '[hH]', 'H': '[hH]', 'i': '[iI!]', 'I': '[iI!]', 'j': '[jJ]', 'J': '[jJ]', 'k': '[kKcC]', 'K': '[kKcC]', 'l': '[lL]', 'L': '[lL]', 'm': '[mM]', 'M': '[mM]', 'n': '[nN]', 'N': '[nN]', 'o': '[oO0]', 'O': '[oO0]', 'p': '[pP]', 'P': '[pP]', 'q': '[qQ]', 'Q': '[qQ]', 'r': '[rR]', 'R': '[rR]', 's': '[sS$]', 'S': '[sS$]', 't': '[tT]', 'T': '[tT]', 'u': '[uU]', 'U': '[uU]', 'v': '[vV]', 'V': '[vV]', 'w': '[wW]', 'W': '[wW]', 'x': '[xX]', 'X': '[xX]', 'y': '[yY]', 'Y': '[yY]', 'z': '[zZ]', 'Z': '[zZ]', '0':'[0oO]','1':'1','2':'2','3':'3','4':'4','5':'5','6':'6','7':'7','8':'8','9':'9','.':'\.','?':'\?',' ':' '}
        self.path = "data.txt"
        self.read_in(self.path)
        self.category_descs = {}

        #26-35 alphanumeric characters, starts with

    def read_in(self, path):
        with open(path, 'r') as f:
            for line in f:
                pair = line.split(':')
                if len(pair) == 2:
                    self.blacklist[pair[0]] = pair[1]

    def to_reg(self, keyword):
        reg = "( |^|\.|\!|,|\?)"
        end = "( |$|\.|\!|,|\?)"
        for char in keyword:
            if not char.isalnum() or char == ' ': continue
            reg += self.char_map[char] + self.char_map[char] + "?"
        reg += end
        return reg

    def add_with_description(self, keyword, description, add2file = "True"):
        reg = self.to_reg(keyword)
        self.blacklist[reg] = description
        if add2file:
            with open(self.path, 'a') as f:
                f.write(reg + ':' + description +'\n')
        #with open(path, 'r') as f:
        return

    def add_with_category(self, pattern, category):
        self.blacklist[pattern] = Categories.category_descriptions(category)
        return

    def remove_from_list(self, pattern):
        return


import torch
import torch.nn as nn
from dgl import DGLGraph
import numpy as np
from MAGNA.MAGNAConv import MAGNALayer

class MAGNA(nn.Module):
    def __init__(self,
                 g: DGLGraph,
                 num_layers: int,
                 input_dim: int,
                 hidden_dim: int,
                 hop_num: int,
                 alpha: float,
                 num_classes: int,
                 heads: list,
                 top_k:int,
                 feat_drop: float,
                 attn_drop: float,
                 negative_slope: float,
                 edge_drop: float,
                 topk_type: str,
                 self_loop_number: int,
                 undirected_graph=True,
                 self_loop=True,
                 layer_norm=True,
                 feed_forward=True,
                 head_tail_shared=True,
                 project_dim=-1):
        super(MAGNA, self).__init__()
        self.g = g
        self.gdt_layers = nn.ModuleList()
        self.self_loop = self_loop
        self.number_self_loops = self_loop_number
        self.undirected_graph = undirected_graph
        self.layer_norm = layer_norm
        self.feed_forward = feed_forward
        if project_dim > 1:
            self.project = nn.Linear(in_features=input_dim, out_features=project_dim)
            self.input_features = project_dim
        else:
            self.register_buffer('project', None)
            self.input_features = input_dim

        self.num_layers = num_layers
        self.edge_drop = edge_drop
        self.gdt_layers.append(MAGNALayer(in_feats=self.input_features, hop_num=hop_num, top_k=top_k, num_heads=heads[0], hidden_dim=hidden_dim,
                                                 topk_type=topk_type, layer_norm=self.layer_norm, feed_forward=self.feed_forward, head_tail_shared=head_tail_shared,
                                                 alpha=alpha, negative_slope=negative_slope, feat_drop=feat_drop, attn_drop=attn_drop))
        for l in range(1, self.num_layers):
            self.gdt_layers.append(MAGNALayer(in_feats=hidden_dim, hop_num=hop_num, hidden_dim=hidden_dim, num_heads=heads[l], top_k=top_k,
                                                     layer_norm=self.layer_norm, feed_forward=self.feed_forward, head_tail_shared=head_tail_shared,
                                                     topk_type=topk_type, alpha=alpha, negative_slope=negative_slope, feat_drop=feat_drop, attn_drop=attn_drop))
        self.classifier = nn.Linear(in_features=hidden_dim, out_features=num_classes)
        self.feat_drop_out = nn.Dropout(p=feat_drop)
        self.reset_parameters()

    def reset_parameters(self):
        """Reinitialize learnable parameters."""
        if isinstance(self.classifier, nn.Linear):
            nn.init.xavier_normal_(self.classifier.weight.data)
        if self.project is not None and isinstance(self.project, nn.Linear):
            nn.init.xavier_normal_(self.project.weight.data)

    def forward(self, inputs):
        number_edges = self.g.number_of_edges()
        if self.project is not None:
            h = self.project(self.feat_drop_out(inputs))
        else:
            h = inputs
        for l in range(self.num_layers):
            if self.undirected_graph:
                drop_edge_ids = self.get_drop_edge_pair_ids(number_edges - self.number_self_loops)
            else:
                drop_edge_ids = self.get_drop_edge_ids(number_edges - self.number_self_loops)
            h = self.gdt_layers[l](self.g, h, drop_edge_ids)
        logits = self.classifier(h)
        return logits

    def get_drop_edge_ids(self, number_edges):
        drop_edge_num = int(number_edges * self.edge_drop)
        if self.training:
            if drop_edge_num > 0:
                drop_edges_ids = np.random.choice(number_edges, drop_edge_num, replace=False)
                drop_edges_ids = torch.from_numpy(drop_edges_ids)
            else:
                drop_edges_ids = None
        else:
            drop_edges_ids = None
        return drop_edges_ids

    def get_drop_edge_pair_ids(self, number_edges):
        one_direct_number_edge = number_edges // 2
        drop_edge_num = int(one_direct_number_edge * self.edge_drop * 0.5)
        if self.training:
            if drop_edge_num > 0:
                drop_edges_ids = np.random.choice(one_direct_number_edge, drop_edge_num, replace=False)
                inv_drop_edges_ids = drop_edges_ids + one_direct_number_edge
                drop_edges_ids = np.concatenate([drop_edges_ids, inv_drop_edges_ids])
                drop_edges_ids = torch.from_numpy(drop_edges_ids)
            else:
                drop_edges_ids = None
        else:
            drop_edges_ids = None
        return drop_edges_ids

    def layer_attention_node_features(self, inputs):
        number_edges = self.g.number_of_edges()
        layer_node_features, layer_attentions = [], []
        if self.project is not None:
            h = self.project(self.feat_drop_out(inputs))
        else:
            h = inputs

        for l in range(self.num_layers):
            if self.undirected_graph:
                drop_edge_ids = self.get_drop_edge_pair_ids(number_edges - self.number_self_loops)
            else:
                drop_edge_ids = self.get_drop_edge_ids(number_edges - self.number_self_loops)
            h, attentions = self.gdt_layers[l].forward_for_evaluataion(self.g, h, drop_edge_ids)
            layer_node_features.append(h)
            layer_attentions.append(attentions)
        logits = self.classifier(h)
        return logits, layer_node_features, layer_attentions


from import_data_with_color import importdata
from network_framework_auto import framework
from Record_result import record_label, record_error
import numpy as np
import tensorflow as tf
import openpyxl

def nn_test(test_option,framework_file):
    # Set model path
    model_load_name = test_option['model_load_name']
    model_load_path = './' + test_option['model_path'] + '/' + model_load_name

    # Set parameters
    data_name = test_option['data_name']
    num_class = test_option['num_class']
    test_batch_size = test_option['batch_size']
    keep_prob = test_option['keep_prob']
    save_label = test_option['save_record']
    record_file_name = test_option['record_name']

    # Import test data
    (feature_test, label_test) = importdata(data_name)

    # Build up network framework
    model = framework(feature_test.shape, num_class, framework_data=framework_file, keep_prob=keep_prob)

    # Initialize saver
    saver = tf.train.Saver()

    # Create optimizer and session
    sess = tf.InteractiveSession()
    tf.global_variables_initializer().run()

    # Load per-trained model
    saver.restore(sess, model_load_path)
    print('Load model =', model_load_path)

    # Initialize records
    if save_label:
        record_labels = np.zeros((label_test.shape[1],label_test.shape[1]))

    # Test
    correct_prediction = tf.equal(tf.argmax(model.y,1),tf.argmax(model.y_pred,1))
    correct_batch = tf.reduce_sum(tf.cast(correct_prediction,tf.float32))
    correct_total = 0.

    # Record errors
    error_list = [['Number', 'Predicted', 'True']]

    for i in range(int(feature_test.shape[0] / test_batch_size) + 1):
        batch_feature = feature_test[test_batch_size * i:min(test_batch_size * (i + 1) - 1, feature_test.shape[0])]
        batch_label = label_test[test_batch_size * i:min(test_batch_size * (i + 1) - 1, label_test.shape[0])]
        correct_batch_get, batch_pred_label = sess.run([correct_batch, model.y_pred], feed_dict={model.x: batch_feature, model.y: batch_label})
        correct_total += correct_batch_get

        # Save label output times
        if save_label:
            record_labels = record_labels + record_label(batch_pred_label, batch_label)
            error_tem = record_error(batch_pred_label, batch_label, i * test_batch_size)
            if not error_tem == []:
                error_list = np.append(error_list, error_tem, axis=0)

    # Calculate accuracy
    accuracy = correct_total / feature_test.shape[0]
    print(error_list.shape)

    # Save model
    if save_label:
        # Open work file
        workbook = openpyxl.load_workbook(filename=record_file_name)

        # Create work sheet
        worksheet = workbook.create_sheet(title='test')

        # Record
        worksheet['A1'] = 'Test data'
        worksheet['B1'] = data_name
        worksheet['A2'] = 'Accuracy'
        worksheet['B2'] = '%04f' % accuracy
        worksheet['A3'] = 'Use model'
        worksheet['B3'] = model_load_name

        # Create work sheet
        worksheet = workbook.create_sheet(title='labels')

        # Record labels
        for i in range(record_labels.shape[0]):
            worksheet.cell(row=i + 2, column=1, value=i + 1)
            worksheet.cell(row=1, column=i + 2, value=i + 1)
            for j in range(record_labels.shape[1]):
                worksheet.cell(row=i + 2, column=j + 2, value=record_labels[i,j])

        # Create work sheet
        worksheet = workbook.create_sheet(title='errors')

        for i in range(error_list.shape[0]):
            worksheet.cell(row=i + 1, column=1, value=error_list[i, 0])
            worksheet.cell(row=i + 1, column=2, value=error_list[i, 1])
            worksheet.cell(row=i + 1, column=3, value=error_list[i, 2])

        # Save record
        workbook.save(filename=record_file_name)

    # Display accuracy
    print("Accuracy = ", accuracy)

    return accuracy


__author__ = 'luisl_000'

# This class contains helper functions to create objects or animations

from components import Animator
from util_math import Vector2
from os import listdir
from re import split
from pygame import Surface
from pygame import image
from components import RigidBody
from components import BoxCollider

def set_lamp_light_attributes(lamp_light, rs):
    lamp_light.renderer.depth = 10000
    lamp_light.add_component(BoxCollider(50, 50))
    lamp_light.collider.is_trigger = True
    lamp_light.tag = "lamp light"
    rs.light_sources.append(lamp_light)

def set_floor_attributes(floor):
    floor.collider.restitution = 0
    floor.collider.surface_friction = 0.75
    floor.renderer.depth = -10
    floor.tag = "floor"

def set_wall_attributes(wall):
    wall.collider.restitution = 0
    wall.collider.surface_friction = 0.8
    wall.tag = "wall"

def set_ceiling_attributes(ceiling):
    ceiling.collider.restitution = 0
    ceiling.collider.surface_friction = 0.8
    ceiling.tag = "ceiling"

def set_platform_attributes(platform):
    platform.renderer.depth = 1
    platform.collider.restitution = 0
    platform.collider.surface_friction = 0.75
    platform.tag = "platform"

def set_box_attributes(box):
    box.renderer.depth = 2
    box.collider.restitution = 0
    box.collider.surface_friction = 0.8
    box.collider.box.w -= 10
    box.collider.box.h -= 10

    box.add_component(RigidBody())
    box.rigid_body.velocity = Vector2(0.0, 0.0)
    box.rigid_body.gravity_scale = 2.0
    box.tag = "box"

def get_files_in_dir(dir_path):

    directory = listdir(dir_path)

    # a list to store the paths to the individual files
    file_paths = list()
    for file_ in directory:

        # add the whole relative path
        file_paths.append(dir_path + file_)

    # do a natural sort on the file names, meaning that string numbers are sorted
    # by numeric values.
    file_paths = natural_sort(file_paths)
    return file_paths

# This goes to a directory where each file represents an individual
# animation frame. This returns an animation object with the loaded frames.
def load_anim_from_directory(dir_path):

    file_list = get_files_in_dir(dir_path)

    # set up animation
    animation = Animator.Animation()
    for file_ in file_list:
        frame = image.load(file_).convert_alpha()
        animation.add_frame(frame)

    return animation

# Excerpt from stack overflow, Mark Byers
def natural_sort(l):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [convert(c) for c in split('([0-9]+)', key)]
    return sorted(l, key=alphanum_key)

# given a tile image, this function will create a quilt of tiles onto a another surface
# given the width and height.
def create_img_from_tile(tile_surface, width, height):

    tile_w = tile_surface.get_width()
    tile_h = tile_surface.get_height()

    size = (width, height)
    dst_surface = Surface(size).convert()

    # make the dst_surface transparent
    color_mask = (7, 13, 17)
    dst_surface.fill(color_mask)
    dst_surface.set_colorkey(color_mask)

    # fill the dst_surface horizontally first with the tile, once we get to the edge
    # go down a column and repeat.
    for y in range(0, height, tile_h):
        for x in range(0, width, tile_w):
            dst_surface.blit(tile_surface, (x, y))

    return dst_surface

# conjoins surface b on the bottom of surface a
def conjoin_surfaces_vertically(surface_a, surface_b):
    width = max(surface_a.get_width(), surface_b.get_width())
    height = surface_a.get_height() + surface_b.get_height()

    size = (width, height)
    dst_surface = Surface(size).convert()

    # make the dst_surface transparent
    color_mask = (123, 54, 33)
    dst_surface.fill(color_mask)
    dst_surface.set_colorkey(color_mask)

    # center each surface relative to the destination surface
    center_a = width/2 - surface_a.get_width()/2
    center_b = width/2 - surface_b.get_width()/2

    dst_surface.blit(surface_a, (center_a, 0))
    dst_surface.blit(surface_b, (center_b, surface_a.get_height()))

    return dst_surface


import sys
sys.path.append('../src/')

import torch
from optimizer import minimizeLoss
from torch.autograd import Variable

def computeCost(parameters, input):
    weight, bias = parameters
    return torch.add(torch.matmul(input, weight), bias)

input = torch.randn(1, 10)
weight = Variable(torch.randn(10, 1), requires_grad = True)
bias = Variable(torch.randn(1, 1), requires_grad = True)

parameters = [weight, bias]
optimizer = torch.optim.Adam(parameters, lr = 1)
lossArgs = (parameters, input)

print(computeCost(parameters, input))
minimizeLoss(100, optimizer, computeCost, lossArgs)
print(computeCost(parameters, input))


import logging

import matplotlib.pyplot as plt

from kuibit import argparse_helper as kah
from kuibit.simdir import SimDir
from kuibit.visualize_matplotlib import (
    get_figname,
    save_from_dir_filename_ext,
    set_axis_limits_from_args,
    setup_matplotlib,
)

if __name__ == "__main__":
    desc = f"""\
{kah.get_program_name()} plots the times intervals at which the different given apparent horizons were found. """

    parser = kah.init_argparse(desc)
    kah.add_figure_to_parser(parser, add_limits=True)

    parser.add_argument(
        "-a",
        "--horizons",
        type=int,
        required=True,
        help="Apparent horizons to plot",
        nargs="+",
    )

    args = kah.get_args(parser)
    setup_matplotlib(rc_par_file=args.mpl_rc_file)

    # Parse arguments

    logger = logging.getLogger(__name__)

    if args.verbose:
        logging.basicConfig(format="%(asctime)s - %(message)s")
        logger.setLevel(logging.DEBUG)

    horizons = "_".join([str(h) for h in args.horizons])
    figname = get_figname(args, default=f"ah_{horizons}_found")
    logger.debug(f"Figname: {figname}")

    with SimDir(
        args.datadir,
        ignore_symlinks=args.ignore_symlinks,
        pickle_file=args.pickle_file,
    ) as sim:
        logger.debug("Prepared SimDir")

        sim_hor = sim.horizons

        logger.debug(
            f"Apparent horizons available: {sim_hor.available_apparent_horizons}"
        )

        for ah in args.horizons:
            if ah in sim_hor.available_apparent_horizons:
                logger.debug(f"Reading horizon {ah}")
                # We can use any index for the qlm index, it will be thrown away
                current_horizon = sim_hor.get_apparent_horizon(ah)
                time_found = current_horizon.ah.cctk_iteration.t
                # We prepare an array with the same length of time_found and with
                # constant value of ah
                ah_num = [ah] * len(time_found)
                plt.scatter(time_found, ah_num, marker="o", s=0.1)

        # Plot
        logger.debug("Plotting")
        plt.ylabel("Apparent horizon")
        plt.xlabel("Time")

        # Fix ticks
        plt.gca().tick_params(axis="y", which="minor", left=False)
        plt.ylim(min(args.horizons) - 1, max(args.horizons) + 1)
        plt.yticks(args.horizons)

        set_axis_limits_from_args(args)
        logger.debug("Plotted")

        logger.debug("Saving")
        save_from_dir_filename_ext(
            args.outdir,
            figname,
            args.fig_extension,
            tikz_clean_figure=args.tikz_clean_figure,
        )
        logger.debug("DONE")


import numpy as np
import pygame

from common import window, colors, DEFAULT_FONT, quitProgram
from drawing import drawTarget, drawTracker, writeText
from fuzzy_tracker import FuzzyTracker

pygame.display.set_caption("Fuzzy Logic Target Tracking by Jelena Ristić and Mihailo Grbić")

window_dim = np.array(window.get_size())
freeze = False

tracker = FuzzyTracker(position=window_dim / 2)

while True:
    window.fill(colors['White'])

    # Event handling
    events = pygame.event.get()
    for event in events:
        if event.type == pygame.QUIT:
            quitProgram()

        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                quitProgram()

        if event.type == pygame.MOUSEBUTTONDOWN:
            freeze = not freeze

    # Make target follow mouse pointer if not frozen
    if not freeze:
        target_pos = np.array(pygame.mouse.get_pos())

    tracker.updateTrackerPos(target_pos)

    drawTarget(target_pos)
    drawTracker(tracker.position)
    writeText()

    # Draw everything on the screen
    pygame.display.update()


#!/usr/bin/env python

import matplotlib.pyplot as plt
import json

def main():
    filename = "3D.txt"

    values = []
    with open(filename, "r") as f:
        line = f.readline()
        while line:
            rho, theta, length = line.split(',')
            values.append((float(rho), float(theta), float(length)))
            line = f.readline()

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    for rho, theta, length in values:
        ax.scatter(rho, theta, length)

    ax.set_xlabel("rho")
    ax.set_ylabel("theta")
    ax.set_zlabel("length")

    plt.show()

if __name__ == "__main__":
    main()