#调用该模块返回一个或多个可用的代理

#1.爬取所有代理并保存到本地 upgrade_ip
#2.从本地文件中测试选出可用代理，保存在数组中

import requests
import telnetlib
import threading
import  queue
from multiprocessing import Process
from lxml import etree

q = queue.Queue()

def sava_proxy(url):
   headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"}
   response = requests.get(url,headers = headers)
   html = response.text
   x_html = etree.HTML(html)
   ip_info = x_html.xpath("//tr[@class]")

   with open("ip.txt",'a') as f:
       for i in ip_info:
           ip = i.xpath("./td[2]")[0].text
           port = i.xpath("./td[3]")[0].text
           ip_port = ip+":"+port
           f.write(ip_port+"\n")

#将文件中的Ip存放在queue中
def get_proxy():
    with open("ip.txt")as f:
        while True:
            str = f.readline()
            if not str:
                break
            q.put(str)

#验证代理是否有效
def get_test_proxy():
    while not q.empty():
        ip = q.get().split(":")[0].strip()
        port = q.get().split(":")[1].strip()
        if test_proxy(ip,port):
            return ip,port
        else:
            pass

def threas(fun):
    for i in range(30):
        t = threading.Thread(target = fun )
        t.start()
        t.join()

def test_proxy(ip,port):
    try:
         telnetlib.Telnet(ip,port=port,timeout=3)
    except:
        print("field   "+ip+":"+port)
        return False
    else:
        print("success   "+ip+":"+port)
        return True

if __name__ == '__main__':
    url = "http://www.xicidaili.com/nt/"
    sava_proxy(url)
    get_proxy()
    threas(get_test_proxy)


# Core Pkg
import pandas as pd
import seaborn as sns
import streamlit as st
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier

# Custom function
# st.cache is used to load the function into memory
@st.cache
def train_model(model_choisi, X_train, y_train, X_test, y_test) :
    if model_choisi == 'Regression Logisitic' :
        model = LogisticRegression()
    elif model_choisi == 'Decision Tree' :
        model = DecisionTreeClassifier()
    elif model_choisi == 'KNN' :
        model = KNeighborsClassifier()
    model.fit(X_train, y_train)
    score = model.score(X_test, y_test)
    return score

def demo_streamlit():

    ### Create Title
    st.title("Streamlit Machine Learning Web app with Streamlit")
    st.header("The beginning of a great adventure")
    st.subheader("By DataScientest team")

    ### Add a picture
    st.write("Below is a picture of the Titanic:")
    st.image("titanic.jpeg")

    ### using Markdown
    st.markdown("### We will use the well famous titanic dataset for our study. Can we predict who survived?")

    ### reading dataset
    # Normally, you will store all the necessary path and env variables in a .env file
    dataset_path = '../data/titanic.csv'
    df = pd.read_csv(dataset_path)

    ### Showing code
    st.text("importing dataset with the folowing command: ")
    with st.echo():
        df = pd.read_csv(dataset_path)

    ### Showing the data
    if st.checkbox("Showing the data") :
        line_to_plot = st.slider("select le number of lines to show", min_value=3, max_value=df.shape[0])
        st.dataframe(df.head(line_to_plot))

    if st.checkbox("Missing values") :
        st.dataframe(df.isna().sum())

    ### Preprocessing

    # Drop Missing values
    df = df.dropna()

    # Drop some columns
    df = df.drop(['sex', 'title', 'cabin', 'embarked'], axis = 1)

    # Select the target
    y = df['survived']

    # Select the features
    X = df.drop('survived', axis =1 )

    # select how to split the data
    train_size = st.sidebar.slider(label = "Choix de la taille de l'échantilllon de train", min_value = 0.2, max_value = 1.0, step = 0.05)

    # Splitting the data
    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = train_size)

    ### Display graph
    st.text('Class distribution with seaborn')
    st.set_option('deprecation.showPyplotGlobalUse', False)
    sns.countplot(df['pclass'])
    st.pyplot()

    ### Classification

    #  Baseline model
    model = LogisticRegression()

    # Model training
    model.fit(X_train, y_train)

    # Benchmark Model evaluation
    st.write("Logisitic regression accuracy (This is my Benchmark):" , model.score(X_test,y_test))

    # Other models
    model_list = ['Decision Tree', 'KNN']
    model_choisi = st.selectbox(label = "Select a model" , options = model_list)

    # Showing the accuracy for the orthers models (for comparison)
    st.write("Accuracy for some models for comparison: ")
    st.write("Score test", train_model(model_choisi, X_train, y_train, X_test, y_test))


#!/usr/bin/env python

import argcomplete
import argparse
import atexit
import coloredlogs
import functools
import gc
import json
import katcp
import logging
import os
import ipaddress
import socket
import sys
import time
import threading

from collections import OrderedDict
from itertools import izip_longest
from ast import literal_eval as evaluate
from pprint import PrettyPrinter

def retry(func, count=20, wait_time=300):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        retExc = TypeError
        for _ in xrange(count):
            while True:
                try:
                    return func(*args, **kwargs)
                except Exception as exc:
                    retExc = exc
                    time.sleep(wait_time)
                    continue
                break
        raise retExc
    return wrapper

def combined_Dict_List(*args):
    """
    Combining two/more dictionaries into one with the same keys

    Params
    =======
    args: list
        list of dicts to combine

    Return
    =======
    result: dict
        combined dictionaries
    """
    result = {}
    for _dict in args:
        for key in result.viewkeys() | _dict.keys():
            if key in _dict:
                result.setdefault(key, []).extend([_dict[key]])
    return result

# This class could be imported from a utility module
class LoggingClass(object):
    @property
    def logger(self):
        log_format = "%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(pathname)s : %(lineno)d - %(message)s"
        name = ".".join([os.path.basename(sys.argv[0]), self.__class__.__name__])
        logging.basicConfig(format=log_format)
        coloredlogs.install(fmt=log_format)
        return logging.getLogger(name)

class SensorPoll(LoggingClass):
    def __init__(self, katcp_ip=None, katcp_port=7147):
        """
        Parameters
        =========
        katcp_ip: str
            IP to connect to!
        katcp_port: int
            Port to connect to! [Defaults: 7147]
        """

        try:
            assert katcp_ip
            self.katcp_ip = katcp_ip
            ipaddress.ip_address(u"{}".format(self.katcp_ip))
            self.hostname = ''.join(socket.gethostbyaddr(katcp_ip)[1])
        except Exception:
            self.logger.exception("Invalid KATCP_IP!")
            raise
        self.katcp_port = katcp_port
        self._kcp_connect()

    @retry
    def _kcp_connect(self):
        try:
            self._started = False
            self.primary_client = self.katcp_request(which_port=self.katcp_port)
            atexit.register(self.cleanup, self.primary_client)
            assert isinstance(self.primary_client, katcp.client.BlockingClient)
            reply, informs = self.sensor_request(self.primary_client)
            assert reply.reply_ok()
            katcp_array_list = informs[0].arguments
            assert isinstance(katcp_array_list, list)
            self.katcp_array_port, self.katcp_sensor_port = [
                int(i) for i in katcp_array_list[1].split(",")
            ]
            self.array_name = katcp_array_list[0]
            assert isinstance(self.katcp_array_port, int)
            assert isinstance(self.katcp_sensor_port, int)
        except Exception:
            self.logger.error(
                "No running array on {}:{}!!!!".format(self.katcp_ip, self.katcp_port),
                #exc_info=True
            )
            if self.primary_client.is_connected():
                self.cleanup(self.primary_client)
            raise
        else:
            if self._started:
                self._started = False
                self.sec_client = self.katcp_request(which_port=self.katcp_array_port)
                atexit.register(self.cleanup, self.sec_client)

            if self._started:
                self._started = False
                self.sec_sensors_katcp_con = self.katcp_request(
                    which_port=self.katcp_sensor_port
                )
                atexit.register(self.cleanup, self.sec_sensors_katcp_con)

            self.logger.info(
                "Katcp connection established: IP {}, Primary Port: {}, Array Port: {}, "
                "Sensor Port: {}".format(
                    self.katcp_ip,
                    self.katcp_port,
                    self.katcp_array_port,
                    self.katcp_sensor_port,
                )
            )
            time.sleep(1)
            try:
                self.input_mapping, self.hostname_mapping = self.do_mapping()
            except Exception:
                self.cleanup(self.sec_client)
                self.cleanup(self.sec_sensors_katcp_con)
                self.logger.error(
                    "Ayeyeyeye! it broke cannot do mappings",
                             # exc_info=True
                    )
                raise

    def katcp_request(
        self, which_port, katcprequest="array-list", katcprequestArg=None, timeout=10
    ):
        """
        Katcp requests

        Parameters
        =========
        which_port: str
            Katcp port to connect to!
        katcprequest: str
            Katcp requests messages [Defaults: 'array-list']
        katcprequestArg: str
            katcp requests messages arguments eg. array-list array0 [Defaults: None]
        timeout: int
            katcp timeout [Defaults :10]

        Return
        ======
        reply, informs : tuple
            katcp request messages
        """
        # self._started = False
        if not self._started:
            self._started = True
            self.logger.info(
                "Establishing katcp connection on {}:{}".format(
                    self.katcp_ip, which_port
                )
            )
            client = katcp.BlockingClient(self.katcp_ip, which_port)
            client.setDaemon(True)
            client.start()
            time.sleep(0.1)
            try:
                is_connected = client.wait_running(timeout)
                assert is_connected
                self.logger.info(
                    "Katcp client connected to {}:{}\n".format(
                        self.katcp_ip, which_port
                    )
                )
                return client
            except Exception:
                client.stop()
                self.logger.error("Could not connect to katcp, timed out.")

    def sensor_request(
        self, client, katcprequest="array-list", katcprequestArg=None, timeout=10
    ):
        """
        Katcp requests

        Parameters
        =========
        client: katcp.client.BlockingClient
            katcp running client
        katcprequest: str
            Katcp requests messages [Defaults: 'array-list']
        katcprequestArg: str
            katcp requests messages arguments eg. array-list array0 [Defaults: None]
        timeout: int
            katcp timeout [Defaults :10]

        Return
        ======
        reply, informs : tuple
            katcp request messages
        """
        try:
            if katcprequestArg:
                reply, informs = client.blocking_request(
                    katcp.Message.request(katcprequest, katcprequestArg),
                    timeout=timeout,
                )
            else:
                reply, informs = client.blocking_request(
                    katcp.Message.request(katcprequest), timeout=timeout
                )
            assert reply.reply_ok()
        except Exception:
            self.logger.error("Failed to execute katcp command")
            # time.sleep(20)
            raise
        else:
            return reply, informs

    def cleanup(self, client):
        if self._started:
            self.logger.debug("Some Cleaning Up!!!")
            client.stop()
            time.sleep(0.1)
            if client.is_connected():
                self.logger.error("Did not clean up client properly, %s" % client.bind_address)

    @property
    def get_sensor_values(self):
        try:
            assert self.katcp_sensor_port
            reply, informs = self.sensor_request(
                self.sec_sensors_katcp_con, katcprequest="sensor-value")
            assert reply.reply_ok()
            assert int(reply.arguments[-1])
            yield [inform.arguments for inform in informs]
        except AssertionError:
            self.logger.error("No Sensors!!! Exiting!!!")
            raise

    @property
    def get_hostmapping(self):
        try:
            assert self.katcp_sensor_port
            reply, informs = self.sensor_request(
                self.sec_sensors_katcp_con,
                katcprequest="sensor-value",
                katcprequestArg="hostname-functional-mapping",
            )
            assert reply.reply_ok()
            assert int(reply.arguments[-1])
            yield [inform.arguments for inform in informs]
        except AssertionError:
            self.cleanup(self.sec_sensors_katcp_con)
            self.logger.error("No Sensors!!! Exiting!!!")
            raise

    @property
    def get_inputlabel(self, i=1):
        try:
            assert self.katcp_array_port
            for i in xrange(i):
                reply, informs = self.sensor_request(
                    self.sec_client,
                    katcprequest="sensor-value",
                    katcprequestArg="input-labelling",
                )
                assert reply.reply_ok()
            assert int(reply.arguments[-1])
            yield [inform.arguments for inform in informs]
        except AssertionError:
            self.cleanup(self.sec_client)
            self.logger.error("No Sensors!!! Exiting!!!")
            raise

    @property
    def get_sensor_dict(self):
        sensor_value_informs = next(self.get_sensor_values)
        self.logger.debug("Converting sensor list to dict!!!")
        # sensors name + status + value
        self.original_sensors = dict(
            (x[0], x[1:]) for x in [i[2:] for i in sensor_value_informs]
        )
        # sensors name and status
        simplified_sensors = dict(
            (x[0], x[1]) for x in [i[2:] for i in sensor_value_informs]
        )
        simplified_ordered_sensors = OrderedDict(sorted(simplified_sensors.items()))
        try:
            return simplified_ordered_sensors
        except Exception as exc:
            raise exc

    def do_mapping(self):
        try:
            self.logger.debug("Mapping input labels and hostnames")
            hostname_mapping = next(self.get_hostmapping)[-1][-1]
            input_mapping = next(self.get_inputlabel)[-1][-1]
        except Exception:
            self.logger.error(
                "Serious error occurred, cannot continue!!! Missing sensors"
            )
            raise
        else:
            input_mapping = dict(list(i)[0:3:2] for i in evaluate(input_mapping))
            input_mapping = dict((v, k) for k, v in input_mapping.iteritems())
            update_maps = []
            for i in input_mapping.values():
                if "_y" in i:
                    i = i.replace("_y", "_xy")
                elif "_x" in i:
                    i = i.replace("_x", "_xy")
                elif "v" in i:
                    i = i.replace("v", "_vh")
                elif "h" in i:
                    i = i.replace("h", "_vh")

                update_maps.append(i)

            update_maps = sorted(update_maps)
            input_mapping = dict(zip(sorted(input_mapping.keys()), update_maps))
            hostname_mapping = dict(
                (v, k) for k, v in evaluate(hostname_mapping).iteritems()
            )
            return [input_mapping, hostname_mapping]

    def get_list_index(self, String, List):
        """
        Find the index of a string in a list

        Params
        =======
        String: str
            String to search in list
        List: list
            List to be searched!

        Return
        =======
        list
        """
        try:
            return [_c for _c, i in enumerate(List) if any(String in x for x in i)][0]
        except Exception:
            self.logger.error("Failed to find the index of string in list")

    def new_mapping(self, _host):
        try:
            self.logger.debug("Sorting ordered sensor dict by %ss!!!" % _host)
            ordered_sensor_dict = self.get_sensor_dict
            assert isinstance(ordered_sensor_dict, OrderedDict)
        except Exception:
            self.logger.error("Failed to retrieve sensor dict", exc_info=True)
        else:
            mapping = []
            for key, value in ordered_sensor_dict.iteritems():
                key_s = key.split(".")
                host = key_s[0].lower()
                if host.startswith(_host) and ("device-status" in key_s):
                    new_value = [x.replace("device-status", value) for x in key_s[1:]]
                    if "network-reorder" in new_value:
                        # rename such that, it fits on html/button
                        _indices = new_value.index("network-reorder")
                        new_value[_indices] = new_value[_indices].replace(
                            "network-reorder", "Net-ReOrd"
                        )
                    if "missing-pkts" in new_value:
                        # rename such that, it fits on html/button
                        _indices = new_value.index("missing-pkts")
                        new_value[_indices] = new_value[_indices].replace(
                            "missing-pkts", "hmcReOrd"
                        )
                    if "bram-reorder" in new_value:
                        # rename such that, it fits on html/button
                        _indices = new_value.index("bram-reorder")
                        new_value[_indices] = new_value[_indices].replace(
                            "bram-reorder", "bramReOrd"
                        )

                    new_dict = dict(
                        izip_longest(*[iter([host, new_value])] * 2, fillvalue="")
                    )
                    mapping.append(new_dict)

            new_mapping = combined_Dict_List(*mapping)
            for host, _list in new_mapping.iteritems():
                if host in self.hostname_mapping:
                    new_hostname = host.replace(_host, "") + self.hostname_mapping[
                        host
                    ].replace("skarab", "-").replace("-01", "")
                _ = [value.insert(0, new_hostname)
                    for value in _list if len(value) == 1]

            return new_mapping

    @property
    def map_xhost_sensors(self):
        """
        Needs to be in this format:
            'host03': [
                    ['03-020308', 'warn'],
                    ['network', 'warn'],
                    ['spead-rx', 'nominal'],
                    ['Net-ReOrd', 'nominal'],
                    ['hmcReOrd', 'warn'],
                    ['bram-reorder', 'error'],
                    ['vacc', 'error'],
                    ['spead-tx', 'nominal']
                ]
            }

        """
        xhost_sig_chain = [
            "-02",
            "network",
            "spead-rx",
            "Net-ReOrd",
            "hmcReOrd",
            "bramReOrd",
            "vacc",
            "spead-tx",
        ]
        try:
            new_mapping = self.new_mapping("xhost")
            assert isinstance(new_mapping, dict)
        except Exception:
            self.logger.error("Failed to map xhosts", exc_info=True)
        else:
            new_dict_mapping = {}
            for keys, values in new_mapping.iteritems():
                keys_ = keys[1:]
                new_dict_mapping[keys_] = []
                for value in values:
                    if (len(value) <= 2) and (not value[0].startswith("xeng")):
                        new_dict_mapping[keys_].append(value)
                    if (
                        value[0].startswith("xeng")
                        and value not in new_dict_mapping.values()
                    ):
                        if "vacc" in value:
                            new_dict_mapping[keys_].append(value[1:])
                        if "spead-tx" in value:
                            new_dict_mapping[keys_].append(value[1:])
                        if "bramReOrd" in value:
                            new_dict_mapping[keys_].append(value[1:])

            # _ = [listA.insert(_index, listA.pop(self.get_list_index(_sig, listA)))
            #      for _, listA in new_dict_mapping.iteritems() for _index, _sig in enumerate(xhost_sig_chain)]
            # return new_dict_mapping
            fixed_dict_mapping = {}
            for host_, listA in new_dict_mapping.iteritems():
                listA = listA[: len(xhost_sig_chain)]
                for _index, _sig in enumerate(xhost_sig_chain):
                    listA.insert(_index, listA.pop(self.get_list_index(_sig, listA)))
                fixed_dict_mapping[host_] = listA
            return fixed_dict_mapping

    @property
    def map_fhost_sensors(self):
        """
        {
             'fhost03': [
                            ['SKA-020709', 'warn'],
                            ['fhost00', 'skarab020709-01'],
                            ['ant0_y', 'inputlabel'],
                            ['network', 'nominal'],
                            ['spead-rx', 'failure'],
                            ['Net-ReOrd', 'nominal'],
                            ['cd', 'warn'],
                            ['pfb', 'warn'],
                            ['ct', 'nominal'],
                            ['spead-tx', 'nominal'],
                            ['->XEngine', 'xhost']
                        ]
        }

        """

        # Abbreviated signal chain
        # F_LRU -> Host -> input_label -> network-trx -> spead-rx -> network-reorder -> cd -> pfb -->>
        #    -->> ct -> spead-tx -> network-trx : [To Xengine ]
        # issue reading cmc3 input labels
        # fhost_sig_chain = ['SKA', 'fhost', 'network', 'spead-rx', 'Net-ReOrd', 'cd', 'pfb',
        fhost_sig_chain = [
            "-02",
            "input",
            "network",
            "spead-rx",
            "Net-ReOrd",
            "cd",
            "pfb",
            "ct",
            "spead-tx",
        ]
        try:
            new_mapping = self.new_mapping("fhost")
            assert isinstance(new_mapping, dict)
        except Exception:
            self.logger.error("Failed to map fhosts", exc_info=True)
        else:
            for host, values in new_mapping.iteritems():
                if host in self.hostname_mapping:
                    values.insert(
                        2, [self.input_mapping[self.hostname_mapping[host]], "inputlabel"]
                    )
                    # values.append(['->XEngine', 'xhost'])

            new_dict_mapping = {}
            for host, values in new_mapping.iteritems():
                host_ = host[1:]
                new_dict_mapping[host_] = values
            # Update mappings
            _ = [listA.insert(_index, listA.pop(self.get_list_index(_sig, listA)))
                for _, listA in new_dict_mapping.iteritems()
                for _index, _sig in enumerate(fhost_sig_chain)
                ]

            return new_dict_mapping

    @property
    def get_original_mapped_sensors(self):
        mapping = []
        for key, value in self.original_sensors.iteritems():
            host = key.split(".")[0].lower()
            if host[1:].startswith("host"):
                if value[0] != "nominal":
                    value.insert(0, key)
                    new_dict = dict(
                        izip_longest(*[iter([host, value])] * 2, fillvalue="")
                    )
                    mapping.append(new_dict)

        new_mapping = combined_Dict_List(*mapping)
        return new_mapping

    @staticmethod
    def merged_sensors_dict(dict1, dict2):
        """
        merge two dictionaries
        https://stackoverflow.com/questions/38987/how-to-merge-two-dictionaries-in-a-single-expression#26853961
        """
        for key, value in dict2.iteritems():
            dict1.setdefault(key, []).extend(value)
        return dict1
        # merged_sensors = f_sensors.copy()
        # merged_sensors.update(x_sensors)
        # return merged_sensors

    def create_dumps_dir(self):
        """
        Create json dumps directory
        """
        # Conflicted: To store in /tmp or not to store in , that is the question
        try:
            _dir, _name = os.path.split(os.path.dirname(os.path.realpath(__file__)))
        except Exception:
            _dir, _name = os.path.split(os.path.dirname(os.path.realpath(__name__)))
        path = _dir + "/json_dumps"
        if not os.path.exists(path):
            self.logger.info("Created %s for storing json dumps." % path)
            os.makedirs(path)

    def write_sorted_sensors_to_file(self):
        try:
            sensors = self.merged_sensors_dict(
                self.map_fhost_sensors, self.map_xhost_sensors
            )
        except Exception:
            self.logger.error(
                "Failed to map the host sensors",
                exc_info=True
                )
            raise
        else:
            self.create_dumps_dir()
            try:
                cur_path = os.path.split(os.path.dirname(os.path.abspath(__file__)))[0]
            except Exception:
                cur_path = os.path.split(os.path.dirname(os.path.abspath(__name__)))[0]
            else:
                _filename = "{}/json_dumps/{}.{}.sensor_values.json".format(
                    cur_path, self.hostname, self.array_name)
                _sensor_filename = "{}/json_dumps/{}.{}.ordered_sensor_values.json".format(
                    cur_path, self.hostname, self.array_name)
                self.logger.info("Updating file: %s" % _filename)
                with open(_filename, "w") as outfile:
                    json.dump(sensors, outfile, indent=4, sort_keys=True)
                with open(_sensor_filename, "w") as outfile:
                    json.dump(
                        self.get_original_mapped_sensors,
                        outfile,
                        indent=4,
                        sort_keys=True,
                    )
                self.logger.info("Updated: %s" % _filename)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Receive data from a CBF and play.")
    parser.add_argument(
        "--hostip",
        dest="katcp_host_ip",
        action="store",
        help="IP address of primary interface [eg: 10.103.254.6]",
        required=True,
    )
    parser.add_argument(
        "--port",
        dest="katcp_host_port",
        action="store",
        default=7147,
        help="Primary Port to connect to. [Default: 7147]",
    )
    parser.add_argument(
        "--poll-time",
        dest="poll",
        action="store",
        default=10,
        type=int,
        help="Poll the sensors every x seconds [Default: 10]",
    )
    parser.add_argument(
        "--loglevel",
        dest="log_level",
        action="store",
        default="INFO",
        help="log level to use, default INFO, options INFO, DEBUG, ERROR",
    )

    argcomplete.autocomplete(parser)
    args = vars(parser.parse_args())
    pp = PrettyPrinter(indent=4)
    log_level = None
    log_format = "%(asctime)s - %(name)s - %(levelname)s - %(module)s - %(pathname)s : %(lineno)d - %(message)s"
    if args.get("log_level", "INFO"):
        log_level = args.get("log_level", "INFO").upper()
        try:
            logging.basicConfig(level=getattr(logging, log_level), format=log_format)
        except AttributeError:
            raise RuntimeError("No such log level: %s" % log_level)
        else:
            if log_level == "DEBUG":
                coloredlogs.install(level=log_level, fmt=log_format)
            else:
                coloredlogs.install(level=log_level)

    katcp_ip = args.get("katcp_host_ip")
    katcp_port = args.get("katcp_host_port")

    sensor_poll = SensorPoll(katcp_ip, katcp_port)
    main_logger = LoggingClass()
    try:
        poll_time = args.get("poll")
        main_logger.logger.info("Begin sensor polling every %s seconds!!!" % poll_time)
        while True:
            sensor_poll.write_sorted_sensors_to_file()
            main_logger.logger.debug("Updating sensor on dashboard!!!")
            main_logger.logger.info("---------------------RELOADING SENSORS---------------------")
            time.sleep(poll_time)
    except Exception:
        main_logger.logger.error("Error occurred now breaking...")


from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession

from app.crud.exercises import writing_crud, EX_TYPES
from app.crud.conditions import condition_crud
from app.models import Condition
from app.models.exercises import Exercise, Writing
from app.models.enums import Level

MIN_TOKENS = 100

async def check_exercise_type(exercise_type: str):
    if exercise_type not in EX_TYPES:
        raise HTTPException(404, 'Incorrect exercise type!')

async def check_exercise_exist(
        exercise_type: str, level: Level, session: AsyncSession
) -> Exercise:
    await check_exercise_type(exercise_type)
    exercise = await EX_TYPES[exercise_type].get_random(session, level)
    if not exercise:
        raise HTTPException(404, 'No exercises with these criteria!')
    return exercise

async def check_exercise_by_id(
        exercise_type: str,
        exercise_id: int,
        session: AsyncSession
) -> Writing:
    if exercise_type not in EX_TYPES:
        raise HTTPException(404, 'Incorrect exercise type!')
    exercise = await EX_TYPES[exercise_type].get(exercise_id, session)
    if not exercise:
        raise HTTPException(404, 'No exercises with this ID!')
    return exercise

async def check_conditions_by_id(
    conditions_ids: list[int],
    session: AsyncSession
) -> list[Condition]:
    conditions = []
    for obj_id in conditions_ids:
        condition = await condition_crud.get(obj_id, session)
        if condition is None:
            raise HTTPException(404, f"Don't have condition with ID: {obj_id}!")
        conditions.append(condition)
    return conditions

async def check_user_balance(user):
    if user.tokens < MIN_TOKENS:
        raise HTTPException(400, 'Not enough money!')


import graf
import  config

import copy

def create_temlate(dex_factoty: str, tokens: list, amount=None, minimum_receive=None, max_spread=None):
    tp = copy.deepcopy(config.template)
    operations_to_add = []
    swap = config.dex_names[dex_factoty]
    op = {swap: config.operation.copy()}

    for i in range(len(tokens) - 1):

        if tokens[i] in config.native_tokens:
            offer = config.asset_info[1]
            offer["native_token"]["denom"] = tokens[i]
        else:
            offer = config.asset_info[0]
            offer["token"]["contract_addr"] = tokens[i]
        op[swap]["offer_asset_info"] = copy.deepcopy(offer)

        if tokens[i + 1] in config.native_tokens:
            ask = config.asset_info[1]
            ask["native_token"]["denom"] = tokens[i + 1]
        else:
            ask = config.asset_info[0]
            ask["token"]["contract_addr"] = tokens[i + 1]

        op[swap]["ask_asset_info"] = copy.deepcopy(ask)
        operations_to_add.append(copy.deepcopy(op))

    tp["execute_swap_operations"]["operations"] = operations_to_add

    if amount:
        tp["execute_swap_operations"]["offer_amount"] = str(int(amount * config.to_million))
    if minimum_receive:
        tp["execute_swap_operations"]["minimum_receive"] = str(int(minimum_receive * config.to_million))
    if max_spread:
        tp["execute_swap_operations"]["max_spread"] = str(max_spread)
    return tp

print(create_temlate("terra16t7dpwwgx9n3lq6l6te3753lsjqwhxwpday9zx",('terra15gwkyepfc6xgca5t5zefzwy42uts8l2m4g40k6', 'uusd', 'uluna')))

def simulation(wasm, dex, amount, *tokens):
    tp = create_temlate(dex, tokens, amount=amount)
    tp_query = {"simulate_swap_operations": tp["execute_swap_operations"]}
    res = wasm.contract_query(dex, tp_query)
    return res


# -*- coding: utf-8 -*-
"""
Created on Sat May  2 17:57:08 2020

@author: pselt
"""
import numpy as np
import cpmg_utils as cpmg
import matplotlib.pyplot as plt

font1 = {'family': 'serif',
        'color':  'black',
        'weight': 'normal',
        'size': 12,
        }

###############################################################################
#
# Path to example data
#
path = 'C:/Users/pselt/OneDrive/old_sciebo_share/NMR-Rohdaten/NMR300/p_selt01/nmr/LOH93'

##############################################################################
# Definiton of the window function
#
# The total window function is created by generating an arbitrary window covering
# a single echo. This is then concetaneted to form the window function over the whole
# echo train. additional points can be added at the front or back.

##########################
# Example 1:
# define the gaussian bell
#
# gauss = cpmg.create_gauss(npoints=402,width=20,center=170)
#
# parameter 1 - npoints: number of complex points in echo. Note: top spin stores real and
# imaginary part separately, while python treats them as a single point
# i.e. TD=800 means np=400

# parameter 2 - width: depends on the window function in question, refer to cpmg-utils

# parameter 3 - center: at which point is the center of the window function.
# This is used to for example shift the top of a gaussian bell through the echo

# gauss = cpmg.create_lorentzian(npoints=402,width=5,center=170)
# gauss = cpmg.create_rectangle(npoints=402,width=100,center=170)
# gauss = cpmg.create_cheby(npoints=402,width=50,center=170)
# gauss = cpmg.create_flattop(npoints=352,width=50,center=170)
# gauss = cpmg.create_rectangle(npoints=452,width=452,center=226)

# gauss = cpmg.create_sinc(npoints=402,width=100,center=170)
# concatenated the gaussian bells
multigauss = cpmg.create_rp_wdw(0,gauss,10,0)

test1 = cpmg.cpmg_data(path,expno=10)
# test1 = cpmg.cpmg_data(path,expno=301)

time, data = test1.return_fid()
test1.apply_wdw(multigauss)

time, data2 = test1.return_fid()
freq, fftdata5 = test1.return_fft(256*1024,128*1024)

test1.reload_fid()
time, data3 = test1.return_fid()
freq, fftdata2 = test1.return_fft(256*1024,128*1024)

###############################################################################
#
# Original Spectrum
#
###################
fig = plt.figure(figsize=(5,3),dpi=200)
ax=plt.gca()

plt.plot(freq, np.abs(fftdata5)/10000,linewidth=0.1,color='k')
plt.xlim(freq[-1],freq[0])

plt.tick_params(axis='x', which='both', labelleft=False, left=False, labelsize=12)
plt.tick_params(axis='y', which='both', labelleft=False, left=False, labelsize=12)

ax.set_xlabel(r'$\nu$ '+'($^{195}$Pt) / Hz', fontdict=font1)
# ax.set_ylabel(r'Intensity / ppm', fontdict=font1)

ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)

plt.show()
# fig.savefig("CPMG_spec_org_TDEFF_100echoes.png",dpi=300, format='png', bbox_inches='tight',  pad_inches=0.2,transparent=True)
###################
###############################################################################

###############################################################################
#
# New processing
#
###################
fig = plt.figure(figsize=(5,3),dpi=200)
ax=plt.gca()
sumtest = np.zeros(256*1024)

for k in range(0,200,1):
    gauss = cpmg.create_sinc(npoints=452,width=15,center=126+k)
    # gauss = cpmg.create_flattop(npoints=452,width=100,center=126+k)
    # gauss = cpmg.create_rectangle(npoints=452,width=100,center=126+k)
    # gauss = cpmg.create_tukey(npoints=352,width=100,center=126+k,alpha=0.5)
    # gauss = cpmg.create_cheby(npoints=452,width=75,center=126+k)
    # gauss = cpmg.create_lorentzian(npoints=452,width=50,center=126+k)
    # gauss = cpmg.create_gauss(npoints=352,width=50,center=76+k)
    # concatenated the gaussian bells
    multigauss = cpmg.create_rp_wdw(0,gauss,10,0)
    test1.apply_wdw(multigauss)
    freq, fftdata = test1.return_fft(256*1024)
    # sumtest = sumtest+np.abs(fftdata)
    sumtest = np.fmax(np.abs(sumtest),np.abs(fftdata))
    # plt.plot(freq,np.abs(fftdata))
    test1.reload_fid()
plt.plot(freq, np.abs(fftdata5)/1000000,linewidth=0.2,color='r')
plt.plot(freq,sumtest/1000000,linewidth=0.2,color='k')

plt.xlim(freq[-1],freq[0])

plt.tick_params(axis='x', which='both', labelleft=False, left=False, labelsize=12)
plt.tick_params(axis='y', which='both', labelleft=False, left=False, labelsize=12)

ax.set_xlabel(r'$\nu$ '+'($^{195}$Pt) / Hz', fontdict=font1)
# ax.set_ylabel(r'Intensity / ppm', fontdict=font1)

ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
plt.show()
# fig.savefig("CPMG_spec_sinc15_td100.png",dpi=300, format='png', bbox_inches='tight',  pad_inches=0.2,transparent=True)
###############################################################################


from uuid import uuid4
from threading import Thread
from queue import Queue

from stardust.actor import ActorRef
from stardust.events import *
from stardust.system.types import ActorID

class IncomingEventManager(Thread):

    def __init__(self, system, input_queue, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.system = system
        self.input_queue = input_queue
        self.pending_spawn_requests = dict()

    def run(self) -> None:
        while True:
            event = self.input_queue.get()

            if isinstance(event, StopExecution):
                break

            else:
                self.process_event(event)

        self.system.load_balancer.queue.put(StopExecution())
        self.system.outgoing_queue.put(StopExecution())

    def process_event(self, event):
        if isinstance(event, SpawnRequestEvent):
            self.spawn(event)
        elif isinstance(event, KillRequestEvent):
            self.kill(event)
        elif isinstance(event, SpawnResponseEvent):
            self.notify_spawned(event)
        elif isinstance(event, KillResponseEvent):
            self.notify_killed(event)
        elif isinstance(event, MessageEvent):
            self.receive_message(event)

    def spawn(self, event: SpawnRequestEvent):
        ref = ActorRef(node_id=self.system.ref.node_id, actor_id=ActorID(uuid4().int))
        lb_event = BalanceSpawnEvent(event, ref)
        self.pending_spawn_requests[ref] = event.actor_class
        self.system.load_balancer.queue.put(lb_event)

    def kill(self, event: KillRequestEvent):
        self.system.load_balancer.queue.put(event)

    def notify_spawned(self, event: SpawnResponseEvent):
        lb_event = BalancerUpdateSpawnedEvent(
            self.pending_spawn_requests.pop(event.child),
            event
        )
        self.system.load_balancer.queue.put(lb_event)

    def notify_killed(self, event: KillResponseEvent):
        lb_event = BalanceKillEvent(
            event
        )
        self.system.load_balancer.queue.put(lb_event)

    def receive_message(self, event: MessageEvent):
        if event.target == self.system.ref and self.system.awaited_context == event.context_code:
            self.system.communication_queue.put(event.message)


import logging
import os
import time

import allure
import tools
from conftest import PLATFORM_NAME, ENV, DEVICE_NAME
from constants import PROJECT_ROOT

class BasePage:
    """Base class to initialize the base page that will be called from all
    pages"""

    def __init__(self, driver):
        self.driver = driver
        self.platform_name = PLATFORM_NAME
        self.device_name = DEVICE_NAME
        self.env = ENV

    current_time = time.strftime("%Y_%m_%d_%H%M%S")

    def take_screenshot(self, screenshot_name):
        directory_new = f'{PROJECT_ROOT}/screenshots/{self.platform_name}/new/{self.device_name}'
        self.check_exist_dir(directory_new)
        path = f'{directory_new}/{screenshot_name}_{self.current_time}.png'
        self.driver.save_screenshot(path)

        tools.crop_status_bar(path, self.platform_name)

    def check_exist_dir(self, dir):
        if not os.path.exists(dir):
            os.makedirs(dir)

    def compare_image(self, org, new):
        directory_org = f'{PROJECT_ROOT}/screenshots/{self.platform_name}/org/{self.device_name}'
        directory_new = f'{PROJECT_ROOT}/screenshots/{self.platform_name}/new/{self.device_name}'
        directory_changes = f'{PROJECT_ROOT}/screenshots/{self.platform_name}/changes/{self.device_name}'
        directory_result = f'{PROJECT_ROOT}/screenshots/{self.platform_name}/result/{self.device_name}'
        self.check_exist_dir(directory_org)
        self.check_exist_dir(directory_new)
        self.check_exist_dir(directory_changes)
        self.check_exist_dir(directory_result)

        if os.path.exists(f'{directory_org}/{org}.png'):
            original_path = f'{directory_org}/{org}.png'
            copy_original_path = f'{directory_org}/{org}_copy.png'
            new_path = f'{directory_new}/{new}_{self.current_time}.png'
            changes_path = f'{directory_changes}/{org}_changes_{self.current_time}.png'
            result_path = f'{directory_result}/{org}_diff_comb_{self.current_time}.png'
            is_same = tools.compare(driver=self.driver, platform=self.platform_name, device_name=self.device_name, img1=original_path, img2=new_path,
                                    changes_path=changes_path)
            if not is_same:
                tools.draw_text('new', new_path)
                tools.draw_text('original', original_path, copy_original_path)
                tools.draw_text('changes', changes_path)
                tools.combine_image(img1=new_path, img2=copy_original_path, img3=changes_path, result_path=result_path)
                allure.attach.file(result_path, name="result_changes_screenshot",
                                   attachment_type=allure.attachment_type.PNG)
                os.remove(copy_original_path)
            with allure.step('Проверка стартовой страницы визуально'):
                assert is_same, f'Скриншоты различаются - разница будет показана на картинке {result_path}'
        else:
            path = f'{directory_org}/{org}.png'
            self.driver.save_screenshot(path)
            tools.crop_status_bar(path, self.platform_name)


from __future__ import absolute_import, print_function
from math import isnan, isinf
from itertools import combinations

import rdkit
from rdkit import Chem
from rdkit.Chem import AllChem
from rdkit.Chem.rdchem import BondType
import sys

METALS = (3, 4, 11, 12, 13, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
          37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 55, 56, 57,
          58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74,
          75, 76, 77, 78, 79, 80, 81, 82, 83, 87, 88, 89, 90, 91, 92, 93, 94,
          95, 96, 97, 98, 99, 100, 101, 102, 103)

def PathFromAtomList(mol, amap):
    out = []
    for i, j in combinations(amap, 2):
        bond = mol.GetBondBetweenAtoms(i, j)
        if bond:
            out.append(bond.GetIdx())
    return out

def AtomListToSubMol(mol, amap, includeConformer=False):
    """
    Parameters
    ----------
        mol: rdkit.Chem.rdchem.Mol
            Molecule
        amap: array-like
            List of atom indices (zero-based)
        includeConformer: bool (default=True)
            Toogle to include atoms coordinates in submolecule.

    Returns
    -------
        submol: rdkit.Chem.rdchem.RWMol
            Submol determined by specified atom list
    """
    if not isinstance(amap, list):
        amap = list(amap)
    submol = Chem.RWMol(Chem.Mol())
    for aix in amap:
        submol.AddAtom(mol.GetAtomWithIdx(aix))
    for i, j in combinations(amap, 2):
        bond = mol.GetBondBetweenAtoms(i, j)
        if bond:
            submol.AddBond(amap.index(i),
                           amap.index(j),
                           bond.GetBondType())
    if includeConformer:
        for conf in mol.GetConformers():
            new_conf = Chem.Conformer(len(amap))
            for i in range(len(amap)):
                new_conf.SetAtomPosition(i, conf.GetAtomPosition(amap[i]))
                new_conf.SetId(conf.GetId())
                new_conf.Set3D(conf.Is3D())
            submol.AddConformer(new_conf)
    return submol

def MolFromPDBBlock(molBlock,
                    sanitize=True,
                    removeHs=True,
                    flavor=0):
    # before 2019.03 pre-sanitization is required
    pre_sanitize = False
    if sanitize and rdkit.__version__ <= '2018.09':
        pre_sanitize = True

    mol = Chem.MolFromPDBBlock(molBlock,
                               sanitize=pre_sanitize,
                               removeHs=removeHs,
                               flavor=flavor)
    if mol is None:
        return None
    # Adjust connectivity
    for atom in mol.GetAtoms():
        res = atom.GetPDBResidueInfo()
        if res is None:
            continue
        res_name = res.GetResidueName()
        atom_name = res.GetName().strip()

        # Fix missing double bonds in RDKit - double bonds
        if atom_name == 'O' and not res.GetIsHeteroAtom() and atom.GetDegree() == 1:
            atom.SetNoImplicit(True)
            atom.GetBonds()[0].SetBondType(Chem.BondType.DOUBLE)

        # Double bonds in sidechains
        if res_name in ['HID', 'HIE', 'HIP']:
            if atom_name == 'CD2':
                for bond in atom.GetBonds():
                    if bond.GetOtherAtom(atom).GetPDBResidueInfo().GetName().strip() == 'CG':
                        bond.SetBondType(Chem.BondType.DOUBLE)
                        break
            if res_name == 'HID':
                if atom_name == 'CE1':
                    for bond in atom.GetBonds():
                        if bond.GetOtherAtom(atom).GetPDBResidueInfo().GetName().strip() == 'ND1':
                            bond.SetBondType(Chem.BondType.DOUBLE)
                            break
            elif res_name in ['HIE', 'HIP']:
                if atom_name == 'CE1':
                    for bond in atom.GetBonds():
                        if bond.GetOtherAtom(atom).GetPDBResidueInfo().GetName().strip() == 'NE2':
                            bond.SetBondType(Chem.BondType.DOUBLE)
                            break
    mol.UpdatePropertyCache(strict=sanitize)

    # Set metal coordination (zero order) bond orders to single to prevent adding Hs
    if rdkit.__version__ >= '2018.03':
        for bond in mol.GetBonds():
            if bond.GetBondType() == Chem.BondType.ZERO:
                a1 = bond.GetBeginAtom()
                a2 = bond.GetEndAtom()
                # single bonds only if there are enough electrons
                if ((a1.GetAtomicNum() in METALS and
                     a2.GetNumImplicitHs() + a2.GetNumExplicitHs() > 0) or
                    (a2.GetAtomicNum() in METALS and
                     a1.GetNumImplicitHs() + a1.GetNumExplicitHs() > 0)):
                    bond.SetBondType(Chem.BondType.SINGLE)

    if sanitize:
        result = Chem.SanitizeMol(mol)
        if result != 0:
            return None

    # Debug
    # for atom in mol.GetAtoms():
    #     res = atom.GetPDBResidueInfo()
    #     if res is None:
    #         continue
    #     res_name = res.GetResidueName()
    #     atom_name = res.GetName().strip()
    #     if atom_name in ['NE2', 'ND1'] and res_name in ['HID', 'HIE', 'HIS']:
    #         print(res_name,
    #               atom_name,
    #               atom.GetDegree(),
    #               atom.GetTotalValence(),
    #               atom.GetNumExplicitHs(),
    #               atom.GetNumImplicitHs(),
    #               sum(n.GetAtomicNum() == 1 for n in atom.GetNeighbors()),
    #               sep='\t')

    return mol

# Mol2 Atom typing
def _sybyl_atom_type(atom):
    """ Asign sybyl atom type
    Reference #1: http://www.tripos.com/mol2/atom_types.html
    Reference #2: http://chemyang.ccnu.edu.cn/ccb/server/AIMMS/mol2.pdf
    """
    sybyl = None
    atom_symbol = atom.GetSymbol()
    atomic_num = atom.GetAtomicNum()
    hyb = atom.GetHybridization()-1  # -1 since 1 = sp, 2 = sp1 etc
    hyb = min(hyb, 3)
    degree = atom.GetDegree()
    aromtic = atom.GetIsAromatic()

    # define groups for atom types
    guanidine = '[NX3,NX2]([!O,!S])!@C(!@[NX3,NX2]([!O,!S]))!@[NX3,NX2]([!O,!S])'  # strict
    # guanidine = '[NX3]([!O])([!O])!:C!:[NX3]([!O])([!O])' # corina compatible
    # guanidine = '[NX3]!@C(!@[NX3])!@[NX3,NX2]'
    # guanidine = '[NX3]C([NX3])=[NX2]'
    # guanidine = '[NX3H1,NX2,NX3H2]C(=[NH1])[NH2]' # previous
    #

    if atomic_num == 6:
        if aromtic:
            sybyl = 'C.ar'
        elif degree == 3 and _atom_matches_smarts(atom, guanidine):
            sybyl = 'C.cat'
        else:
            sybyl = '%s.%i' % (atom_symbol, hyb)
    elif atomic_num == 7:
        if aromtic:
            sybyl = 'N.ar'
        elif _atom_matches_smarts(atom, 'C(=[O,S])-N'):
            sybyl = 'N.am'
        elif degree == 3 and _atom_matches_smarts(atom, '[$(N!-*),$([NX3H1]-*!-*)]'):
            sybyl = 'N.pl3'
        elif _atom_matches_smarts(atom, guanidine):  # guanidine has N.pl3
            sybyl = 'N.pl3'
        elif degree == 4 or hyb == 3 and atom.GetFormalCharge():
            sybyl = 'N.4'
        else:
            sybyl = '%s.%i' % (atom_symbol, hyb)
    elif atomic_num == 8:
        # http://www.daylight.com/dayhtml_tutorials/languages/smarts/smarts_examples.html
        if degree == 1 and _atom_matches_smarts(atom, '[CX3](=O)[OX1H0-]'):
            sybyl = 'O.co2'
        elif degree == 2 and not aromtic:  # Aromatic Os are sp2
            sybyl = 'O.3'
        else:
            sybyl = 'O.2'
    elif atomic_num == 16:
        # http://www.daylight.com/dayhtml_tutorials/languages/smarts/smarts_examples.html
        if degree == 3 and _atom_matches_smarts(atom, '[$([#16X3]=[OX1]),$([#16X3+][OX1-])]'):
            sybyl = 'S.O'
        # https://github.com/rdkit/rdkit/blob/master/Data/FragmentDescriptors.csv
        elif _atom_matches_smarts(atom, 'S(=,-[OX1;+0,-1])(=,-[OX1;+0,-1])(-[#6])-[#6]'):
            sybyl = 'S.o2'
        else:
            sybyl = '%s.%i' % (atom_symbol, hyb)
    elif atomic_num == 15 and hyb == 3:
        sybyl = '%s.%i' % (atom_symbol, hyb)

    if not sybyl:
        sybyl = atom_symbol
    return sybyl

def _atom_matches_smarts(atom, smarts):
    idx = atom.GetIdx()
    patt = Chem.MolFromSmarts(smarts)
    for m in atom.GetOwningMol().GetSubstructMatches(patt):
        if idx in m:
            return True
    return False

def _amide_bond(bond):
    a1 = bond.GetBeginAtom()
    a2 = bond.GetEndAtom()
    if (a1.GetAtomicNum() == 6 and a2.GetAtomicNum() == 7 or
            a2.GetAtomicNum() == 6 and a1.GetAtomicNum() == 7):
        # https://github.com/rdkit/rdkit/blob/master/Data/FragmentDescriptors.csv
        patt = Chem.MolFromSmarts('C(=O)-N')
        for m in bond.GetOwningMol().GetSubstructMatches(patt):
            if a1.GetIdx() in m and a2.GetIdx() in m:
                return True
    return False

def MolFromPDBQTBlock(block, sanitize=True, removeHs=True):
    """Read PDBQT block to a RDKit Molecule

    Parameters
    ----------
        block: string
            Residue name which explicitly pint to a ligand(s).
        sanitize: bool (default=True)
            Should the sanitization be performed
        removeHs: bool (default=True)
            Should hydrogens be removed when reading molecule.

    Returns
    -------
        mol: rdkit.Chem.rdchem.Mol
            Molecule read from PDBQT
    """
    pdb_lines = []
    name = ''
    data = {}
    for line in block.split('\n'):
        # Get all know data from REMARK section
        if line[:12] == 'REMARK  Name':
            name = line[15:].strip()
        elif line[:18] == 'REMARK VINA RESULT':
            tmp = line[19:].split()
            data['vina_affinity'] = tmp[0]
            data['vina_rmsd_lb'] = tmp[1]
            data['vina_rmsd_ub'] = tmp[2]

        # no more data to collect
        if line[:4] != 'ATOM':
            continue

        pdb_line = line[:56]
        pdb_line += '1.00  0.00           '

        # Do proper atom type lookup
        atom_type = line[71:].split()[1]
        if atom_type == 'A':
            atom_type = 'C'
        elif atom_type[:1] == 'O':
            atom_type = 'O'
        elif atom_type[:1] == 'H':
            atom_type = 'H'
            if removeHs:
                continue
        elif atom_type == 'NA':
            atom_type = 'N'
        elif atom_type == 'SA':
            atom_type = 'S'

        pdb_lines.append(pdb_line + atom_type)
    mol = Chem.MolFromPDBBlock('\n'.join(pdb_lines), sanitize=False)
    if sanitize:
        Chem.SanitizeMol(mol)
    else:
        Chem.GetSSSR(mol)
    # reorder atoms using serial
    new_order = sorted(range(mol.GetNumAtoms()),
                       key=lambda i: (mol.GetAtomWithIdx(i)
                                      .GetPDBResidueInfo()
                                      .GetSerialNumber()))
    mol = Chem.RenumberAtoms(mol, new_order)

    # properties must be set on final copy of Mol, RenumberAtoms purges data
    mol.SetProp('_Name', name)
    for k, v in data.items():
        mol.SetProp(str(k), str(v))

    return mol

def PDBQTAtomLines(mol, donors, acceptors):
    """Create a list with PDBQT atom lines for each atom in molecule. Donors
    and acceptors are given as a list of atom indices.
    """

    atom_lines = [line.replace('HETATM', 'ATOM  ')
                  for line in Chem.MolToPDBBlock(mol).split('\n')
                  if line.startswith('HETATM') or line.startswith('ATOM')]

    pdbqt_lines = []
    for idx, atom in enumerate(mol.GetAtoms()):
        pdbqt_line = atom_lines[idx][:56]

        pdbqt_line += '0.00  0.00    '  # append empty vdW and ele
        # Get charge
        charge = 0.
        fields = ['_MMFF94Charge', '_GasteigerCharge', '_TriposPartialCharge']
        for f in fields:
            if atom.HasProp(f):
                charge = atom.GetDoubleProp(f)
                break
        # FIXME: this should not happen, blame RDKit
        if isnan(charge) or isinf(charge):
            charge = 0.
        pdbqt_line += ('%.3f' % charge).rjust(6)

        # Get atom type
        pdbqt_line += ' '
        atomicnum = atom.GetAtomicNum()
        atomhybridization = atom.GetHybridization()
        atombondsnum = atom.GetDegree()
        if atomicnum == 6 and atom.GetIsAromatic():
            pdbqt_line += 'A '
        elif atomicnum == 7 and ((idx in acceptors) or (atomhybridization == Chem.HybridizationType.SP3 and atombondsnum == 3) ):
            pdbqt_line += 'NA'
        elif atomicnum == 8 and idx in acceptors:
            pdbqt_line += 'OA'
        elif atomicnum == 1 and atom.GetNeighbors()[0].GetIdx() in donors:
            pdbqt_line += 'HD'
        elif atomicnum == 1 and atom.GetNeighbors()[0].GetIdx() not in donors:
            pdbqt_line += 'H '
        elif atomicnum == 16 and ( (atomhybridization == Chem.HybridizationType.SP3 and atombondsnum != 4) or atomhybridization == Chem.HybridizationType.SP2 ):
            pdbqt_line += 'SA'
        else:
            if len(atom.GetSymbol()) >1:
                pdbqt_line += atom.GetSymbol()
            else:
                pdbqt_line += (atom.GetSymbol() + ' ')
        pdbqt_lines.append(pdbqt_line)
    return pdbqt_lines

def MolToPDBQTBlock(mol, flexible=True, addHs=False, computeCharges=False):
    """Write RDKit Molecule to a PDBQT block

    Parameters
    ----------
        mol: rdkit.Chem.rdchem.Mol
            Molecule with a protein ligand complex
        flexible: bool (default=True)
            Should the molecule encode torsions. Ligands should be flexible,
            proteins in turn can be rigid.
        addHs: bool (default=False)
            The PDBQT format requires at least polar Hs on donors. By default Hs
            are added.
        computeCharges: bool (default=False)
            Should the partial charges be automatically computed. If the Hs are
            added the charges must and will be recomputed. If there are no
            partial charge information, they are set to 0.0.

    Returns
    -------
        block: str
            String wit PDBQT encoded molecule
    """
    # make a copy of molecule
    mol = Chem.Mol(mol)

    # if flexible molecule contains multiple fragments write them separately
    if flexible and len(Chem.GetMolFrags(mol)) > 1:
        return ''.join(MolToPDBQTBlock(frag, flexible=flexible, addHs=addHs, computeCharges=computeCharges)
                       for frag in Chem.GetMolFrags(mol, asMols=True))

    # Identify donors and acceptors for atom typing
    # Acceptors
    patt = Chem.MolFromSmarts('[$([O;H1;v2]),'
                              '$([O;H0;v2;!$(O=N-*),'
                              '$([O;-;!$(*-N=O)]),'
                              '$([o;+0])]),'
                              '$([n;+0;!X3;!$([n;H1](cc)cc),'
                              '$([$([N;H0]#[C&v4])]),'
                              '$([N&v3;H0;$(Nc)])]),'
                              '$([F;$(F-[#6]);!$(FC[F,Cl,Br,I])])]')
    acceptors = list(map(lambda x: x[0], mol.GetSubstructMatches(patt, maxMatches=mol.GetNumAtoms())))
    # Donors
    patt = Chem.MolFromSmarts('[$([N&!H0&v3,N&!H0&+1&v4,n&H1&+0,$([$([Nv3](-C)(-C)-C)]),'
                              '$([$(n[n;H1]),'
                              '$(nc[n;H1])])]),'
                              # Guanidine can be tautormeic - e.g. Arginine
                              '$([NX3,NX2]([!O,!S])!@C(!@[NX3,NX2]([!O,!S]))!@[NX3,NX2]([!O,!S])),'
                              '$([O,S;H1;+0])]')
    donors = list(map(lambda x: x[0], mol.GetSubstructMatches(patt, maxMatches=mol.GetNumAtoms())))
    if addHs:
        mol = Chem.AddHs(mol, addCoords=True, onlyOnAtoms=donors, )
    if addHs or computeCharges:
        AllChem.ComputeGasteigerCharges(mol)

    atom_lines = PDBQTAtomLines(mol, donors, acceptors)
    assert len(atom_lines) == mol.GetNumAtoms()

    pdbqt_lines = []

    # vina scores
    if (mol.HasProp('vina_affinity') and mol.HasProp('vina_rmsd_lb') and mol.HasProp('vina_rmsd_lb')):
        pdbqt_lines.append('REMARK VINA RESULT:  ' +
                           ('%.1f' % float(mol.GetProp('vina_affinity'))).rjust(8) +
                           ('%.3f' % float(mol.GetProp('vina_rmsd_lb'))).rjust(11) +
                           ('%.3f' % float(mol.GetProp('vina_rmsd_ub'))).rjust(11))

    pdbqt_lines.append('REMARK  Name = ' + (mol.GetProp('_Name') if mol.HasProp('_Name') else ''))
    if flexible:
        # Find rotatable bonds
        '''
        rot_bond = Chem.MolFromSmarts('[!$(*#*)&!D1&!$(C(F)(F)F)&'
                                      '!$(C(Cl)(Cl)Cl)&'
                                      '!$(C(Br)(Br)Br)&'
                                      '!$(C([CH3])([CH3])[CH3])&'
                                      '!$([CD3](=[N,O,S])-!@[#7,O,S!D1])&'
                                      '!$([#7,O,S!D1]-!@[CD3]=[N,O,S])&'
                                      '!$([CD3](=[N+])-!@[#7!D1])&'
                                      '!$([#7!D1]-!@[CD3]=[N+])]-!@[!$(*#*)&'
                                      '!D1&!$(C(F)(F)F)&'
                                      '!$(C(Cl)(Cl)Cl)&'
                                      '!$(C(Br)(Br)Br)&'
                                      '!$(C([CH3])([CH3])[CH3])]')
        '''

        #rot_bond = Chem.MolFromSmarts('[!$([NH]!@C(=O))&!D1&!$(*#*)]-&!@[!$([NH]!@C(=O))&!D1&!$(*#*)]') # From Chemaxon
        rot_bond  = Chem.MolFromSmarts('[!$(*#*)&!D1]-&!@[!$(*#*)&!D1]') #single and not ring, really not in ring?

        exclude_list = ['[NX3]-[CX3]=[O,N]','C(=[O;!R])N']
        #, '[!#1]-[C;H3,Cl3,F3,Br3]']
        excluded_atoms = []
        for excluded_smarts in exclude_list:
            excluded_bond = Chem.MolFromSmarts(excluded_smarts)
            excluded_atoms += [(x[0],x[1]) for x in list(mol.GetSubstructMatches(excluded_bond))]

        tertiary_amide_bonds = Chem.MolFromSmarts('[NX3]([!#1])([!#1])-[CX3]=[O,N]')

        bond_atoms = list(mol.GetSubstructMatches(rot_bond))

        tertiary_amide_bond_atoms=[(x[0],x[3]) for x in list(mol.GetSubstructMatches(tertiary_amide_bonds))]
        for excld_atom in excluded_atoms:
            #bond_atoms.remove(amide_bond_atom)
            excld_atom_reverse=(excld_atom[1],excld_atom[0])
            if excld_atom in bond_atoms:
                bond_atoms.remove(excld_atom)
            elif excld_atom_reverse in bond_atoms:
                bond_atoms.remove(excld_atom_reverse)

        if len(tertiary_amide_bond_atoms) > 0:
            for tertiary_amide_bond_atom in tertiary_amide_bond_atoms:
                bond_atoms.append(tertiary_amide_bond_atom)

        # Fragment molecule on bonds to ge rigid fragments
        bond_ids = [mol.GetBondBetweenAtoms(a1, a2).GetIdx()
                    for a1, a2 in bond_atoms]
        if bond_ids:
            for i, b_index in enumerate(bond_ids):
                tmp_frags= Chem.FragmentOnBonds(mol, [b_index], addDummies=False)
                tmp_frags_list=list(Chem.GetMolFrags(tmp_frags))
                #tmp_bigger=0
                if len(tmp_frags_list) == 1:
                    del bond_ids[i]
                    del bond_atoms[i]
                #else:
                #    tmp_bigger= max(len(tmp_frags_list[0]), len(tmp_frags_list[1]))
                #mol.GetBonds()[b_index].SetProp("large_part", str(tmp_bigger))
            mol_rigid_frags = Chem.FragmentOnBonds(mol, bond_ids, addDummies=False)
        else:
            mol_rigid_frags = mol

        num_torsions = len(bond_atoms)
        # Active torsions header
        pdbqt_lines.append('REMARK  %i active torsions:' % num_torsions)
        pdbqt_lines.append('REMARK  status: (\'A\' for Active; \'I\' for Inactive)')
        for i, (a1, a2) in enumerate(bond_atoms):
            pdbqt_lines.append('REMARK%5.0i  A    between atoms: _%i  and  _%i'
                               % (i + 1, a1 + 1, a2 + 1))

        frags = list(Chem.GetMolFrags(mol_rigid_frags))
        #list frag  from which bonds ?
        fg_bonds=[]
        fg_num_rotbonds={}
        for fg in frags:
            tmp_bonds=[]
            for a1,a2 in bond_atoms:
                if a1 in fg or a2 in fg:
                    tmp_bonds.append(mol.GetBondBetweenAtoms(a1, a2).GetIdx())
            if tmp_bonds:
                fg_bonds.append(tmp_bonds)
            else:
                fg_bonds.append(None)
            fg_num_rotbonds[fg] = len(tmp_bonds)

        # frag with long branch ?
        fg_bigbranch={}
        for i, fg_bond in enumerate(fg_bonds):
            tmp_bigger=0
            frag_i_mol=frags[i]
            if fg_bond != None: # for rigid mol
                tmp_frags= Chem.FragmentOnBonds(mol, fg_bond, addDummies=False)
                tmp_frags_list=list(Chem.GetMolFrags(tmp_frags))
                for tmp_frag_j in tmp_frags_list:
                    len_tmp_fg_j=len(tmp_frag_j)
                    if frag_i_mol == tmp_frag_j:
                        pass
                    else:
                        if len_tmp_fg_j > tmp_bigger:
                            tmp_bigger=len_tmp_fg_j
                #print(f'REMARK FRAG: {i} : {len(frags[i])} : {tmp_bigger} ')
            fg_bigbranch[frags[i]] = tmp_bigger

        #def weigh_frags(frag):
            """sort by the fragment size and the number of bonds (secondary)"""
            #num_bonds = 0
            # bond_weight = 0
            #big_frag_size=0
            #for a1, a2 in bond_atoms:
            #    if a1 in frag or a2 in frag:
            #        num_bonds += 1
                    #big_frag_size = max(big_frag_size, int(mol.GetBondBetweenAtoms(a1, a2).GetProp("large_part")))
            # changed signs are fixing mixed sorting type (ascending/descending)
            #return -num_bonds, -len(frag),  # bond_weight
        def weigh_frags(frag):
            return fg_bigbranch[frag], -fg_num_rotbonds[frag],   # bond_weight
        frags = sorted(frags, key=weigh_frags)

        # Start writting the lines with ROOT
        pdbqt_lines.append('ROOT')
        frag = frags.pop(0)
        for idx in frag:
            pdbqt_lines.append(atom_lines[idx])
        pdbqt_lines.append('ENDROOT')

        # Now build the tree of torsions usign DFS algorithm. Keep track of last
        # route with following variables to move down the tree and close branches
        branch_queue = []
        current_root = frag
        old_roots = [frag]

        visited_frags = []
        visited_bonds = []
        while len(frags) > len(visited_frags):
            end_branch = True
            for frag_num, frag in enumerate(frags):
                for bond_num, (a1, a2) in enumerate(bond_atoms):
                    if (frag_num not in visited_frags and
                        bond_num not in visited_bonds and
                        (a1 in current_root and a2 in frag or
                         a2 in current_root and a1 in frag)):
                        # direction of bonds is important
                        if a1 in current_root:
                            bond_dir = '%i %i' % (a1 + 1, a2 + 1)
                        else:
                            bond_dir = '%i %i' % (a2 + 1, a1 + 1)
                        pdbqt_lines.append('BRANCH %s' % bond_dir)
                        for idx in frag:
                            pdbqt_lines.append(atom_lines[idx])
                        branch_queue.append('ENDBRANCH %s' % bond_dir)

                        # Overwrite current root and stash previous one in queue
                        old_roots.append(current_root)
                        current_root = frag

                        # remove used elements from stack
                        visited_frags.append(frag_num)
                        visited_bonds.append(bond_num)

                        # mark that we dont want to end branch yet
                        end_branch = False
                        break
                    else:
                        continue
                    break  # break the outer loop as well

            if end_branch:
                pdbqt_lines.append(branch_queue.pop())
                if old_roots:
                    current_root = old_roots.pop()
        # close opened branches if any is open
        while len(branch_queue):
            pdbqt_lines.append(branch_queue.pop())
        pdbqt_lines.append('TORSDOF %i' % num_torsions)
    else:
        pdbqt_lines.extend(atom_lines)

    return '\n'.join(pdbqt_lines)

def HelpAndExit():
    print("USAGE: rdkit2pdbqt.py -r protein.pdb")
    print("USAGE: rdkit2pdbqt.py -l ligand.sdf")
    exit()

if __name__ == "__main__":
    #parsing the command
    if len(sys.argv) != 3:
        HelpAndExit()

    if sys.argv[1] == '-r':
        try:
            receptor_mol=Chem.MolFromPDBFile(sys.argv[2], removeHs=False)
            #AllChem.AssignBondOrdersFromTemplate(receptor_mol,receptor_mol)
            # detect aromaticity
            #AllChem.AssignAtomChiralTagsFromStructure(receptor_mol)
            #AllChem.AssignBondOrdersFromTemplate(Chem.MolFromSmarts('[a]'), receptor_mol)
            #Chem.AssignStereochemistry(receptor_mol)

            receptor_lines=MolToPDBQTBlock(receptor_mol, False, False, True)
            print(receptor_lines)
        except:
            raise TypeError('Protein is not readable by rdkit.')

    elif sys.argv[1] == '-l':
        mol=Chem.MolFromMolFile(sys.argv[2],removeHs=False, sanitize=False)
        mol_problems = Chem.DetectChemistryProblems(mol)
        if len(mol_problems) > 0:
            for problem in mol_problems:
                if "N, 4" in problem.Message():
                    at_idx = problem.GetAtomIdx()
                    atom = mol.GetAtomWithIdx(at_idx)
                    chg = atom.GetFormalCharge()
                    print(f'REMARK    N {at_idx} with formal charge {chg}')
                    atom.SetFormalCharge(1)
                    atom.UpdatePropertyCache()
                else:
                    print(problem.Message())
                    exit()
        Chem.SanitizeMol(mol)
        pdbqtlines=MolToPDBQTBlock(mol, True, False, True)
        print(pdbqtlines)
        exit()

    else:
        HelpAndExit()


from django.contrib import admin
from jet.filters import DateRangeFilter
from django.db.models import Sum, Avg

from admin_totals.admin import ModelAdminTotals
from import_export.admin import ImportExportModelAdmin, ExportMixin

from .models import inventory, VaForm

class InventoryProfile(ImportExportModelAdmin):
    list_display = ('date_lead_received', 'type_of_form',
                    'client_full_name', 'full_name_of_lead',
                    'phone_number', 'status', 'lead_conversion', 'financial_status',
                    'call_duration', 'total_time_transferring_leads', 'total_mins')
    list_filter = ('type_of_form', 'status', 'financial_status',
                   'client_full_name', 'lead_conversion',
                   ('transferred_date', DateRangeFilter),
                   ('date_lead_received', DateRangeFilter))
    list_totals = [('call_duration', Sum), ('total_time_transferring_leads', Sum),
                   ('total_mins', Sum)]
    search_fields = ('client_full_name',
                     'customer_representative', 'type_of_form', 'status',
                     'financial_status')
    readonly_fields = ('total_mins',)
    fieldsets = (
        ("Call Me Inventory Informations", {
            'fields': (
                'date_lead_received',
                'type_of_form',
                'client_full_name',
                'full_name_of_lead',
                'phone_number',
                'email',
                'lead_conversion',
                'customer_representative',
                'lead_transferred_by',
                'transferred_date',
                'status',
                'financial_status',
                'call_duration',
                'total_time_transferring_leads',
                'total_mins',
                'notes',
            )
        }),
    )

class VaFormProfile(ImportExportModelAdmin):
    list_display = ('client_full_name', 'type_of_script',
                    'client_call_forwarding_number', 'company_call_forwarding_number',
                    'created_at', 'updated_at')
    list_filter = ('client_full_name', 'type_of_script',
                   ('created_at', DateRangeFilter),
                   ('updated_at', DateRangeFilter))
    search_fields = ('client_full_name__full_name',)
    readonly_fields = ('created_at', 'updated_at')
    fieldsets = (
        ("Call Me Forms Information", {
            'fields': (
                'client_full_name',
                'type_of_script',
                'script_link',
                'gs_integration',
                'client_call_forwarding_number',
                'company_call_forwarding_number',
                'notes'
            )
        }),
    )

admin.site.register(inventory, InventoryProfile)
admin.site.register(VaForm, VaFormProfile)


from django.db import transaction

from tulius.forum.threads import models
from tulius.games import models as game_models

def test_room_smoke(
        game, variation_forum, user, murderer, detective, admin):
    game.status = game_models.GAME_STATUS_IN_PROGRESS
    with transaction.atomic():
        game.save()
    # create room
    response = admin.put(
        variation_forum.get_absolute_url(), {
            'title': 'room', 'body': 'room description',
            'room': True, 'default_rights': models.ACCESS_READ,
            'granted_rights': []})
    assert response.status_code == 200

def test_threads_counters(game, variation_forum, user, detective, admin):
    game.status = game_models.GAME_STATUS_IN_PROGRESS
    with transaction.atomic():
        game.save()
    # create room
    response = admin.put(
        variation_forum.get_absolute_url(), {
            'title': 'room', 'body': 'room description',
            'room': True, 'default_rights': None,
            'granted_rights': []})
    assert response.status_code == 200
    room = response.json()
    # create hidden thread
    response = admin.put(
        room['url'], {
            'title': 'room', 'body': 'room description',
            'room': False, 'default_rights': models.NO_ACCESS,
            'granted_rights': [], 'media': {}, 'important': False})
    assert response.status_code == 200
    thread = response.json()
    # check initial counters state
    response = user.get(variation_forum.get_absolute_url())
    assert response.status_code == 200
    data = response.json()
    assert data['rooms'][0]['threads_count'] == 0
    # grant rights
    response = admin.post(
        thread['url'] + 'granted_rights/', {
            'user': {'id': detective.pk},
            'access_level': models.ACCESS_READ
        }
    )
    assert response.status_code == 200
    # check counters state
    response = user.get(variation_forum.get_absolute_url())
    assert response.status_code == 200
    data = response.json()
    assert data['rooms'][0]['threads_count'] == 1


import requests
from rest_framework.serializers import (
    ModelSerializer,
    SerializerMethodField,
    CharField,
    ValidationError,
)

from pokeapi.settings import POKEMON_API_URL

from pokemons.models import Pokemon

class PokemonSerializer(ModelSerializer):
    """Basic Pokemon serializer."""
    class Meta:
        model = Pokemon
        fields = [
            'uuid',
            'name',
        ]

class PokemonDetailSerializer(ModelSerializer):
    """Pokemon serializer with details from external API."""

    name = CharField()
    info = SerializerMethodField()

    class Meta:
        model = Pokemon
        fields = [
            'uuid',
            'name',
            'info',
        ]

    def get_pokemon(self, name):
        """
        Get pokemon info from external API
        """
        response = requests.get(POKEMON_API_URL + name)
        if response.status_code == 404:
            raise ValidationError('Pokemon not found')
        if response.status_code != 200:
            raise ValidationError('Pokemon API is down')
        return response.json()

    def get_info(self, obj):
        pokemon = self.get_pokemon(obj.name)
        return pokemon

    def validate(self, attrs):
        self.get_pokemon(attrs['name'].lower())
        attrs['trainer'] = self.context['request'].user.trainer
        return attrs


# -*- coding: utf-8 -*-
import requests
import random
import socket
import struct

HEADERS = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
    'Referer': 'http://www.chsi.com.cn/cet',
    'X-FORWARDED-FOR':'',
	'CLIENT-IP':''
}

param={
        'zkzh':'',
        'xm':''}

param['xm']='顾振园' #单引号内修改为自己的姓名，例如param['xm']='张三'
xxdm = 420013 #学校代码，就是同校考试的同学准考证前六位，一样的
type = 2 #四级修改为1，六级修改为2
kc = 1 #考场默认从1开始，可以自行修改
zwh = 1 #座位号默认从1开始，可以自行修改
zwh_gd = 0 #已知座位号的请把0修改为1，并把上一行的1改为你的座位号

zkzh = (((xxdm*1000 + 172)*10+ type)*1000 + kc) * 100 + zwh #切勿修改此处
param['zkzh']=zkzh
print (param['zkzh'])

while 1:
    IP = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
    HEADERS['X-FORWARDED-FOR'] = IP
    HEADERS['CLIENT-IP'] = IP
    try:
        rsp = requests.get('http://www.chsi.com.cn/cet/query',params=param, headers=HEADERS)
    except requests.exceptions.ConnectionError:
        continue
    except requests.exceptions.HTTPError:
        continue
    if '写作和翻译' in rsp.text:
        print(param['zkzh'], '查询成功')
        break
    else:
        print(param['zkzh'], '尝试失败')
        if zwh_gd:
            zkzh = zkzh + 100
        else:
            zkzh += 1
            temp = zkzh - 31
            if temp % 100 == 0:
                zkzh = zkzh + 70
        param['zkzh'] = zkzh
raw_input()


from imblearn.pipeline import Pipeline
from imblearn.under_sampling import RandomUnderSampler
from scikeras.wrappers import KerasClassifier
from sklearn.compose import ColumnTransformer
from sklearn.decomposition import PCA
from sklearn.ensemble import RandomForestClassifier, StackingClassifier
from sklearn.linear_model import LogisticRegression, RidgeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from tensorflow import keras
import tensorflow as tf
from cursos_df import seed_value, X

def to_dense_f(x):
    return x.toarray()

to_dense = FunctionTransformer(to_dense_f, accept_sparse=True)

def logistic_regression(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("logistic_regression", LogisticRegression(random_state=seed_value)),
        ]),
        "params": {
            'logistic_regression__C': [0.1, 1.0, 10.0],
            'logistic_regression__solver': ['lbfgs', 'saga'],
            'logistic_regression__max_iter': [100, 200],
            'logistic_regression__penalty': ['l1', 'l2'],
            **additional_params
        }
    }

def ridge_classifier(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("ridge_classifier", RidgeClassifier(random_state=seed_value)),
        ]),
        "params": {
            'ridge_classifier__alpha': [0.5],
            'ridge_classifier__solver': ['sag', 'lsqr', 'sparse_cg', 'lbfgs'],
            'ridge_classifier__max_iter': [100, 200],
            **additional_params
        }
    }

def decision_tree(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("decision_tree", DecisionTreeClassifier(random_state=seed_value)),
        ]),
        "params": {
            'decision_tree__max_depth': [10, 30, 50],
            'decision_tree__min_samples_split': [5, 10, 30],
            'decision_tree__min_samples_leaf': [1, 5, 10],
            **additional_params
        }
    }

def random_forest(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("random_forest", RandomForestClassifier(random_state=seed_value)),
        ]),
        "params": {
            'random_forest__n_estimators': [150, 300],
            'random_forest__max_features': [5, 20, 50],
            'random_forest__max_depth': [20, 50],
            'random_forest__min_samples_leaf': [1, 2, 4],
            **additional_params
        }
    }

def naive_bayes(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("naive_bayes", GaussianNB()),
        ]),
        "params": {
            'naive_bayes__var_smoothing': [1e-9, 1e-7],
            **additional_params
        }
    }

def mlp(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("mlp", MLPClassifier(random_state=seed_value)),
        ]),
        "params": {
            'mlp__hidden_layer_sizes': [(50, 50)],
            'mlp__activation': ['logistic', 'relu'],
            'mlp__solver': ['adam'],
            'mlp__alpha': [0.1],
            'mlp__learning_rate': ['adaptive'],
            **additional_params
        }
    }

def svm(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("svm", SVC(random_state=seed_value)),
        ]),
        "params": {
            'svm__C': [0.1, 1, 10],
            'svm__gamma': ['auto'],
            'svm__kernel': ['linear'],
            **additional_params
        }
    }

def get_model(hidden_layer_dim, meta):
    n_features_in_ = meta["n_features_in_"]

    model = keras.models.Sequential()
    model.add(keras.layers.Dense(hidden_layer_dim, input_shape=(n_features_in_,), activation='relu'))
    model.add(keras.layers.Dense(hidden_layer_dim, activation='relu'))
    model.add(keras.layers.Dense(1, activation='sigmoid'))

    return model

def keras_model(additional_steps, additional_params={}):
    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            ("keras", KerasClassifier(get_model, verbose=0)),
        ]),
        "params": {
            "keras__model__hidden_layer_dim": [50, 100, 200],
            "keras__optimizer": ["adam"],
            "keras__loss": ["binary_crossentropy"],
            "keras__epochs": [1, 5, 10, 20],
            **additional_params
        }
    }

def stacking(additional_steps, additional_params={}):
    stacking_prefix = 'stacking (naive bayes, decision tree, keras) + logistic regression'

    return {
        "pipeline": Pipeline(steps=[
            *additional_steps,
            (stacking_prefix, StackingClassifier(estimators=[
                ("naive_bayes", Pipeline(steps=[("to_dense", to_dense), ("naive_bayes", GaussianNB())])),
                ("decision_tree", DecisionTreeClassifier(random_state=seed_value)),
                ("keras", KerasClassifier(get_model, verbose=0))
            ], final_estimator=LogisticRegression(random_state=seed_value)))
        ]),
        "params": {
            ('%s__naive_bayes__naive_bayes__var_smoothing' % stacking_prefix): [1e-9, 1e-7],
            ('%s__decision_tree__max_depth' % stacking_prefix): [30],
            ('%s__decision_tree__min_samples_split' % stacking_prefix): [5, 10],
            ('%s__decision_tree__min_samples_leaf' % stacking_prefix): [3, 5],
            ('%s__keras__model__hidden_layer_dim' % stacking_prefix): [100],
            ('%s__keras__optimizer' % stacking_prefix): ["adam"],
            ('%s__keras__loss' % stacking_prefix): ["binary_crossentropy"],
            ('%s__keras__epochs' % stacking_prefix): [20],
            ('%s__final_estimator__C' % stacking_prefix): [0.1],
            ('%s__final_estimator__solver' % stacking_prefix): ['saga'],
            ('%s__final_estimator__max_iter' % stacking_prefix): [200],
            ('%s__final_estimator__penalty' % stacking_prefix): ['l1'],
            **additional_params
        }
    }

preprocessor = ('preprocessor (onehot + scaler)', ColumnTransformer(
    transformers=[
        ('scaler', StandardScaler(), X.select_dtypes(include=['float64', 'int']).columns),
        ('onehot', OneHotEncoder(handle_unknown='ignore'), X.select_dtypes(include=['object', 'category']).columns),
    ],
    remainder='passthrough'
))

sampler = ("undersampler", RandomUnderSampler(random_state=seed_value))
pca = [("to_dense", to_dense), ("pca", PCA())]
pca_params = {'pca__n_components': [15, 22]}

tf.config.set_visible_devices([], 'GPU')

pipelines = [
    # logistic_regression([preprocessor, sampler]),
    decision_tree([preprocessor, sampler]),
    # random_forest([preprocessor, sampler]),
    naive_bayes([preprocessor, sampler, ("to_dense", to_dense)]),
    svm([preprocessor, sampler, *pca], additional_params={**pca_params}),
    # mlp([preprocessor, sampler]),
    keras_model([preprocessor, sampler]),
    stacking([preprocessor, sampler])
]


from decimal import Decimal
from purchases.models import Payment

from django.db.models import Sum

from .models import CreditCard

def set_payment_records(cards, purchase):
    """Chooses the best options for payment."""

    purchase_value = purchase.value

    for card in cards:
        available_before = card.available

        if not purchase_value:
            break

        if purchase_value > card.available:
            purchase_value -= card.available
            card.available = 0
        else:
            card.available -= purchase_value
            purchase_value = 0
        card.save()

        value = available_before - card.available
        if value:
            Payment.objects.create(
                purchase=purchase,
                credit_card=card,
                value=value
            )

def new_card_update_wallet(credit_card):
    """Updates the related wallet everytime a new card is inserted."""
    related_wallet = credit_card.wallet

    # sets the available limit
    credit_card.available = credit_card.limit
    credit_card.save()

    # updates related wallet limit and available
    if not related_wallet.credit_available:
        related_wallet.credit_available = Decimal('0.00')

    related_wallet.credit_limit += Decimal(credit_card.limit)
    related_wallet.credit_available += Decimal(credit_card.limit)
    related_wallet.save()

def new_purchase_update_wallet(purchase):
    """Updates the related wallet everytime a new purchase is made."""
    related_wallet = purchase.wallet

    cards = related_wallet.credit_cards.all()
    set_payment_records(cards, purchase)

    result = CreditCard.objects.aggregate(Sum('available'))
    related_wallet.credit_available = result['available__sum']
    related_wallet.save()

def bill_paid_update_wallet(bill, value_paid):
    """Updates the related wallet everytime a bill is paid."""
    wallet = bill.credit_card.wallet

    wallet.credit_available += value_paid
    wallet.save()

def bill_paid_update_credit_card(bill, value_paid):
    """Updates the related wallet everytime a bill is paid."""
    credit_card = bill.credit_card

    credit_card.available += value_paid
    credit_card.save()

def card_delete_update_wallet(card):
    """Updates the related wallet everytime a card is deleted."""
    wallet = card.wallet

    wallet.credit_limit -= card.limit
    wallet.credit_available -= card.available
    wallet.save()


import pandas as pd
import math
import cmath
import numpy as np
import matplotlib.pyplot as plt
pi2 = cmath.pi * 2.0
def DFT(fnList):
    N = len(fnList)
    FmList = []
    for m in range(N):
        Fm = 0.0
        for n in range(N):
            Fm += fnList[n] * cmath.exp(- 1j * pi2 * m * n / N)
        FmList.append(Fm / N)
        print(m)
    return FmList

def inverseDft(df2):
    FmList=np.array(df2)
    N = len(FmList)
    fnList = []
    for n in range(N):
        fn = 0.0
        for m in range(N):
            fn += FmList[m] * cmath.exp(1j * pi2 * m * n / N)
        fnList.append(fn)
        print(n)
    return fnList

w=[]
index =[]
wlow=[]
reader=pd.read_csv('level2/2016years_level2/test1.csv')
load_number=5
df = pd.DataFrame(reader)
df['time'] = pd.to_datetime(df['time'])
df_show1 = df.loc[(df["time"].dt.month ==10 )&(df["time"].dt.day >=15 ),:]
sample=df_show1.load
sample=sample/load_number

FT=DFT(np.array(sample))
df11= np.zeros(len(FT))
df22= np.zeros(len(FT))
df33= np.zeros(len(FT))
df44= np.zeros(len(FT))
df1= pd.DataFrame(FT)
df2= pd.DataFrame(FT)
df3= pd.DataFrame(FT)
df4= pd.DataFrame(FT)
df2.to_csv('DFT1.csv')
wday=[0]
wweek=[]
wlow=[]
whigh=[]
T=1440

for i in range(1,len(sample)):
    if T%(len(sample)/i)==0:
        wday.append(i)
    if ((T*7)%(len(sample)/i)==0):
        wweek.append(i)
    if ((T*7)%(len(sample)/i)!=0)&(T%(len(sample)/i)!=0)&((len(sample)/i)<=T):
        wlow.append(i)
    if ((T*7)%(len(sample)/i)!=0)&(T%(len(sample)/i)!=0)&((len(sample)/i)>T):
        whigh.append(i)

for i in range(len(wday)):
        df11[ wday[i] ]=df1.loc[ wday[i] ]
for i in range(len(wweek)):
        df22[ wweek[i] ]=df2.loc[ wweek[i] ]
for i in range(len(wlow)):
        df33[ wlow[i] ]=df3.loc[ wlow[i] ]
for i in range(len(whigh)):
        df44[ whigh[i] ]=df4.loc[ whigh[i] ]

plt.subplot(2,2,1)
plt.plot(inverseDft(df11),label = 'day')
plt.legend(loc='upper left')
plt.xlabel('samples')
plt.ylabel('Load Distribution')
plt.subplot(2,2,2)
plt.plot(inverseDft(df22),label = 'week')
plt.legend(loc='upper left')
plt.xlabel('samples')
plt.ylabel('Load Distribution')
plt.subplot(2,2,3)
plt.plot(inverseDft(df33),label = 'low')
plt.legend(loc='upper left')
plt.xlabel('samples')
plt.ylabel('Load Distribution')
plt.subplot(2,2,4)
plt.plot(inverseDft(df44),label = 'high')
plt.legend(loc='upper left')
plt.xlabel('samples')
plt.ylabel('Load Distribution')

'''
# TEST
print ("Input Sine Wave Signal:")
N = 360 # degrees (Number of samples)
a = float(random.randint(1, 100))
f = float(random.randint(1, 100))
p = float(random.randint(0, 360))
print ("frequency = " + str(f))
print ("amplitude = " + str(a))
print ("phase ang = " + str(p))
fnList = []
for n in range(N):
    t = float(n) / N * pi2
    fn = a * math.sin(f * t + p / 360 * pi2)
    fnList.append(fn)

print ("DFT Calculation Results:")
FmList = DFT(fnList)
threshold = 0.001
for (i, Fm) in enumerate(FmList):
    if abs(Fm) > threshold:
        print ("frequency = " + str(i))
        print ("amplitude = " + str(abs(Fm) * 2.0))
        p = int(((cmath.phase(Fm) + pi2 + pi2 / 4.0) % pi2) / pi2 * 360 + 0.5)
        print ("phase ang = " + str(p))
        print
        '''