from typing import Optional
from xml.etree.ElementTree import Element, Comment

from api.mvc.model.data.content_model import ContentModel
from api.mvc.model.data.data_model import DataModel
from api.mvc.model.data.data_type import DataType
from api_core.exception.api_exception import ApiException
from api_core.helper.file_folder_helper import FileFolderHelper
from api_core.helper.string_helper import StringHelper
from api_core.mvc.service.file.xml_file_service import XmlFileService

class ContentModelFileService(XmlFileService):
    """
    Class used to manage content-model XML files.
    """

    def __init__(self):
        """
        Initialize a new instance of 'ContentModelService' class.
        """
        super().__init__(True, {"xmlns": "http://www.alfresco.org/model/dictionary/1.0"})

    def extract_content_model_prefix(self, content_model_file_path: str) -> str:
        """
        Extracts the content model prefix.
        :param content_model_file_path: The path to the content model file.
        :return: The content model prefix.
        """
        root: Element = self._get_root(content_model_file_path)
        filename: str = FileFolderHelper.extract_filename_from_path(content_model_file_path)

        # Verification that the attribute exists.
        if not ("name" in root.attrib):
            raise ApiException("Content model file '{0}' does not have the necessary 'namespace' node."
                               .format(filename))
        # Verification that the attribute has a value.
        if StringHelper.is_empty(root.attrib["name"]):
            raise ApiException("The 'name' attribute of the content model file '{0}' is not entered. The latter is "
                               "mandatory.".format(filename))
        # Data recovery.
        try:
            return root.attrib["name"].rsplit(":", 1)[0]
        except IndexError:
            raise ApiException("The value of the 'name' attribute of the source node is invalid. This must be composed "
                               "as follows: prefix:name")

    def extract_content_model_name(self, content_model_file_path: str) -> str:
        """
        Extracts the content model name.
        :param content_model_file_path: The path to the content model file.
        :return: The content model name.
        """
        root: Element = self._get_root(content_model_file_path)
        filename: str = FileFolderHelper.extract_filename_from_path(content_model_file_path)

        # Verification that the attribute exists.
        if not ("name" in root.attrib):
            raise ApiException("Content model file '{0}' does not have the necessary 'namespace' node."
                               .format(filename))
        # Verification that the attribute has a value.
        if StringHelper.is_empty(root.attrib["name"]):
            raise ApiException("The 'name' attribute of the content model file '{0}' is not entered. The latter is "
                               "mandatory.".format(filename))
        # Data recovery.
        try:
            return root.attrib["name"].rsplit(":", 1)[1]
        except IndexError:
            raise ApiException("The value of the 'name' attribute of the source node is invalid. This must be composed "
                               "as follows: prefix:name")

    def create_content_model(self, content_model_file_path: str, prefix: str, name: str, description: Optional[str],
                             author: Optional[str]):
        model: Element = Element("model")
        model.set("name", "{0}:{1}".format(prefix, name))

        # Set xml namespace.
        if self.namespaces is not None:
            for item in self.namespaces:
                model.set(item[0], item[1])

        model.append(Comment(" Optional meta-data about the model "))

        # Set the description
        description_node: Element = Element("description")
        description_node.text = description if description is not None else "SET THE PROJECT DESCRIPTION"
        model.append(description_node)

        # Set the author
        author_node: Element = Element("author")
        author_node.text = author if author is not None else "Alfresco Helper Script 1.0.0"
        model.append(author_node)

        # Set the version
        version_node: Element = Element("version")
        version_node.text = "1.0.0"
        model.append(version_node)

        # Set the imports
        imports_node: Element = Element("imports")
        imports_node.append(Comment(" Import Alfresco Dictionary Definitions "))
        # First import
        import1: Element = Element("import")
        import1.set("uri", "http://www.alfresco.org/model/dictionary/1.0")
        import1.set("prefix", "d")
        imports_node.append(import1)

        # Second import
        import2: Element = Element("import")
        import2.set("uri", "http://www.alfresco.org/model/content/1.0")
        import2.set("prefix", "cm")
        imports_node.append(import2)

        # Set the namespaces.
        namespaces_node: Element = Element("namespaces")
        # Set a namespace
        namespace_node: Element = Element("namespace")
        namespace_node.set("uri", "http://www.{0}.org/model/content/1.0".format(name.lower()))
        namespace_node.set("prefix", prefix)
        namespaces_node.append(namespace_node)

        # Add the import to the model.
        model.append(imports_node)
        # Add the import to the model.
        model.append(Comment(" Custom namespace for the '{0}:{1}' model ".format(prefix, name)))
        model.append(namespaces_node)

        types: Element = Element("types")
        aspects: Element = Element("aspects")

        model.append(types)
        model.append(aspects)

        # Write the XML file.
        self._write(model, content_model_file_path)

    def find_data(self, content_model: ContentModel, typology: str, data: str) -> Optional[Element]:
        """
        Finds data's node in its content model.
        :param content_model: A data model of a content-model.
        :param typology: The data typology.
        :param data: The name of the data.
        :return: The data node, otherwise None.
        """
        return self._get_root(content_model.path).find(".//{0}{3}s/{0}{3}[@name='{1}:{2}']".format(
            self.get_namespace("xmlns"), content_model.prefix, data, typology))

    def find_aspect(self, content_model: ContentModel, aspect: str) -> Optional[Element]:
        """
        Finds an aspect's node in its content model.
        :param content_model: A data model of a content-model.
        :param aspect: The name of the aspect.
        :return: The aspect node, otherwise None.
        """
        return self._get_root(content_model.path).find(".//{0}aspects/{0}aspect[@name='{1}:{2}']".format(
            self.get_namespace("xmlns"), content_model.prefix, aspect))

    def get_aspects_name(self, content_model: ContentModel) -> list[str]:
        """
        Finds an all aspects name in its content model.
        :param content_model: A data model of a content-model.
        :return: The list of aspects name.
        """
        aspects: list[str] = []
        filename: str = FileFolderHelper.extract_filename_from_path(content_model.path)
        for aspect in self._get_root(content_model.path).findall(".//{0}aspects/{0}aspect".format(
                self.get_namespace("xmlns"))):
            aspects.append(self.__extract_aspect_name(aspect, filename))
        return aspects

    def get_data_names(self, content_model: ContentModel, typology: str) -> list[str]:
        """
        Finds an all aspects name in its content model.
        :param typology: The type of the data to get.
        :param content_model: A data model of a content-model.
        :return: The list of aspects name.
        """
        data_names: list[str] = []
        filename: str = FileFolderHelper.extract_filename_from_path(content_model.path)
        for data in self._get_root(content_model.path).findall(".//{0}{1}s/{0}{1}".format(
                self.get_namespace("xmlns"), typology)):
            data_names.append(self.__extract_data_name(data, typology, filename))
        return data_names

    def find_type(self, content_model: ContentModel, type_name: str) -> Optional[Element]:
        return self._get_root(content_model.path).find(".//{0}types/{0}type[@name='{1}:{2}']".format(
            self.get_namespace("xmlns"), content_model.prefix, type_name))

    def add_aspect(self, content_model: ContentModel, name: str, title: str, description: str):
        root: Element = self._get_root(content_model.path)

        aspect: Element = Element("aspect")
        aspect.set("name", "{0}:{1}".format(content_model.prefix, name))

        if not StringHelper.is_empty(title):
            title_node: Element = Element("title")
            title_node.text = title
            aspect.append(title_node)

        if not StringHelper.is_empty(description):
            description_node: Element = Element("description")
            description_node.text = description
            aspect.append(description_node)

        properties: Element = Element("properties")
        aspect.append(properties)

        add_to_root: bool = False
        aspects: Element = root.find(".//{0}aspects".format(self.get_namespace("xmlns"), content_model.prefix, aspect))
        if aspects is None:
            aspects = Element("aspects")
            add_to_root = True

        aspects.append(Comment(" Definition of aspect '{0}'. ".format(name)))
        aspects.append(aspect)

        if add_to_root:
            root.append(aspects)

        self._write(root, content_model.path)

    def add_type(self, content_model: ContentModel, name: str, title: str, description: str):
        root: Element = self._get_root(content_model.path)

        type_node: Element = Element("type")
        type_node.set("name", "{0}:{1}".format(content_model.prefix, name))

        if not StringHelper.is_empty(title):
            title_node: Element = Element("title")
            title_node.text = title
            type_node.append(title_node)

        if not StringHelper.is_empty(description):
            description_node: Element = Element("description")
            description_node.text = description
            type_node.append(description_node)

        properties: Element = Element("properties")
        type_node.append(properties)

        add_to_root: bool = False
        types: Element = root.find(".//{0}types".format(self.get_namespace("xmlns"), content_model.prefix, type_node))
        if types is None:
            types = Element("types")
            add_to_root = True

        types.append(Comment(" Definition of type '{0}'. ".format(name)))
        types.append(type_node)

        self._write(root, content_model.path)

    def add_property(self, content_model: ContentModel, data: DataModel, name: str, title: Optional[str],
                     description: Optional[str], typology: str, mandatory: bool):
        root: Element = self._get_root(content_model.path)
        # Create the property
        prop: Element = Element("property")
        prop.set("name", "{0}:{1}".format(content_model.prefix, name))

        # Set the property's title.
        if not StringHelper.is_empty(title):
            title_node: Element = Element("title")
            title_node.text = title
            prop.append(title_node)

        # Set the property's description.
        if not StringHelper.is_empty(description):
            description_node: Element = Element("description")
            description_node.text = description
            prop.append(description_node)

        # Set the property's type.
        type_node: Element = Element("type")
        type_node.text = "d:{0}".format(typology)
        prop.append(type_node)

        # Set the property's mandatory.
        mandatory_node: Element = Element("mandatory")
        mandatory_node.text = "true" if mandatory else "false"
        prop.append(mandatory_node)

        data_node: Optional[Element] = root.find(".//{0}{3}s/{0}{3}[@name='{1}:{2}']"
                                                 .format(self.get_namespace("xmlns"), content_model.prefix, data.name,
                                                         data.typology))

        add_to_data: bool = False
        properties_node: Element = data_node.find(".//{0}properties".format(self.get_namespace("xmlns")))
        if properties_node is None:
            properties_node = Element("properties")
            add_to_data = True

        properties_node.append(prop)
        if add_to_data:
            data_node.append(properties_node)

        self._write(root, content_model.path)

    def add_extension(self, content_model: ContentModel, source: DataModel, parent: DataModel):
        namespace: str = self.get_namespace("xmlns")
        root: Element = self._get_root(content_model.path)

        source_node: Element = root.find(".//{0}{1}s/{0}{1}[@name='{2}:{3}']"
                                         .format(namespace, source.typology, content_model.prefix, source.name))
        parent_node: Optional[Element] = source_node.find("./{0}parent".format(namespace))

        add_parent: bool = True if parent_node is None else False
        if add_parent:
            parent_node = Element("parent")

        parent_node.text = "{0}".format(parent.complete_name)

        if add_parent:
            source_node.insert(self.__get_properties_node_index(source_node), parent_node)

        self._write(root, content_model.path)

    def add_mandatory(self, content_model: ContentModel, source: DataModel, mandatory: DataModel):
        namespace: str = self.get_namespace("xmlns")
        root: Element = self._get_root(content_model.path)

        source_node: Element = root.find(".//{0}{1}s/{0}{1}[@name='{2}:{3}']"
                                         .format(namespace, source.typology, content_model.prefix, source.name))
        mandatory_node: Optional[Element] = source_node.find("./{0}mandatory-aspects".format(namespace))

        aspect: Element = Element("aspect")
        aspect.text = "{0}:{1}".format(content_model.prefix, mandatory.name)

        add_mandatory_node: bool = True if mandatory_node is None else False
        if add_mandatory_node:
            mandatory_node = Element("mandatory-aspects")
        mandatory_node.append(aspect)
        if add_mandatory_node:
            source_node.append(mandatory_node)

        self._write(root, content_model.path)

    def get_aspect_description(self, content_model: ContentModel, name: str) -> Optional[str]:
        """
        Retrieve the value of the description node of an aspect node.
        :param content_model: A data model of a content-model.
        :param name: The name of the aspect node.
        :return: The value of the aspect's description node.
        """
        return self.__get_data_description(content_model, DataType.ASPECT.name, name)

    def get_type_description(self, content_model: ContentModel, name: str) -> Optional[str]:
        """
        Retrieve the value of the description node of a type node.
        :param content_model: A data model of a content-model.
        :param name: The name of the type node.
        :return: The value of the type's description node.
        """
        return self.__get_data_description(content_model, DataType.TYPE.name, name)

    def get_aspect_title(self, content_model: ContentModel, name: str) -> Optional[str]:
        """
        Retrieve the value of the title node of an aspect node.
        :param content_model: A data model of a content-model.
        :param name: The name of the aspect node.
        :return: The value of the aspect's title node.
        """
        return self.__get_data_title(content_model, DataType.ASPECT.value, name)

    def get_aspect_parent(self, content_model: ContentModel, name: str) -> Optional[str]:
        """
        Retrieve the value of the parent node of an aspect node.
        :param content_model: A data model of a content-model.
        :param name: The name of the aspect node.
        :return: The value of the aspect's parent node.
        """
        return self.get_data_parent(content_model, DataType.ASPECT.value, name)

    def get_aspect_mandatory_aspects(self, content_model: ContentModel, name: str) -> list[str]:
        return self.__get_data_mandatory_aspects(content_model, DataType.ASPECT.value, name)

    def get_type_title(self, content_model: ContentModel, name: str) -> Optional[str]:
        """
        Retrieve the value of the title node of a type node.
        :param content_model: A data model of a content-model.
        :param name: The name of the type node.
        :return: The value of the type's title node.
        """
        return self.__get_data_title(content_model, DataType.TYPE.name, name)

    def get_type_parent(self, content_model: ContentModel, name: str) -> Optional[str]:
        """
        Retrieve the value of the title node of a type node.
        :param content_model: A data model of a content-model.
        :param name: The name of the type node.
        :return: The value of the type's title node.
        """
        return self.__get_data_title(content_model, DataType.TYPE.name, name)

    def __extract_aspect_name(self, aspect: Element, filename: str) -> str:
        """
        Extracts the aspect node name.
        :param aspect: The aspect node.
        :return: The aspect name.
        """
        return self.__extract_data_name(aspect, DataType.ASPECT.name, filename)

    def get_type_mandatory_aspects(self, content_model: ContentModel, name: str) -> list[str]:
        return self.__get_data_mandatory_aspects(content_model, DataType.TYPE.value, name)

    def __extract_type_name(self, type_node: Element, filename: str) -> str:
        """
        Extracts the aspect node name.
        :param type_node: The type node.
        :return: The type name.
        """
        return self.__extract_data_name(type_node, DataType.TYPE.value, filename)

    @staticmethod
    def __extract_data_name(data: Element, typology: str, filename: str) -> str:
        """
        Extracts the aspect node name.
        :param data: The aspect model.
        :return: The aspect name.
        """
        # Verification that the attribute exists.
        if not ("name" in data.attrib):
            raise ApiException("There is {1} in file '{0}' that has not been defined correctly. It lacks the "
                               "'name' attribute."
                               .format(filename, "an aspect" if typology.__eq__("aspect") else "a type"))

        # Verification that the attribute has a value.
        if StringHelper.is_empty(data.attrib["name"]):
            raise ApiException("There is {1} in file '{0}' that has not been defined correctly. The 'name' "
                               "attribute is null or empty."
                               .format(filename, "an aspect" if typology.__eq__("aspect") else "a type"))

        # Data recovery.
        try:
            return data.attrib["name"].rsplit(":", 1)[1]
        except IndexError:
            raise ApiException("There is {1} in file '{0}' whose name attribute was not set correctly. The "
                               "attribute value must be composed as follows: prefix:name"
                               .format(filename, "an aspect" if typology.__eq__("aspect") else "a type"))

    @staticmethod
    def __extract_property_name(prop: Element, filename: str) -> str:
        """
        Extracts the aspect node name.
        :param prop: The property node.
        :return: The aspect name.
        """
        # Verification that the attribute exists.
        if not ("name" in prop.attrib):
            raise ApiException("There is a property in file '{0}' that has not been defined correctly. It lacks the "
                               "'name' attribute."
                               .format(filename))

        # Verification that the attribute has a value.
        if StringHelper.is_empty(prop.attrib["name"]):
            raise ApiException("There is a property in file '{0}' that has not been defined correctly. The 'name' "
                               "attribute is null or empty."
                               .format(filename))

        # Data recovery.
        try:
            return prop.attrib["name"].rsplit(":", 1)[1]
        except IndexError:
            raise ApiException("There is a property in file '{0}' whose name attribute was not set correctly. The "
                               "attribute value must be composed as follows: prefix:name"
                               .format(filename))

    def __get_data_description(self, content_model: ContentModel, typology: str, name: str) -> Optional[str]:
        """
        Retrieve the value of the description node of a data node (aspect or type).
        :param content_model: A data model of a content-model.
        :param typology: The type of the node (aspect or type).
        :param name: The name of the data node.
        :return: The value of the data node description node.
        """
        description: Element = self._get_root(content_model.path) \
            .find(".//{0}{1}s/{0}{1}[@name='{2}:{3}']/{0}description"
                  .format(self.get_namespace("xmlns"), typology, content_model.prefix, name))
        return None if description is None else description.text

    def __get_data_title(self, content_model: ContentModel, typology: str, name: str) -> Optional[str]:
        """
        Retrieve the value of the title node of a data node (aspect or type).
        :param content_model: A data model of a content-model.
        :param typology: The type of the node (aspect or type).
        :param name: The name of the data node.
        :return: The value of the data node title node.
        """
        title: Element = self._get_root(content_model.path) \
            .find(".//{0}{1}s/{0}{1}[@name='{2}:{3}']/{0}title"
                  .format(self.get_namespace("xmlns"), typology, content_model.prefix, name))
        return None if title is None else title.text

    def get_data_parent(self, content_model: ContentModel, typology: str, name: str) -> Optional[str]:
        """
        Retrieve the value of the parent node of a data node (aspect or type).
        :param content_model: A data model of a content-model.
        :param typology: The type of the node (aspect or type).
        :param name: The name of the data node.
        :return: The value of the data node title node.
        """
        parent: Element = self._get_root(content_model.path) \
            .find(".//{0}{1}s/{0}{1}[@name='{2}:{3}']/{0}parent"
                  .format(self.get_namespace("xmlns"), typology, content_model.prefix, name))
        return None if parent is None else parent.text

    def __get_data_mandatory_aspects(self, content_model: ContentModel, typology: str, name: str) -> list[str]:
        result: list[str] = []
        root: Element = self._get_root(content_model.path)
        mandatory_aspects: list[Element] = root.findall(".//{0}{1}s/{0}{1}[@name='{2}:{3}']/{0}mandatory-aspects"
                                                        "/{0}aspect".format(self.get_namespace("xmlns"), typology,
                                                                            content_model.prefix, name))
        for mandatory_aspect in mandatory_aspects:
            result.append(mandatory_aspect.text)

        return result

    def __get_properties_node_index(self, data_node: Element) -> int:
        namespace: str = self.get_namespace("xmlns")
        children: list[Element] = data_node.findall(".//{0}*".format(namespace))
        maximum: int = len(children)
        index: int = 0

        while index.__lt__(maximum) and children[index].tag.__ne__("{0}properties".format(namespace)):
            index += 1

        return index if index.__lt__(maximum) else (index - 1)

    def get_property(self, content_model: ContentModel, data: DataModel, property_name: str) \
            -> tuple[str, str, str, str, bool]:
        namespace: str = self.get_namespace("xmlns")
        root: Element = self._get_root(content_model.path)
        filename: str = FileFolderHelper.extract_filename_from_path(content_model.path)

        node: Element = root.find(".//{0}{1}s/{0}{1}[@name='{2}:{3}']/{0}properties/{0}property[@name='{2}:{4}']"
                                  .format(namespace, data.typology, content_model.prefix, data.name, property_name))
        if node is None:
            ApiException("There is no property named '{0}' in {1} '{2}' in content model '{3}' in file '{4}'."
                         .format(property_name, data.typology, data.name, content_model.complete_name, filename))

        title_node: Element = node.find("./{0}title".format(namespace))
        title: Optional[str] = None if title_node is None else title_node.text

        description_node: Element = node.find("./{0}description".format(namespace))
        description: Optional[str] = None if description_node is None else description_node.text

        type_node: Element = node.find("./{0}type".format(namespace))
        typology: Optional[str] = None
        if type_node is not None:
            if StringHelper.is_empty(type_node.text):
                raise ApiException("The type of the {0} property from the content-model '{1}' of file '{2}' is invalid."
                                   " It cannot be empty or None."
                                   .format(data.typology, content_model.complete_name,
                                           FileFolderHelper.extract_filename_from_path(content_model.path)))
            elif StringHelper.has_space(type_node.text):
                raise ApiException("The type of the {0} property from the content-model '{1}' of file '{2}' is invalid."
                                   " There can be no space in it."
                                   .format(data.typology, content_model.complete_name,
                                           FileFolderHelper.extract_filename_from_path(content_model.path)))
            try:
                typology = type_node.text.rsplit(":", 1)[1]
                if (typology.__ne__("text") and typology.__ne__("int") and typology.__ne__("long")
                        and typology.__ne__("float") and typology.__ne__("double") and typology.__ne__("date")
                        and typology.__ne__("datetime") and typology.__ne__("boolean") and typology.__ne__("encrypted")
                        and typology.__ne__("noderef")):
                    raise ApiException(
                        "The type of the {0} property from the content-model '{1}' of file '{2}' is invalid. Its value"
                        " must be: text, int, long, float, double, date, datetime, boolean, encrypted or noderef."
                        .format(data.typology, content_model.complete_name,
                                FileFolderHelper.extract_filename_from_path(content_model.path)))

            except IndexError:
                raise ApiException("The value of property type '{0}' of {1} '{2}' of content model '{3}' of file '{4}'"
                                   " is invalid. It should be formed like this: d:[type]"
                                   .format(property_name, data.typology, data.name, content_model.complete_name,
                                           filename))

        mandatory_node: Element = node.find("./{0}mandatory".format(namespace))
        mandatory: bool = False
        if mandatory_node is not None:

            if StringHelper.is_empty(mandatory_node.text):
                raise ApiException("The value of property 'mandatory' '{0}' of {1} {2} of content model {3} of file "
                                   "{4} is invalid. A value must be set ('true' or 'false').")

            elif mandatory_node.text.__eq__("true"):
                mandatory = True

            elif mandatory_node.text.__eq__("false"):
                mandatory = False
            else:
                raise ApiException("The value of property 'mandatory' '{0}' of {1} {2} of content model {3} of file "
                                   "{4} is invalid. The value can only be 'true' or 'false'."
                                   .format(property_name, data.typology, data.name, content_model.complete_name,
                                           filename))

        return property_name, title, description, typology, mandatory

    def get_properties(self, content_model: ContentModel) -> list[str]:
        result: list[str] = []
        filename: str = FileFolderHelper.extract_filename_from_path(content_model.path)
        root: Element = self._get_root(content_model.path)
        for prop in root.findall(".//{0}aspects/{0}aspect/{0}properties/{0}property".format(
                self.get_namespace("xmlns"))):
            result.append(self.__extract_property_name(prop, filename))
        for prop in root.findall(".//{0}types/{0}type/{0}properties/{0}property".format(
                self.get_namespace("xmlns"))):
            result.append(self.__extract_property_name(prop, filename))
        return result

    def get_data_property_names(self, content_model: ContentModel, data: DataModel) -> list[str]:
        """
        Retrieve a list of property names from a data model.
        :param content_model: A data model of a content-model.
        :param data:
        :return:
        """
        # Result initialization.
        result: list[str] = []
        # Retrieving model properties.
        namespace: str = self.get_namespace("xmlns")
        root: Element = self._get_root(content_model.path)
        filename: str = FileFolderHelper.extract_filename_from_path(content_model.path)
        properties: list[Element] = root.findall(".//{0}{1}s/{0}{1}[@name='{2}:{3}']/{0}properties/{0}property"
                                                 .format(namespace, data.typology, content_model.prefix, data.name))
        # Extract property names.
        for prop in properties:
            result.append(self.__extract_property_name(prop, filename))
        # Return of the result.
        return result


from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
                        print_function, unicode_literals)

import os
import subprocess
from contextlib import closing

from StringIO import StringIO

from twitter.common.collections import maybe_list

from pants.backend.core.tasks.task import Task

from pants.backend.core.tasks.console_task import ConsoleTask
from pants.base.cmd_line_spec_parser import CmdLineSpecParser
from pants.base.target import Target
from pants.goal.context import Context
from pants.goal.goal import Goal
from pants.option.bootstrap_options import register_bootstrap_options
from pants.option.options import Options
from pants_test.base_test import BaseTest
from pants_test.base.context_utils import create_config, create_run_tracker

def is_exe(name):
  result = subprocess.call(['which', name], stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)
  return result == 0

class TaskTest(BaseTest):
  """A baseclass useful for testing Tasks."""

  @classmethod
  def task_type(cls):
    """Subclasses must return the type of the ConsoleTask subclass under test."""
    raise NotImplementedError()

  def prepare_task(self,
                   config=None,
                   args=None,
                   targets=None,
                   build_graph=None,
                   build_file_parser=None,
                   address_mapper=None,
                   console_outstream=None,
                   workspace=None):
    """Prepares a Task for execution.

    task_type: The class of the Task to create.
    config: An optional string representing the contents of a pants.ini config.
    args: optional list of command line flags, these should be prefixed with '--test-'.
    targets: optional list of Target objects passed on the command line.

    Returns a new Task ready to execute.
    """

    task_type = self.task_type()
    assert issubclass(task_type, Task), 'task_type must be a Task subclass, got %s' % task_type

    config = create_config(config or '')
    workdir = os.path.join(config.getdefault('pants_workdir'), 'test', task_type.__name__)

    new_options = Options(env={}, config=config, known_scopes=['', 'test'], args=args or [])
    # A lot of basic code uses these options, so always register them.
    register_bootstrap_options(new_options.register_global)

    task_type.options_scope = 'test'
    task_type.register_options_on_scope(new_options)

    run_tracker = create_run_tracker()

    context = Context(config,
                      new_options,
                      run_tracker,
                      targets or [],
                      build_graph=build_graph,
                      build_file_parser=build_file_parser,
                      address_mapper=address_mapper,
                      console_outstream=console_outstream,
                      workspace=workspace)
    return task_type(context, workdir)

  def targets(self, spec):
    """Resolves a target spec to one or more Target objects.

    spec: Either BUILD target address or else a target glob using the siblings ':' or
          descendants '::' suffixes.

    Returns the set of all Targets found.
    """

    spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
    addresses = list(spec_parser.parse_addresses(spec))
    for address in addresses:
      self.build_graph.inject_address_closure(address)
    targets = [self.build_graph.get_target(address) for address in addresses]
    return targets

  def assertDeps(self, target, expected_deps=None):
    """Check that actual and expected dependencies of the given target match.

    :param target: :class:`pants.base.target.Target` to check
      dependencies of.
    :param expected_deps: :class:`pants.base.target.Target` or list of
      ``Target`` instances that are expected dependencies of ``target``.
    """
    expected_deps_list = maybe_list(expected_deps or [], expected_type=Target)
    self.assertEquals(set(expected_deps_list), set(target.dependencies))

class ConsoleTaskTest(TaskTest):
  """A baseclass useful for testing ConsoleTasks."""

  def setUp(self):
    Goal.clear()
    super(ConsoleTaskTest, self).setUp()

    task_type = self.task_type()
    assert issubclass(task_type, ConsoleTask), \
        'task_type() must return a ConsoleTask subclass, got %s' % task_type

  def execute_task(self, config=None, args=None, targets=None):
    """Creates a new task and executes it with the given config, command line args and targets.

    config:        an optional string representing the contents of a pants.ini config.
    args:          optional list of command line flags, these should be prefixed with '--test-'.
    targets:       optional list of Target objects passed on the command line.
    Returns the text output of the task.
    """
    with closing(StringIO()) as output:
      task = self.prepare_task(config=config,
                               args=args,
                               targets=targets,
                               build_graph=self.build_graph,
                               build_file_parser=self.build_file_parser,
                               address_mapper=self.address_mapper,
                               console_outstream=output)
      task.execute()
      return output.getvalue()

  def execute_console_task(self, config=None, args=None, targets=None, extra_targets=None,
                           workspace=None):
    """Creates a new task and executes it with the given config, command line args and targets.

    config:        an optional string representing the contents of a pants.ini config.
    args:          optional list of command line flags, these should be prefixed with '--test-'.
    targets:       optional list of Target objects passed on the command line.
    extra_targets: optional list of extra targets in the context in addition to those passed on the
                   command line.
    workspace:     optional Workspace to pass into the context.

    Returns the list of items returned from invoking the console task's console_output method.
    """
    task = self.prepare_task(config=config,
                             args=args,
                             targets=targets,
                             build_graph=self.build_graph,
                             build_file_parser=self.build_file_parser,
                             address_mapper=self.address_mapper,
                             workspace=workspace)
    return list(task.console_output(list(task.context.targets()) + list(extra_targets or ())))

  def assert_entries(self, sep, *output, **kwargs):
    """Verifies the expected output text is flushed by the console task under test.

    NB: order of entries is not tested, just presence.

    sep:      the expected output separator.
    *output:  the output entries expected between the separators
    **kwargs: additional kwargs passed to execute_task.
    """
    # We expect each output line to be suffixed with the separator, so for , and [1,2,3] we expect:
    # '1,2,3,' - splitting this by the separator we should get ['1', '2', '3', ''] - always an extra
    # empty string if the separator is properly always a suffix and not applied just between
    # entries.
    self.assertEqual(sorted(list(output) + ['']), sorted((self.execute_task(**kwargs)).split(sep)))

  def assert_console_output(self, *output, **kwargs):
    """Verifies the expected output entries are emitted by the console task under test.

    NB: order of entries is not tested, just presence.

    *output:  the expected output entries
    **kwargs: additional kwargs passed to execute_console_task.
    """
    self.assertEqual(sorted(output), sorted(self.execute_console_task(**kwargs)))

  def assert_console_output_ordered(self, *output, **kwargs):
    """Verifies the expected output entries are emitted by the console task under test.

    NB: order of entries is tested.

    *output:  the expected output entries in expected order
    **kwargs: additional kwargs passed to execute_console_task.
    """
    self.assertEqual(list(output), self.execute_console_task(**kwargs))

  def assert_console_raises(self, exception, **kwargs):
    """Verifies the expected exception is raised by the console task under test.

    **kwargs: additional kwargs are passed to execute_console_task.
    """
    with self.assertRaises(exception):
      self.execute_console_task(**kwargs)


from extra_streamlit_tools._logging import logging as logger
import streamlit as st
from typing import Any, Optional

def clear_cache(keep_keys: Optional[list[str]] = None) -> None:
    """
    Resets the Streamlit cache.

    Parameters
    ----------
        keep_keys:Optional[list[str]]
            Keys to not be cleared from cache
    """

    logger.debug("Clearing cache")
    for key in st.session_state.keys():
        if keep_keys is None or key not in keep_keys:
            logger.debug(f"Deleting key: {key}")
            del st.session_state[key]
        else:
            logger.debug(f"Keeping key: {key}")

def init_session_keys(key_value_pairs: dict[str, Any]) -> None:
    """
    The init_session_keys function is a helper function that initializes the session state with keys and values.

    Parameters
    ----------
        key_value_pairs:dict[str, Any]
            A dictionairy of key_value_pairs
    """  # noqa
    for key, value in key_value_pairs.items():
        if key not in st.session_state:
            st.session_state[key] = value

def change_in_session_state(key_value_pairs: dict[str, Any]):
    """
    The change_in_session_state function is a helper function that allows you to change session state values.

    Parameters
    ----------
        key_value_pairs:dict[str, Any]
            Dictionairy with the Streamlit session_state key and the its new value
    """  # noqa
    for key, value in key_value_pairs.items():
        st.session_state[key] = value

def set_selectbox_index(
    selectbox_key: str, session_state_var_name: str, values: list[Any]
) -> None:
    """
    The set_selectbox_index function is a helper function that sets the index of a selectbox to the value
    of another session state variable. This is useful when you want to set the default value of one selectbox
    to be equal to another, but you don't know what that other's default value will be until runtime.

    Parameters
    ----------
        selectbox_key:str
            Specify the key of the selectbox
        session_state_var_name:str
            Set the session state variable name
        values:list[Any]
            The list of values in the selectbox
    """  # noqa
    st.session_state[session_state_var_name] = values.index(
        st.session_state[selectbox_key]
    )


"""
Created on Thu Sep 8 14:37:34 2016

@author: Patrick Trainor
@course: Artificial Intelligence
@title: Project 2

Code for embedding of figure in tk credited to:
    http://matplotlib.org/examples/user_interfaces/embedding_in_tk.html

Code for labeling points on figure credited to "unknown" @
    http://stackoverflow.com/posts/5147430/revisions
"""

# Imports:
import time
import numpy as np
import re
import sklearn.metrics.pairwise as prw
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg
from matplotlib.backend_bases import key_press_handler
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
from itertools import permutations
import sys
import Tkinter as Tk

# Create Tk object:
root=Tk.Tk()
root.wm_title("BFS and DFS Search")

# Get filename from system args
filename = sys.argv[-1]
tsp_file=open(filename)

# Open and read the file lines
tsp_read=tsp_file.read().splitlines()
tsp_file.close()

# Find the number of cities
for line in tsp_read:
    if line.startswith('DIMENSION'):
        cities=int(re.findall(r'\d+', line)[0])

# Find the line of the file in which coordinates start
start_line=tsp_read.index('NODE_COORD_SECTION')+1

# Create matrix of pairwise distances
crds=[str.split(line) for line in tsp_read[start_line:(start_line+cities)]]
crds=np.matrix(crds).astype(np.float)[:,1:]
pdist=prw.pairwise_distances(crds)

#Add adjacency matrix:
adj=np.array([[0,1,1,1,0,0,0,0,0,0,0],[0,0,1,0,0,0,0,0,0,0,0],
               [0,0,0,1,1,0,0,0,0,0,0],[0,0,0,0,1,1,1,0,0,0,0],
               [0,0,0,0,0,0,1,1,0,0,0],[0,0,0,0,0,0,0,1,0,0,0],
               [0,0,0,0,0,0,0,0,1,1,0],[0,0,0,0,0,0,0,0,1,1,1],
               [0,0,0,0,0,0,0,0,0,0,1],[0,0,0,0,0,0,0,0,0,0,1]],dtype="bool")

#Breadth first algorithm:
def bfs(adj,start,goal):
    vertex=start
    goalAcheived=vertex==goal
    visit=[vertex]
    edges=[]
    while goalAcheived is False:
        neighbors=np.where(adj[vertex,:])[0].tolist() #Find neighbors
        for neighbor in neighbors:
            visit=visit+[neighbor] #visit neighbors
            edges=edges+[[vertex,neighbor]] #Edges to neighbor
            goalAcheived=neighbor==goal #Check is neighbor goal?
            if goalAcheived: #If neighbor is goal then stop
                break
        visit=[x for x in visit if x!=vertex] #Remove city from queue
        vertex=visit[0] #Choose next city in queue
    path=[edges.pop()] #Add edge to path
    while path[0][0]!=start: #Backtrace
        path=[[x for x in edges if x[1]==path[0][0]][0]]+path
    return path

#Depth first algorithm
def dfs(adj,start,goal):
    nextVertex=vertex=start
    goalAcheived=vertex==goal
    visit=[]
    edges=[]
    while goalAcheived is False:
        vertex=nextVertex
        neighbors=np.where(adj[vertex,:])[0].tolist() #Find neighbors
        if neighbors==[]: #Iff no more neighbors in stack go back
            while neighbors==[]:
                vertex=visit.pop()
                neighbors=np.where(adj[vertex,:])[0].tolist()
        visit=visit+neighbors #Add new neighbors to stack
        nextVertex=visit.pop() #Next city is last in stack
        edges=edges+[[vertex,nextVertex]] #add edges
        goalAcheived=edges[-1][-1]==goal #Check goal city?
    path=[edges.pop()] #Add to path
    while path[0][0]!=start: #Backtrace
        path=[[x for x in edges if x[1]==path[0][0]][0]]+path
    return path

def pathMap(path): #Function for mapping path to coordinates
    xCrds=[]
    yCrds=[]
    for i in range(len(path)):
        xCrds=xCrds+[[crds[path[i][0]][0,0],crds[path[i][1]][0,0]]]
        yCrds=yCrds+[[crds[path[i][0]][0,1],crds[path[i][1]][0,1]]]
    return([xCrds,yCrds])

#Execute BFS
t0=time.time()
bfsPath=bfs(adj,0,10)
t1=time.time()
bfsTime=t1-t0
bfsCrds=pathMap(bfsPath)

#Execute DFS
t0=time.time()
dfsPath=dfs(adj,0,10)
t1=time.time()
dfsTime=t1-t0
dfsCrds=pathMap(dfsPath)

#Determine the cities and edges of the whole
xCrds=[]
yCrds=[]
for i in range(10):
    for j in range(11):
        if adj[i,j]:
            xCrds=xCrds+np.squeeze(crds[:,0][[i,j]]).tolist()
            yCrds=yCrds+np.squeeze(crds[:,1][[i,j]]).tolist()

#Function for plotting cities, edges, and paths:
def plotFun(xCrds,yCrds,pathCrds=[],plotPath=False):
    fig=plt.figure(figsize=(5, 4), dpi=100)
    f1=fig.add_subplot(111)
    f1.plot(crds[:,0],crds[:,1],'ro')
    f2=fig.add_subplot(111)
    for i in range(len(xCrds)):
        f2.plot(xCrds[i],yCrds[i],'--',color='c')
    if plotPath:
        f3=fig.add_subplot(111)
        for i in range(len(pathCrds[0])):
            f3.plot(pathCrds[0][i],pathCrds[1][i],'-',color='r')
    f4=fig.add_subplot(111)
    labs=map(str,range(11))
    for label, x, y in zip(labs,crds[:,0],crds[:,1]):
        f4.annotate(label, xy = (x, y),xytext = (-10, 10),
            textcoords = 'offset points', ha = 'right', va = 'bottom',
            bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
            arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))
    return fig

f=plotFun(xCrds,yCrds,bfsCrds,plotPath=True)
f2=plotFun(xCrds,yCrds,dfsCrds,plotPath=True)

#Add listbox with results to tk window
listbox = Tk.Listbox(root)
listbox.pack(side=Tk.TOP, fill=Tk.X)
listbox.insert("end","BFS Path: "+str(bfsPath))
listbox.insert("end","DFS Path: " +str(dfsPath))

#Add figure to tk window
canvas = FigureCanvasTkAgg(f, master=root)
canvas.show()
canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)
canvas = FigureCanvasTkAgg(f2, master=root)
canvas.show()
canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)

#Add toolbar to tk window
toolbar = NavigationToolbar2TkAgg(canvas, root)
toolbar.update()
canvas._tkcanvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)

#click event handler
def on_key_event(event):
    print('you pressed %s' % event.key)
    key_press_handler(event, canvas, toolbar)

canvas.mpl_connect('key_press_event', on_key_event)

#Quit event handler
def _quit():
    root.quit()
    root.destroy()
button = Tk.Button(master=root, text='Quit', command=_quit)
button.pack(side=Tk.BOTTOM)

#Execute tk main loop
Tk.mainloop()

#Write results to file
with open(filename.split('.')[0]+'Solution.txt','a') as tf:
    tf.write("Input file: "+filename)
    tf.write("\n")
    tf.write("BFS Path: " +str(bfsPath))
    tf.write("\n")
    tf.write("DFS Path: "+str(dfsPath))
    tf.write("\n")
    tf.write("BFS time: "+str(bfsTime)+" DFS time: "+str(dfsTime))
    tf.close()


# NOTE: mini function for testing your UDP connection w the computer running the server and MAX
from pythonosc import udp_client

PORT_TO_MAX = 5002
IP = "192.168.2.2"

global client
client = udp_client.SimpleUDPClient(IP, PORT_TO_MAX)
input("hello")
while True:
    print("sent")
    client.send_message("/point", 1)
    input("pause")


import logging

from handlers.detectors import detect_is_admin
from keyboards.default.start import start_admin
from keyboards.inline.admin.success_payment import withdraw_money_balance
from loader import dp, bot, db
from data.config import ADMINS
from keyboards.default.back import back

from states.balance import Balance

from aiogram import types
from aiogram.dispatcher import FSMContext

@dp.callback_query_handler(text="withdraw_money", state=Balance.menu)
async def withdraaw_money_from_balance(call: types.CallbackQuery, state: FSMContext):
    user_id = call.from_user.id

    select_user = await db.select_user_data(user_id)
    balance = select_user[0][1]

    if balance >= 10000:
        text = "Pul yechib olish uchun karta raqami kiriting...\n\n" \
               "Mavjud to'lov turlari\n\n▪Humo"

        await call.message.delete()
        await call.message.answer(text=text, reply_markup=back)

        await Balance.withdraw.set()

    else:
        text = "⚠ Pul chiqarish uchun hisobingizda kamida 10.000 so'm bo'lishi shart!"
        await call.answer(text, show_alert=True)

@dp.message_handler(state=Balance.withdraw, content_types=types.ContentType.TEXT)
async def identify_card(message: types.Message, state: FSMContext):
    msg = message.text

    await message.answer(
        text="Yaxshi, endi nech pul chiqarib olmoqchi ekanligingizni yozing\n\nMasalan: 10000",
        reply_markup=back
    )

    await state.update_data(
        {'card_number': msg}
    )

    await Balance.money.set()

@dp.message_handler(state=Balance.money, content_types=types.ContentType.TEXT)
async def identify_how_much_money(message: types.Message, state: FSMContext):
    data = await state.get_data()
    card_number = data.get('card_number')

    user_id = message.from_user.id
    full_name = message.from_user.full_name
    user_mention = message.from_user.get_mention(name=full_name)

    msg = message.text

    select_user = await db.select_user_data(user_id)
    balance = select_user[0][2]

    try:
        summa = int(msg)
        if summa >= 10000:
            if balance >= summa:
                await bot.send_message(
                    chat_id=message.chat.id,
                    text="✅ Pul yechish uchun so'rovingiz qabul qilindi, admin tez orada to'lovni amalga oshiradi",
                    reply_markup=await detect_is_admin(user_id)
                )

                admin_text = f"👤 {user_mention}\n💳 {card_number}\n💸 {summa} so'm\n💰 {balance}"

                await bot.send_message(
                    chat_id=ADMINS[0],
                    text=admin_text,
                    reply_markup=withdraw_money_balance(user_id, summa)
                )

                await state.reset_state(with_data=False)

            elif balance < summa:
                await bot.send_message(
                    chat_id=message.chat.id,
                    text=f"Hisobingizda {balance} so'm pul va siz {summa} so'm "
                         f"chiqarishga harakat qilyapsiz, iltimos boshqattan urinib ko'ring\n\n"
                         f"Masalan: 10000",
                    reply_markup=back
                )

        else:
            text = "⚠ Eng kamida 10000 so'm chiqarib olish mumkin, summa kiriting\n\nMasalan: 10000"
            await message.answer(text, reply_markup=back)

    except ValueError as VE:
        logging.info(VE)
        await message.answer(
            text='Iltimos, faqat raqamlardan foydalaning\n\nMasalan: 10000',
            reply_markup=back
        )

@dp.callback_query_handler(text_contains="tolandi_", state='*')
async def final_withdraw(call: types.CallbackQuery, state: FSMContext):
    data = call.data
    splited = data.split('_')

    user_id = splited[1]
    summa = splited[2]

    select_user_data = await db.select_user_data(int(user_id))
    balance = select_user_data[0][2]

    get_user = await bot.get_chat(user_id)
    full_name = get_user.full_name
    user_mention = call.from_user.get_mention(full_name)

    end_balance = balance - int(summa)
    await db.update_user_balancee(end_balance, int(user_id))

    await call.message.delete()

    await bot.send_message(
        chat_id=call.message.chat.id,
        text="Foydalanuvchi puli to'lab berildi",
        reply_markup=start_admin
    )

    text = f"✅ Pul toʻlandi: {user_mention}\n" \
           f"👤 Foydalanuvchi ID: {user_id}\n" \
           f"💸 Miqdor: {summa} soʻm"

    await bot.send_message(
        chat_id=-1001943689507,
        text=text
    )

    await bot.send_message(
        chat_id=user_id,
        text=f"Sizning pul yechish so'rovingiz qabul qilindi va {summa} so'm to'lab berildi",
        reply_markup=await detect_is_admin(user_id)
    )

    await state.finish()

@dp.callback_query_handler(text_contains="bekor_qilish_", state='*')
async def final_withdraw(call: types.CallbackQuery, state: FSMContext):
    data = call.data
    splited = data.split('_')

    user_id = splited[2]
    summa = splited[3]

    await call.message.delete()

    await state.update_data(
        {'final_id': user_id, 'final_summa': summa}
    )

    await bot.send_message(
        chat_id=call.message.chat.id,
        text="To'lov bekor bo'lishining sababini kiriting",
        reply_markup=back
    )

    await Balance.cancel_payment.set()

@dp.message_handler(state=Balance.cancel_payment, content_types=types.ContentType.TEXT)
async def cancel_payment_user(message: types.Message, state: FSMContext):
    data = await state.get_data()
    user_id = data.get('final_id')
    final_summa = data.get('final_summa')

    msg = message.text

    await bot.send_message(
        chat_id=user_id,
        text=f"Sizning {final_summa} so'm chiqarishdagi harakatingiz bekor qilindi\n\nSabab: {msg}"
    )

    await bot.send_message(
        chat_id=message.chat.id,
        text="Xabar foydalanuvchiga yuborildi va to'lov harakati bekor qilindi",
        reply_markup=start_admin
    )

    await state.finish()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 20 15:50:56 2018

@author: vitorhadad
"""
#
import numpy as np
import networkx as nx
import os

from tqdm import trange
from matching.solver.kidney_solver2 import optimal, greedy, get_two_cycles
from matching.utils.data_utils import clock_seed, evaluate_policy, get_n_matched, get_cycle_probabilities
from matching.utils.env_utils import snapshot, two_cycles
from matching.policy_function.policy_function_lstm import RNN
from matching.policy_function.policy_function_mlp import MLPNet
from matching.policy_function.policy_function_gcn import GCNet
from matching.policy_function.policy_function_rgcn import RGCNet
from matching.policy_function.policy_function_attention import AttentionRNN
from matching.environment.abo_environment import ABOKidneyExchange

#%%
def run_node2vec(G,
                 path = "",
                 input = "edges.txt",
                 output = "emb.txt",
                 d = 10):

    nx.write_edgelist(G, path + input, data = False)

    cmd = "./node2vec -i:{}{} -o:{}{} -d:{} -dr"\
            .format(path, input, path, output, d)

    os.system(cmd)

    with open("emb.txt", "r") as emb:
        lines = emb.readlines()
        n = len(lines) - 1
        features = np.zeros(shape = (n, d))
        for k, line in enumerate(lines[1:]):
            _, *xs = line.split(" ")
            try:
                features[k] = [float(x) for x in xs]
            except:
                import pdb; pdb.set_trace()
    return features

def get_features(env):
    opt= optimal(env)

    features = []
    labels = []

    for t in range(env.time_length):
        liv = np.array(env.get_living(t))
        A = env.A(t)
        has_cycle = np.diag(A @ A) > 0
        liv = liv[has_cycle]

        m = opt["matched"][t]

        Y = np.zeros(len(liv))
        Y[np.isin(liv, list(m))] = 1
        labels.append(Y)

        if len(liv) > 0:
            X = env.X(t)[has_cycle]
            subg = env.subgraph(liv)
            E = run_node2vec(subg)
            features.append(np.hstack([X, E]))

        env.removed_container[t].update()

    return np.vstack(features), np.hstack(labels)

env = ABOKidneyExchange(entry_rate=5, death_rate=.1, time_length=10, seed=clock_seed())

X, Y = get_features(env)
np.save("X.npy", X)
np.save("Y.npy", Y)

#%%

#%%
#

#

#%%
#
#env = ABOKidneyExchange(entry_rate = 5,
#                        death_rate = .1,
#                        time_length = 1500,
#                        seed = clock_seed())
#
#opt = optimal(env)
#gre = greedy(env)
#
##%%
#def evaluate(algo, env, thres):
#
#    env.removed_container.clear()
#    rewards = []
#    for t in trange(env.time_length):
#
#        liv = np.array(env.get_living(t))
#        A = env.A(t)
#        has_cycle = np.diag(A @ A) > 0
#        liv_and_cycle = liv[has_cycle]
#        yhat_full = np.zeros(len(liv), dtype=bool)
#
#        if len(liv_and_cycle) == 0:
#            continue
#
#        X = env.X(t)[has_cycle]
#        subg = env.subgraph(liv_and_cycle)
#        E = run_node2vec(subg)
#        F = np.hstack([X, E])
#
#        yhat = algo.predict_proba(F)[:,1] > thres
#        yhat_full[has_cycle] = yhat
#        potential = liv[yhat_full]
#
#        removed = optimal(env, t, t, subset=potential)["matched"][t]
#        env.removed_container[t].update(removed)
#        rewards.append(len(removed))
#
#    return rewards
#
#
#r = evaluate(pipe, env, .05)
#
#gre_n = get_n_matched(gre["matched"], 0, env.time_length)
#opt_n = get_n_matched(opt["matched"], 0, env.time_length)

#print("\nrewards\n",
#      np.sum(r[500:]),
#      np.sum(gre_n[500:]),
#      np.sum(opt_n[500:]))
#


# This file is part of ZNC-Signal <https://github.com/poppyschmo/znc-signal>,
# licensed under Apache 2.0 <http://www.apache.org/licenses/LICENSE-2.0>.

import pytest
from copy import deepcopy
from collections import namedtuple
from conftest import signal_stub, signal_stub_debug, all_in
signal_stub = signal_stub  # quiet linter
signal_stub_debug = signal_stub_debug

@pytest.fixture
def env_stub(signal_stub):
    import os
    os.environ["SIGNALMOD_FAKE"] = "fake_val"
    os.environ["SIGNALMOD_FOO"] = "foo_val"
    argstring = f"DATADIR={os.devnull} FOO=someval UNKNOWN=ignored"
    signal_stub.__class__.foo = None
    signal_stub.__class__.fake = None
    env_stub = signal_stub.__class__(argstring)
    yield env_stub
    del signal_stub.__class__.foo
    del signal_stub.__class__.fake
    del os.environ["SIGNALMOD_FAKE"]
    del os.environ["SIGNALMOD_FOO"]
    if env_stub._buffer is not None:
        env_stub._buffer.close()

def test_OnLoad(env_stub):
    import os
    # Process environment is not modified
    assert all_in(os.environ, "SIGNALMOD_FAKE", "SIGNALMOD_FOO")
    assert env_stub.fake == os.environ["SIGNALMOD_FAKE"] == "fake_val"
    assert os.environ["SIGNALMOD_FOO"] == "foo_val"
    # OnLoad args override environment variables
    assert env_stub.foo == "someval"
    # Unknown attributes are ignored
    assert not hasattr(env_stub, "unknown")
    assert hasattr(env_stub, "datadir") and env_stub.datadir == os.devnull
    # TODO use pseudo terminal to test debug logger (likely requires Linux)

# NOTE: order doesn't (currently) matter for flattened/narrowed hook data
base_rel = {'body': 'Welcome dummy!',
            'network': 'testnet',
            'away': False,
            'client_count': 1,
            'nick': 'tbo',
            'ident': 'testbot',
            'host': 'znc.in',
            'hostmask': 'tbo!testbot@znc.in',
            # Real thing uses datetime object
            'time': '2018-04-21T01:20:13.751970+00:00'}

rels = namedtuple("Rels",
                  "OnChanTextMessage OnPrivTextMessage")(
    dict(base_rel,
         channel='#test_chan',
         detached=False,
         context='#test_chan'),
    dict(base_rel,
         body="Oi",
         context="dummy")
)

# NOTE OnPrivActionMessage(msg) and OnChanActionMessage(msg) are exactly the
# same as their "Text" counterparts above, once narrowed. Normalized inspector
# output will show: 'type': 'Action', 'params': ('dummy', '\x01ACTION Oi\x01')
# as the only real differences.

def test_reckon(signal_stub_debug):
    # Simulate converted dict passed to Signal.reckon()
    from Signal.reckognize import reckon
    from collections import defaultdict
    defnul = defaultdict(type(None), rels.OnPrivTextMessage)
    assert defnul["channel"] is None
    assert defnul["detached"] is None
    #
    # Load default config
    sig = signal_stub_debug
    sig.manage_config("load")  # same as '[*Signal] select'
    # Quiet "no host" warning
    sig.OnModCommand('update /settings/host localhost')
    # Simulate a single, simple hook case
    rel = rels.OnChanTextMessage
    #
    data_bak = deepcopy(rel)
    conds = sig.config.conditions
    from collections import defaultdict
    data = defaultdict(type(None), rel)
    #
    sig._read()  # clear read buffer
    # Step through default config to ensure test module stays current with
    # future changes to options
    current_defaults = iter(sig.config.conditions["default"])
    #
    assert reckon(sig.config, data, sig.debug) is False
    data_reck = data["reckoning"]
    assert data_reck == ["<default", "!body>"]
    #
    assert next(current_defaults) == "enabled"
    sig.cmd_update("/conditions/default/enabled", "False")
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<default", "enabled>"]
    sig.cmd_update("/conditions/default/enabled", remove=True)
    #
    assert next(current_defaults) == "away_only"
    sig.cmd_update("/conditions/default/away_only", "True")
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<default", "away_only>"]
    sig.cmd_update("/conditions/default/away_only", remove=True)
    #
    assert next(current_defaults) == "scope"
    assert not conds["default"].maps[0]  # updated list is auto initialized
    sig.cmd_update("/conditions/default/scope/attached", remove=True)
    assert conds["default"]["scope"] == ["query", "detached"]
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<default", "scope>"]
    sig.cmd_update("/conditions/default/scope", remove=True)
    #
    # TODO replied_only
    assert next(current_defaults) == "replied_only"
    #
    assert next(current_defaults) == "max_clients"
    data["client_count"] = 2
    sig.cmd_update("/conditions/default/max_clients", "1")
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<default", "max_clients>"]
    sig.cmd_update("/conditions/default/max_clients", remove=True)
    data["client_count"] = 1
    _data = dict(data)
    _data.pop("reckoning")
    # _data.pop("template")
    assert data_bak == _data
    #
    assert sig._read().splitlines() == [
        "Selected: /conditions/default/enabled => False", "Item deleted.",
        "Selected: /conditions/default/enabled => True",
        #
        "Selected: /conditions/default/away_only => True", "Item deleted.",
        "Selected: /conditions/default/away_only => False",
        #
        "Item deleted; current selection has changed",
        "/conditions/default/scope => ['query', 'detached']",
        "Item deleted.",
        "Selected: /conditions/default/scope =>",
        "  ['query', 'detached', ...]",
        #
        "Selected: /conditions/default/max_clients => 1", "Item deleted.",
        "Selected: /conditions/default/max_clients => 0"
    ]
    #
    # TODO mock datetime.now() for time-based conditions
    assert next(current_defaults) == "timeout_post"
    assert next(current_defaults) == "timeout_push"
    assert next(current_defaults) == "timeout_idle"
    #
    # NOTE the rest aren't tested in order, just popped as encountered
    current_defaults = list(current_defaults)
    #
    sig.OnModCommand('update /expressions/custom @@ {"has": "dummy"}')
    sig.OnModCommand('update /templates/standard @@ '
                     '{"recipients": ["+12127365000"]}')
    # Create non-default condition
    current_defaults.remove("template")
    sig.OnModCommand('update /conditions/custom @@ {"template": "custom"}')
    # The "default" condition always runs last
    assert list(sig.manage_config("view")["conditions"]) == ["custom",
                                                             "default"]
    #
    # Network
    current_defaults.remove("network")
    sig.cmd_update("/conditions/custom/network", "$custom")
    assert data["network"] == "testnet"
    assert sig.config.expressions["custom"] == {"has": "dummy"}
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!network>", "<default", "!body>"]
    sig.cmd_update("/conditions/custom/network", "$pass")
    #
    # Channel
    current_defaults.remove("channel")
    sig.OnModCommand('update /conditions/custom/channel $custom')
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!channel>", "<default", "!body>"]
    sig.cmd_update("/expressions/custom/has", "test_chan")
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!body>", "<default", "!body>"]
    sig.cmd_update("/expressions/custom/has", "dummy")
    sig.cmd_update("/conditions/custom/channel", "$pass")
    #
    # Source
    current_defaults.remove("source")
    sig.OnModCommand('update /conditions/custom/source $custom')
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!source>", "<default", "!body>"]
    sig.OnModCommand('update /expressions/custom @@ {"wild": "*testbot*"}')
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!body>", "<default", "!body>"]
    current_defaults.remove("x_source")
    assert conds["custom"]["x_source"] == "hostmask"
    sig.cmd_update("/conditions/custom/x_source", "nick")
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!source>", "<default", "!body>"]
    sig.OnModCommand('update /expressions/custom @@ {"eq": "tbo"}')
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!body>", "<default", "!body>"]
    sig.cmd_update("/conditions/custom/x_source", remove=True)
    sig.cmd_update("/conditions/custom/source", "$pass")
    #
    # Body
    current_defaults.remove("body")
    sig.cmd_update("/conditions/custom/body", "$custom")
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!body>", "<default", "!body>"]
    sig.OnModCommand('update /expressions/custom @@ {"has": "dummy"}')
    assert reckon(sig.config, data, sig.debug) is True
    #
    # Body empty
    data["body"] = ""
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!body>", "<default", "!body>"]
    sig.cmd_update("/conditions/custom/body", "$drop")
    data["body"] = "Welcome dummy!"
    #
    # Inline expression
    sig.OnModCommand(
        'update /conditions/custom/body @@ {"any": []}'
    )  # same as !has ""
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "!body>", "<default", "!body>"]
    sig.OnModCommand(
        'update /conditions/custom/body @@ {"i has": "welcome"}'
    )
    assert reckon(sig.config, data, sig.debug) is True
    assert data_reck == ["<custom", "&>"]
    sig.cmd_update("/conditions/custom/body", "$drop")
    #
    # Make pass the same as drop
    sig.OnModCommand('update /expressions/pass @@ {"!has": ""}')
    assert conds["custom"]["body"] == "$drop"
    sig.cmd_update("/conditions/custom/body", "$pass")
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", '!network>', "<default", '!network>']
    #
    # Use literal expression in condition
    assert conds["custom"]["body"] == "$pass"
    sig.cmd_update("/expressions/pass", remove=True)
    assert sig.config.expressions["pass"] == {"has": ""}
    sig.OnModCommand('update /conditions/custom/body @@ {"!has": ""}')
    assert conds["custom"]["body"] == {"!has": ""}
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", '!body>', "<default", '!body>']
    sig.cmd_update("/conditions/custom/body", remove=True)
    sig.OnModCommand('update /expressions/pass @@ {"any": []}')
    #
    # Change per-condition, collective expressions bias
    current_defaults.remove("x_policy")  # only governs expressions portion
    assert conds["custom"]["x_policy"] == "filter"
    sig.OnModCommand('update /conditions/default/x_policy first')
    assert reckon(sig.config, data, sig.debug) is False
    assert data_reck == ["<custom", "|>", "<default", "|>"]  # Falls through
    #
    assert not current_defaults
    #
    # "FIRST" (short circuit) hit
    sig.OnModCommand('update /conditions/custom/body $custom')
    assert reckon(sig.config, data, sig.debug) is True
    assert data_reck == ["<custom", "body!>"]
    #
    sig.OnModCommand('update /conditions/onetime @@ {}')
    from textwrap import dedent
    sig.cmd_select("../")
    # Clear module buffer (lots of '/foo =>' output so far)
    assert "Error" not in sig._read()
    #
    # Add another condition that runs ahead of 'custom'
    sig.OnModCommand('select')
    assert sig._read().strip() == dedent("""
        /conditions => {'custom': {...}, 'onetime': {}, 'default': {...}}
    """).strip()
    sig.cmd_update("/conditions/onetime", "custom", arrange=True)
    assert sig._read().strip() == dedent("""
        Selected: /conditions =>
          {'onetime': {...}, 'custom': {...}, ...}
    """).strip()
    assert reckon(sig.config, data, sig.debug) is True
    assert data_reck == ["<onetime", "|>", "<custom", "body!>"]

def scope_conditional_stub(data, scope):
    cond = dict(scope=scope)
    # This is was copied verbatim from Signal.reckon, but code creep is
    # inevitable
    #
    channel = data["channel"]
    detached = data["detached"]
    if ((channel and (("detached" not in cond["scope"] and detached) or
                      ("attached" not in cond["scope"] and not detached))
         or (not channel and "query" not in cond["scope"]))):
        return True
    return False

# XXX this used to be justified when the option was "ignored_scopes" (more
# flags, not so trivial); should just merge with test_reckon or delete
def test_reject_scope():
    f = scope_conditional_stub
    c = "channel"
    a = "attached"
    d = "detached"
    q = "query"
    #
    data = {c: True, d: False}
    assert all((f(data, []),
                f(data, [q]),
                f(data, [d]),
                f(data, [d, q]))) is True
    assert not any((f(data, [a]),
                    f(data, [q, a]),
                    f(data, [q, a, d]))) is True
    #
    data = {c: True, d: True}
    assert all((f(data, []),
                f(data, [q]),
                f(data, [a]),
                f(data, [a, q]))) is True
    assert not any((f(data, [d]),
                    f(data, [d, a]),
                    f(data, [d, q, a]))) is True
    #
    data = {c: None, d: None}
    assert f(data, [q]) is False  # pass (not rejected)
    assert all((f(data, []),
                f(data, [d]),
                f(data, [a]),
                f(data, [d, a]))) is True


import logging
import json
from lxml import etree

def get_field_text(tree, path):
    nsmap = {"n1": tree.getroot().nsmap['n1']}
    node = tree.xpath(path, namespaces=nsmap)
    if len(node) > 0:
        return node[0].text
    return ''

def parse_metadata(scene, xml_filename, json_filename):
    logger = logging.getLogger(scene)
    logger.info("Parsing XML metadata from {0}".format(xml_filename))

    result = {'!scene': scene}

    tree = etree.parse(xml_filename)
    with open(json_filename, 'r') as myfile:
        tile_json = myfile.read()
    tile = json.loads(tile_json)

    scene_time = get_field_text(tree, "n1:General_Info/SENSING_TIME")
    result['acquired_date'] = scene_time.split('T')[0]
    result['acquired_time'] = scene_time.split('T')[1]

    coords = tile['tileGeometry']['coordinates'][0]
    result["#scene_corner_UL_x"] = coords[0][0]
    result["#scene_corner_UL_y"] = coords[0][1]
    result["#scene_corner_UR_x"] = coords[1][0]
    result["#scene_corner_UR_y"] = coords[1][1]
    result["#scene_corner_LR_x"] = coords[2][0]
    result["#scene_corner_LR_y"] = coords[2][1]
    result["#scene_corner_LL_x"] = coords[3][0]
    result["#scene_corner_LL_y"] = coords[3][1]

    result["#utm_zone"] = tile["utmZone"]

    return result


#! python3
# sendDuesReminders.py - sends emails based on payment status in spreadsheet

import smtplib
import openpyxl

wb = openpyxl.load_workbook('C:\\Users\\daize\\Desktop\\pythontest\\Automate\\duesRecords.xlsx')
sheet = wb.get_sheet_by_name('Sheet1')

lastCol = sheet.max_column
latestMonth = sheet.cell(row=1, column=lastCol).value

unpaidMembers = {}
for r in range(2, sheet.max_row+1):
    payment = sheet.cell(row=r, column=lastCol).value
    if payment != 'paid':
        name = sheet.cell(row=r, column=1).value
        email = sheet.cell(row=r, column=2).value
        unpaidMembers[name] = email

smtpObj = smtplib.SMTP('smtp.gmail.com', 587)
smtpObj.ehlo()
smtpObj.starttls()      #TLS加密的连接需要此代码进行加密，SSL不需要
smtpObj.login('account', 'passwd')

for name, email in unpaidMembers:
    body = "Suject: %s dues unpaid.\nDear %s...not paid dues for %s....Thanks" % (latestMonth, name, latestMonth)
    print('Sending email to %s' % email)
    sendmailStatus = smtpObj.sendmail('my account', email, body)        #所有收件人都发送成功，会返回空

    if sendmailStatus != {}:
        print('problem send email to %s:%s' % (email, sendmailStatus))
smtpObj.quit()


import asyncio
import websockets

HOST  = '0.0.0.0'
WS_PORT = 3333
TCP_PORT = 8888

loop = asyncio.get_event_loop()

websocket_connections = []
tcp_connections = []

async def send_status(status: str):
	data = status.encode()
	for c in tcp_connections:
		try:
			writer = c[1]
			writer.write(data)
			await writer.drain()
		except Exception as e:
			print(e)

async def executor(command: str):
	command = command.lower()
	words = command.split(" ")
	print(words)
	if 'вправо' in words:
		await send_status('d')
	if 'право' in words:
		await send_status('d')
	if 'права' in words:
			await send_status('d')
	if 'cправа' in words:
			await send_status('d')

	if 'влево' in words:
		await send_status('a')
	if 'лево' in words:
		await send_status('a')
	if 'лего' in words:
		await send_status('a')
	if 'лева' in words:
		await send_status('a')

	if 'назад' in words:
		await send_status('s')

	if 'вперед' in words:
		await send_status('w')
	if 'перед' in words:
		await send_status('w')

async def websock_handler(websocket, path):
	print('WS connect')
	global websocket_connections
	websocket_connections.append(websocket)
	try:
		while True:
			msg = await websocket.recv()
			print('[MSG INCOMING]', msg)
			await executor(msg)
	except websockets.exceptions.ConnectionClosedOK as e:
		pass
	websocket_connections.remove(websocket)
	print('WS disc')

async def tcp_handler(reader, writer):
	print('connected to ue')
	global tcp_connections
	connection = (reader, writer)
	tcp_connections.append(connection)
	writer.write("ping".encode())
	while True:
		data = await reader.read(100)
		if len(data) == 0:
			break
		await writer.drain()
	writer.close()
	tcp_connections.remove(connection)
	print('disconnected UE')

async def run_ws():
	await websockets.serve(websock_handler, HOST, WS_PORT)

async def run_tcp():
	await asyncio.start_server(tcp_handler, HOST, TCP_PORT, loop=loop)

def main():
	loop.create_task(run_ws())
	loop.create_task(run_tcp())
	try:
		loop.run_forever()
	except KeyboardInterrupt:
		print("stoped")

if __name__ == '__main__':
	main()


from __future__ import print_function
#
# -*- coding: utf-8 -*-#
# eso.org
# Copyright 2011 ESO
# Authors:
#   Lars Holm Nielsen <lnielsen@eso.org>
#   Dirk Neumayer <dirk.neumayer@gmail.com>
#
#
# Mantis 12175: Fix release dates for images and videos
#
#
# Find wrong release_dates:
# 1) Check id with release_date - e.g. opo0214a with release date in 2010 must be wrong
#    should be opoYYNNx NN is a cont. number?
# 2) Release dates with 2011-03-03 18:00-18:44 are wrong
#
# Don't bother about images connected to announcements, releases and potws.
#
# For images with long_caption_link or press_release_link follow the link and extract the date.
#
#
#*************************************************************************************************************

from future import standard_library
standard_library.install_aliases()
from djangoplicity.utils import optionparser
from djangoplicity.media.models import Image
from djangoplicity.media.models import Video

import re
import urllib.request, urllib.error, urllib.parse

import logging, sys
import socket

from datetime import datetime
import pytz

import hubblesite

def change_datetime(obj):
    '''
    follows the long_caption_link or the press_release_link to get the correct date
    '''
    # get link to image or press release
    link = None
    success = False
    if obj.long_caption_link.find('http') > -1:
        link = obj.long_caption_link
    elif obj.press_release_link.find('http') > -1:
        link = obj.press_release_link

    # follow link and get new date
    if link:
        release_date = hubblesite.get_release_date(link)
        if release_date:
            try:
                #print '-------------------------------------------------------'
                #print obj.id, obj.release_date.strftime('%Y-%B-%d %I:%M %p %Z')
                release_date = release_date.astimezone( pytz.timezone( 'Europe/Berlin' ) )
                release_date = datetime.replace(release_date, tzinfo=None)
                obj.release_date = release_date
                #print obj.id, obj.release_date.strftime('%Y-%B-%d %I:%M %p %Z')
                obj.save()
                success = True
            except:
                print(obj.id,' save failed!')
                pass
    return success

def process_objects(objs):
    '''
    find the objects that need a correction of the release_date
    '''
    pat = re.compile('[a-zA-Z]+([0-9]{2})\S+')

    count = 0
    finddate1 = datetime.strptime('2011-03-03 18:00:00','%Y-%m-%d %H:%M:%S')
    finddate2 = datetime.strptime('2011-03-03 19:00:00','%Y-%m-%d %H:%M:%S')
    for obj in objs:

        YY = None
        dt = obj.release_date
        if (dt):
            # process all objects with 2011-03-03 18:00:00 - 19:00:00
            if dt >= finddate1 and dt <= finddate2:
                if change_datetime(obj): count = count + 1
                print(obj.id, 'old: ', dt, '\t new: ', obj.release_date ,'\t\t reason: 20110303')
            # process all objects where opoYY YY does not match the year of the release_date
            else:
                #only care about opo... and heic...
                if obj.id.find('opo') == -1 and obj.id.find('heic') == -1: continue
                YY = pat.findall(obj.id)
                if len(YY) > 0:
                    YY = YY[0]
                    #print obj.id, YY, dt.strftime('%y'), dt
                    if YY != dt.strftime('%y'):
                        if change_datetime(obj): count = count + 1
                        print(obj.id, 'old: ', dt, '\t new: ', obj.release_date ,'\t\t reason: ', YY,' != ', dt.strftime('%y'))
        else:
            pass
            #print obj.id, ' no release_date'
    return count

if __name__ == '__main__':
    logger = logging.getLogger('app.' + __name__)
    logger.setLevel(logging.DEBUG)
    logger.addHandler(logging.StreamHandler(sys.stderr))
    logger.propagate = False
    logger.info("Fix release dates for images and videos")

    # timeout in seconds
    timeout = 60
    socket.setdefaulttimeout(timeout)

    test =  '''
News Release Number: STScI-2006-25
'''
    pattern = re.compile('''h2 class="release-number".*?:.*?>\s*(.*?)<.*?h2''')

    print('videos')
    print(process_objects(Video.objects.all()), ' videos have a new release_date')
    print('images')
    print(process_objects(Image.objects.all()), ' images have a new release_date')


import json
from datetime import date,timedelta
path = '/Users/evcu/GitHub/evcu.github.io//assets/nyc365blog/data.json'

data = {}
newel = {}
newel[u'date'] = str(date.today()-timedelta(days=365))
newel[u'mood'] = str(input("Enter mood -1/0/1:\n"))
print(newel)
newel[u'high'] = str(input("Highlights\n"))
newel[u'low'] = str(input("Lowlights:\n"))
newel[u'other'] = str(input("Other:\n"))
newel[u'text'] = str(input("Random Thoughts:\n"))

print(newel)
with open(path,'r') as data_file:
    print('Successfuly read')
    data = json.load(data_file)
    data.append(newel)
with open(path,'w') as data_file:
    data_file.write(json.dumps(data, sort_keys=True, indent=4))
    print('Successfuly written')


# -*-- encoding=utf-8 --*-

import pandas as pd
import xlsxwriter
import os
import platform
from pandas import ExcelWriter
from util import main_function,plot_trend

def __read_one_csv_file(inCsvFileName):
    try:
        callFailData=pd.read_csv(inCsvFileName,
                                     dtype={'呼叫对方号码': object,'运营商': object,
                                            'imei': object,'起呼位置码': object,
                                            '起呼基站编号': object,'结束位置码': object,
                                            '结束基站编号': object,'isim支持情况': object},low_memory=False)
        #print(callFailData.columns)
        #print(callFailData.shape)
        return callFailData
    except:
        return None

def __read_csv_directory(inCsvFileName):
    callFailDataList=[]
    absPath=os.path.abspath(inCsvFileName)
    print(absPath)
    for li in os.listdir(absPath):
        print(li)
        sysstr = platform.system()
        #print('current OS is '+sysstr)
        if(sysstr =="Windows"):
            oldName=absPath+'\\'+li
        elif(sysstr == "Linux"):
            oldName=absPath+'/'+li
        else:
            oldName=absPath+'/'+li

        callFailData1=__read_one_csv_file(oldName)
        if callFailData1 is not None:
            callFailDataList.append(callFailData1)

    callFailData = callFailDataList[0]
    for i in range(1,len(callFailDataList)):
        callFailData = callFailData.append(callFailDataList[i], ignore_index=True)

    print(callFailData.shape)
    return callFailData

def __clean_data_all_data(callFailData):
    #'内部机型', '外部机型', '系统版本', 'emmcid', 'imei', '地区码', '发生时间', '上报时间', '异常进程名', '进程版本名',
    # '进程版本号', '异常进程包名', '软件系统类型', '国家', '省/直辖市', '市', '县/区', '详细地址', '异常类型', '出现异常的卡',
    # '失败原因', '呼入呼出', '起呼位置码', '起呼基站编号', '起呼电话网络', '开始数据网络', '运营商', '结束位置码',
    # '结束基站编号', '结束电话网络', '结束数据网络', 'isim支持情况', 'MBN版本信息', 'VOLTE配置信息', '是否volte',
    # '呼叫对方号码', '保留字段一', '保留字段二', '异常次数', '日志路径', 'log信息'

    rowLength_before=callFailData.shape[0]
    #---原始数据，只是填充null，无任何过滤
    callFailData=callFailData.fillna('null')

    #---只是过滤掉正常的原因（网络释放原因）
    fp=open(os.path.join(os.path.abspath('.'),'config','remove_items.txt'),'r')
    allines=fp.readlines()
    for cause in allines:
        callFailData=callFailData[callFailData['失败原因'].apply(lambda x: x!=cause.strip())]

    print('-----------------------------------'+str(callFailData.shape[0]))

    shape_after_remove_cause=callFailData.shape[0]
    #---移除测试的PLMN
    callFailData = callFailData.loc[(callFailData["运营商"] != "99901") &
                                    (callFailData["运营商"] != "00000") &
                                    (callFailData["运营商"] != "00101") &
                                    (callFailData["运营商"] != "123456") &
                                    (callFailData["运营商"] != "null")]

    #---起呼位置码 0、1
    callFailData=callFailData[callFailData['起呼位置码'].apply(lambda x: x.strip() != 0)]
    callFailData=callFailData[callFailData['起呼位置码'].apply(lambda x: x.strip() != 1)]
    callFailData=callFailData[callFailData['起呼位置码'].apply(lambda x: x.strip() != '0')]
    callFailData=callFailData[callFailData['起呼位置码'].apply(lambda x: x.strip() != '1')]

    #---结束位置码 0、1
    callFailData=callFailData[callFailData['结束位置码'].apply(lambda x: x.strip() != 0)]
    callFailData=callFailData[callFailData['结束位置码'].apply(lambda x: x.strip() != 1)]
    callFailData=callFailData[callFailData['结束位置码'].apply(lambda x: x.strip() != '0')]
    callFailData=callFailData[callFailData['结束位置码'].apply(lambda x: x.strip() != '1')]

    #---起呼基站编号 0、1
    callFailData=callFailData[callFailData['起呼基站编号'].apply(lambda x: x.strip() != 0)]
    callFailData=callFailData[callFailData['起呼基站编号'].apply(lambda x: x.strip() != 1)]
    callFailData=callFailData[callFailData['起呼基站编号'].apply(lambda x: x.strip() != '0')]
    callFailData=callFailData[callFailData['起呼基站编号'].apply(lambda x: x.strip() != '1')]

    #---结束基站编号 0、1
    callFailData=callFailData[callFailData['结束基站编号'].apply(lambda x: x.strip() != 0)]
    callFailData=callFailData[callFailData['结束基站编号'].apply(lambda x: x.strip() != 1)]
    callFailData=callFailData[callFailData['结束基站编号'].apply(lambda x: x.strip() != '0')]
    callFailData=callFailData[callFailData['结束基站编号'].apply(lambda x: x.strip() != '1')]

    #---起呼电话网络 UNKNOWN
    callFailData=callFailData[callFailData['起呼电话网络'].apply(lambda x: x != 'UNKNOWN')]

    callFailData = callFailData.loc[(callFailData["imei"] != "123456789012345")]

    #---添加辅助分析项
    callFailData['PLMN_LAC1_CID1']=callFailData['运营商'].str.cat(callFailData['起呼位置码'],sep='/').str.cat(callFailData['起呼基站编号'],sep='/')
    callFailData['PLMN_LAC2_CID2']=callFailData['运营商'].str.cat(callFailData['结束位置码'],sep='/').str.cat(callFailData['结束基站编号'],sep='/')

    callFailData['CS_NW']=callFailData['起呼电话网络'].str.cat(callFailData['结束电话网络'],sep='/')
    callFailData['PS_NW']=callFailData['开始数据网络'].str.cat(callFailData['结束数据网络'],sep='/')
    callFailData['CS_PS_NW']=callFailData['CS_NW'].str.cat(callFailData['PS_NW'],sep='/')

    callFailData['PLMN_CS1'] = callFailData['运营商'].str.cat(callFailData['起呼电话网络'], sep='/')

    callFailData['PLMN_CS_NW'] = callFailData['运营商'].str.cat(callFailData['CS_NW'], sep='/')
    callFailData['PLMN_PS_NW'] = callFailData['运营商'].str.cat(callFailData['PS_NW'], sep='/')
    callFailData['PLMN_CS_PS_NW'] = callFailData['运营商'].str.cat(callFailData['CS_PS_NW'], sep='/')

    callFailData['机型-版本'] = callFailData['外部机型'].str.cat(callFailData['系统版本'], sep='/')

    callFailData['省直辖市']=callFailData['省/直辖市']
    callFailData['县区']=callFailData['县/区']
    callFailData['市1']=callFailData['省直辖市'].str.cat(callFailData['市'],sep='-')
    callFailData['县区1']=callFailData['市1'].str.cat(callFailData['县区'],sep='-')

    callFailData['通话状态']=callFailData['呼叫对方号码'].apply(__removeStateSpace)
    callFailData['信号强度']=callFailData['isim支持情况'].apply(__getRSRP)

    callFailData['发生时间t']=pd.to_datetime(callFailData['发生时间'],infer_datetime_format=True)
    callFailData['发生时间h']=callFailData['发生时间t'].apply(__getHour)

    callFailData['出现异常的卡']=callFailData['出现异常的卡'].apply(__replace_sim)
    callFailData['机型']=callFailData['外部机型']

    #PD1635    PD1616B    PD1619    PD1624    PD1616
    #callFailData = callFailData[callFailData['机型'] == 'PD1619']
    #callFailData = callFailData[callFailData['失败原因'] == 'CALL_END_CAUSE_FADE_V02']

    callFailData['通话类型'] = callFailData['CS_NW'].str.cat(callFailData['是否volte'], sep='/')
    callFailData['通话类型1'] = callFailData['PLMN_CS_NW'].str.cat(callFailData['是否volte'], sep='/')
    callFailData['cause-state'] = callFailData['失败原因'].str.cat(callFailData['通话状态'], sep='/')
    callFailData['CS_sig'] = callFailData['通话类型'].str.cat(callFailData['信号强度'], sep='/')
    callFailData['cause_cs_sig'] = callFailData['失败原因'].str.cat(callFailData['CS_sig'], sep='/')

    #---drop没有利用价值的项
    data_every_file1=callFailData.drop(['外部机型','内部机型','emmcid','地区码','上报时间','异常进程名','进程版本名',
                                    '进程版本号','异常进程包名','软件系统类型','异常类型','isim支持情况',
                                    'MBN版本信息','VOLTE配置信息','呼叫对方号码','保留字段一','保留字段二',
                                    '异常次数','日志路径','log信息','省/直辖市','县/区','发生时间','市',
                                    '县区','发生时间t','机型-版本','起呼位置码','结束位置码','起呼基站编号','结束基站编号',
                                    '结束电话网络','结束数据网络','PS_NW','CS_PS_NW',
                                    'PLMN_PS_NW','PLMN_CS_PS_NW','发生时间h','市1','县区1',
                                    ],axis=1)

    rowLength_after=callFailData.shape[0]
    print('数据清洗之后...'+str(rowLength_after)+'/'+str(rowLength_before))
    return data_every_file1,data_every_file1,shape_after_remove_cause

def __get_mcc(name):
    return(name[:3])

def __replace_sim(sim):
    if(sim==1):
        return '卡1'
    elif(sim==2):
        return '卡2'
    else:
        return 'null'

def __getHour(name):
    returnName=name.to_pydatetime().hour
    return returnName

def __getRSRP(name):
    returnName = name.strip()
    rsrp_list = []
    returnValue = 0
    if(name=='-1' or name=='null'):
        returnValue = str(-1)
    else:
        rsrp_list = returnName.split(',')
        min = 0
        for i in rsrp_list[:-2]:
            temp = eval(i)
            if(min > temp):
                min = temp
        returnValue = int(min / 5) * 5
    return str(returnValue)

def __removeStateSpace(name):
    returnName=name.strip()
    if(' ' in name):
        returnName=','.join(name.split(' '))
    else:
        pass
    return returnName

def __process_zhejiang_IMEI(callFailData,path,file_pre):
    model_list_fp=open(os.path.join(os.path.abspath('.'),'config','云诊断内销浙江统计机型列表.txt'),'r')
    modelList=[]
    for model in model_list_fp.readlines():
        modelList.append(model.strip())

    xls_fileName=os.path.join(path,file_pre+'_数据分析结果_浙江IMEI.xls')
    workbook = xlsxwriter.Workbook(xls_fileName)

    #---对每一个型号进行过滤和对比
    #如果包含在写入excel表格
    list_result=[]
    for model in modelList:
        model0=model.split('_')[0]
        model1=model.split('_')[1]

        worksheet = workbook.add_worksheet(model)
        worksheet.set_column('A:A',20)

        before=str(callFailData.shape[0])
        callFailData_after=callFailData[callFailData['外部机型']==model0]
        after=str(callFailData_after.shape[0])

        print('开始过滤'+model+'...'+after+'/'+before)

        #获取dataframe中的所有IMEI数据
        imeiList_a=[]
        for imei in callFailData_after['imei'].tolist():
            imeiList_a.append(str(imei).strip())

        #获取文件中浙江的IMEI列表
        imeiList_b=[]
        fileName=os.path.join('.','zhejiang_imei',model1+'.txt')
        imeiFile_fp=open(fileName,'r')
        imei_zhejiang=imeiFile_fp.readlines()
        for imei in imei_zhejiang:
            imeiList_b.append(imei.strip())

        #获得浙江IMEI列表和dataframe IMEI中的交集
        IMEI_intersection=list(set(imeiList_a).intersection(set(imeiList_b)))
        #print('a='+str(len(imeiList_a))+',b='+str(len(imeiList_b))+',intersection='+str(len(IMEI_intersection)))

        #按照dataframe的数量排序，获取浙江输出到excel
        callFailData_IMEI=callFailData_after['imei'].value_counts()
        allIMEI=callFailData_IMEI.index.tolist()

        row_i=0
        for imei_i in range(len(allIMEI)):
            for imei_filtered in IMEI_intersection:
                if(imei_filtered==allIMEI[imei_i]):

                    worksheet.write(row_i,0,imei_filtered)
                    worksheet.write(row_i,1,callFailData_IMEI.values[imei_i])
                    list_result.append((imei_filtered,callFailData_IMEI.values[imei_i]),)
                    row_i += 1

    #---对所有过滤出来的浙江IMEI计算Top
    print('ouput all...')
    worksheet = workbook.add_worksheet('all')
    worksheet.set_column('A:A',20)
    mylist=sorted(list_result,key=lambda t:t[1],reverse=True)
    for i in range(len(mylist)):
        worksheet.write(i,0,mylist[i][0])
        worksheet.write(i,1,mylist[i][1])
    workbook.close()

    length_mylist=0
    if(len(mylist) < 1):
        callFailData_internal = pd.DataFrame(columns=callFailData.columns)
    else:
        if(len(mylist) < 10):
            length_mylist=len(mylist)
        else:
            length_mylist=10

        callFailDataList=[]
        for i in range(length_mylist):
            callFailData_internal=callFailData[callFailData['imei']==mylist[i][0]]
            callFailDataList.append(callFailData_internal)

        callFailData_internal = pd.DataFrame(columns=callFailData.columns)
        for i in range(1,len(callFailDataList)):
            callFailData_internal = callFailData_internal.append(callFailDataList[i], ignore_index=True)

    xls_fileName1=os.path.join(path,file_pre+'_数据分析结果_浙江IMEI详细信息.xlsx')
    writer = ExcelWriter(xls_fileName1)
    callFailData_internal.to_excel(writer,'data')
    writer.save()

def __process_trial_IMEI(callFailData,path,inCsvFileName_head):
    modelList=[]
    for model in open(os.path.join('.','config','云诊断内销掉话试用机列表.txt'),'r').readlines():
        modelList.append(model.strip())

    xls_fileName=os.path.join(path,inCsvFileName_head+'_数据分析结果_试用机IMEI.xls')
    workbook = xlsxwriter.Workbook(xls_fileName)

    xls_fileName1=os.path.join(path,inCsvFileName_head+'_数据分析结果_试用机IMEI详细信息.xlsx')
    writer = ExcelWriter(xls_fileName1)

    #---对每一个试用机机型进行过滤和比对
    for model in modelList:
        model0=model.split('_')[0]
        model1=model.split('_')[1]
        worksheet = workbook.add_worksheet(model)
        before=str(callFailData.shape[0])
        private_callFailData=callFailData[callFailData['外部机型']==model0]
        after=str(private_callFailData.shape[0])

        print('开始过滤'+model+'...'+after+'/'+before)

        imeiList_a=[]
        for imei in private_callFailData['imei'].tolist():
            imeiList_a.append(str(imei).strip())

        fileName=os.path.join(os.path.abspath('.'),'trial_imei',model1+'.txt')
        imeiFile_fp=open(fileName,'r')
        imeiList_b=[]
        for imei in imeiFile_fp.readlines():
            imeiList_b.append(imei.split()[0].strip())
            imeiList_b.append(imei.split()[1].strip())

        IMEI_intersection=list(set(imeiList_a).intersection(set(imeiList_b)))
        print('a='+str(len(imeiList_a))+',b='+str(len(imeiList_b))+'intersection='+str(len(IMEI_intersection)))

        private_callFailData1=pd.DataFrame(columns=callFailData.columns)
        for imei_i in range(len(IMEI_intersection)):
            worksheet.write(imei_i,0,IMEI_intersection[imei_i])
            private_callFailData1=private_callFailData1.append(private_callFailData[callFailData['imei']==IMEI_intersection[imei_i]])

        private_callFailData1.to_excel(writer,model)
    writer.save()

def cloud_in_callfail_main(path_raw_data,path_result):
    main_function('云诊断内销掉话', path_raw_data, path_result, __read_one_csv_file, __read_csv_directory,
                  __clean_data_all_data)

def cloud_in_call_fail_plot_trend(path_raw_data,path_result):
    sheet_name_list=['SIM卡', '失败原因', '呼入或呼出', '运营商', '电话网络', '发生时间h', '机型', '系统版本', 'PLMN_CS']

    trend_dics_list={}
    trend_dics_list['出现异常的卡']=['卡1','卡2']

    trend_dics_list['通话类型1'] = ['46000/GSM/GSM/CS', '46001/UMTS/UMTS/CS',
                                '46000/LTE/GSM/CS', '46000/LTE/LTE/VOLTE',
                                '46011/CDMA - 1xRTT/CDMA - 1xRTT/CS']

    trend_dics_list['失败原因']=['CALL_END_CAUSE_RECOVERY_ON_TIMER_EXPIRED_V02', 'CALL_END_CAUSE_FADE_V02', 'CALL_END_CAUSE_RADIO_LINK_LOST_V02', 'CALL_END_CAUSE_UNSPECIFIED_16', 'CALL_END_CAUSE_REQUEST_TERMINATED_V02']

    trend_dics_list['呼入呼出']=['In','Out']

    trend_dics_list['运营商']=['46000','46001','46011','46003']

    trend_dics_list['是否volte']=['CS','VOLTE','VILTE']

    trend_dics_list['系统版本']=['PD1616B_A_1.6.18', 'PD1616B_A_1.7.1', 'PD1616B_A_1.7.7', 'PD1616B_A_1.7.8', 'PD1616B_A_1.7.10', 'PD1616B_A_1.7.13', 'PD1616B_A_1.8.5', 'PD1616B_A_1.8.9']

    trend_dics_list['机型']=['PD1635','PD1624','PD1616B','PD1619','PD1610','PD1616']
#
    trend_dics_list['CS_NW']=['GSM/GSM','UMTS/UMTS','LTE/LTE','LTE/GSM','LTE/UMTS']

    trend_dics_list['省直辖市']=['广东省','河南省','甘肃省','江苏省','河北省','山西省','浙江省','新疆维吾尔自治区',
                             '广西壮族自治区','安徽省','山东省','福建省','湖南省','贵州省','陕西省','云南省',
                             '黑龙江省','四川省','吉林省','辽宁省','湖北省','内蒙古自治区','宁夏回族自治区',
                             '北京市','上海市','江西省','重庆市','青海省','海南省','天津市','西藏自治区']

    plot_trend('云诊断内销掉话', path_raw_data, path_result, trend_dics_list)

if __name__ == '__main__':
    path=os.path.abspath('D:/tools/pycharm_projects/bigdata_analysis/cloud_in_callfail_raw_data/cloud_in_callfail_raw_data_weeks/test')
    cloud_in_callfail_main(path,path)


import pygame
import os
from sys import exit

WIDTH, HEIGHT = 1600, 900
pygame.init()
WIN = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("Buildings")
WHITE = (255, 255, 255)
FPS = 60
indx = 0
font = pygame.font.Font(None, 50)

class Building():
    def __init__(self, name, offset_x, offset_y):
        self.name = name
        self.building = pygame.image.load(os.path.join(
            "assets/buildings", self.name)).convert_alpha()
        self.offset_x = offset_x
        self.offset_y = offset_y
        self.x = offset_x
        self.y = offset_y

    def color_key(self, color):
        self.building.set_colorkey(color)

    def draw(self, WIN, camera_position):
        self.VEL = 20
        self.x = -camera_position[0]+self.offset_x
        self.y = -camera_position[1]+self.offset_y
        WIN.blit(self.building, (self.x, self.y))

    def movements(self, keys_pressed):
        if keys_pressed is None:
            return
        if keys_pressed[pygame.K_a]:
            self.x += self.VEL
            if keys_pressed[pygame.K_LSHIFT]:
                self.x += self.VEL
        elif keys_pressed[pygame.K_d]:
            self.x -= self.VEL
            if keys_pressed[pygame.K_LSHIFT]:
                self.x -= self.VEL

    def movements_updown(self, keys_pressed):
        if keys_pressed[pygame.K_w]:
            self.y += self.VEL
            if keys_pressed[pygame.K_LSHIFT]:
                self.y += self.VEL
        elif keys_pressed[pygame.K_s]:
            self.y -= self.VEL
            if keys_pressed[pygame.K_LSHIFT]:
                self.y -= self.VEL

    def collision_bound(self, camera_position, x_fac, y_fac, width_fac, height_fac, show=False):
        self.x = -camera_position[0]+self.offset_x
        self.y = -camera_position[1]+self.offset_y
        object_collision_bound = pygame.Rect(
            self.x+x_fac, self.y+y_fac, width_fac, height_fac)
        if show is True:
            pygame.draw.rect(WIN, (0, 255, 0), object_collision_bound)

def main():
    building1 = Building("4(1).png", -100, -1250)
    building2 = Building("5(1).png", 2000, -1800)
    building3 = Building("6.png", 5500, -1700)
    clock = pygame.time.Clock()

    while True:
        WIN.fill(WHITE)
        clock.tick(FPS)
        for event in pygame.event.get():
            keys_pressed = pygame.key.get_pressed()
            if event.type == pygame.QUIT:
                pygame.quit()
                exit()
        building1.draw(WIN, [0, 0])
        building1.movements(keys_pressed)
        building1.movements_updown(keys_pressed)
        building2.draw(WIN, [0, 0])
        building2.movements(keys_pressed)
        building2.movements_updown(keys_pressed)
        building3.draw(WIN, [0, 0])
        building3.movements(keys_pressed)
        building3.movements_updown(keys_pressed)
        pygame.display.update()

if __name__ == "__main__":
    main()