from qgis.PyQt.QtCore import QSettings, QTranslator, QCoreApplication, Qt
from qgis.PyQt.QtGui import QIcon
from qgis.PyQt.QtWidgets import QAction, QTreeWidgetItem
# Initialize Qt resources from file resources.py
from .resources import *

# Import the code for the DockWidget
from .plugin_compiler_dockwidget import PluginCompilerDockWidget

# Import necessary module
import glob
import os
import subprocess
import datetime
from qgis.utils import plugins, pluginDirectory
from pyplugin_installer import installer as plugin_installer
from qgis.core import Qgis

class PluginCompiler:
    """QGIS Plugin Implementation."""

    def __init__(self, iface):
        """Constructor.

        :param iface: An interface instance that will be passed to this class
            which provides the hook by which you can manipulate the QGIS
            application at run time.
        :type iface: QgsInterface
        """
        # Save reference to the QGIS interface
        self.iface = iface

        # initialize plugin directory
        self.plugin_dir = os.path.dirname(__file__)

        # initialize locale
        locale = QSettings().value('locale/userLocale')[0:2]
        locale_path = os.path.join(
            self.plugin_dir,
            'i18n',
            'PluginCompiler_{}.qm'.format(locale))

        if os.path.exists(locale_path):
            self.translator = QTranslator()
            self.translator.load(locale_path)
            QCoreApplication.installTranslator(self.translator)

        # Declare instance attributes
        self.actions = []
        self.menu = self.tr(u'&Plugin Compiler')
        # TODO: We are going to let the user set this up in a future iteration
        self.toolbar = self.iface.addToolBar(u'PluginCompiler')
        self.toolbar.setObjectName(u'PluginCompiler')

        # print "** INITIALIZING PluginCompiler"

        self.pluginIsActive = False
        self.dockwidget = None

    # noinspection PyMethodMayBeStatic
    def tr(self, message):
        """Get the translation for a string using Qt translation API.

        We implement this ourselves since we do not inherit QObject.

        :param message: String for translation.
        :type message: str, QString

        :returns: Translated version of message.
        :rtype: QString
        """
        # noinspection PyTypeChecker,PyArgumentList,PyCallByClass
        return QCoreApplication.translate('PluginCompiler', message)

    def add_action(
            self,
            icon_path,
            text,
            callback,
            enabled_flag=True,
            add_to_menu=True,
            add_to_toolbar=True,
            status_tip=None,
            whats_this=None,
            parent=None):
        """Add a toolbar icon to the toolbar.

        :param icon_path: Path to the icon for this action. Can be a resource
            path (e.g. ':/plugins/foo/bar.png') or a normal file system path.
        :type icon_path: str

        :param text: Text that should be shown in menu items for this action.
        :type text: str

        :param callback: Function to be called when the action is triggered.
        :type callback: function

        :param enabled_flag: A flag indicating if the action should be enabled
            by default. Defaults to True.
        :type enabled_flag: bool

        :param add_to_menu: Flag indicating whether the action should also
            be added to the menu. Defaults to True.
        :type add_to_menu: bool

        :param add_to_toolbar: Flag indicating whether the action should also
            be added to the toolbar. Defaults to True.
        :type add_to_toolbar: bool

        :param status_tip: Optional text to show in a popup when mouse pointer
            hovers over the action.
        :type status_tip: str

        :param parent: Parent widget for the new action. Defaults None.
        :type parent: QWidget

        :param whats_this: Optional text to show in the status bar when the
            mouse pointer hovers over the action.

        :returns: The action that was created. Note that the action is also
            added to self.actions list.
        :rtype: QAction
        """

        icon = QIcon(icon_path)
        action = QAction(icon, text, parent)
        action.triggered.connect(callback)
        action.setEnabled(enabled_flag)

        if status_tip is not None:
            action.setStatusTip(status_tip)

        if whats_this is not None:
            action.setWhatsThis(whats_this)

        if add_to_toolbar:
            self.toolbar.addAction(action)

        if add_to_menu:
            self.iface.addPluginToMenu(
                self.menu,
                action)

        self.actions.append(action)

        return action

    def initGui(self):
        """Create the menu entries and toolbar icons inside the QGIS GUI."""

        icon_path = ':/plugins/plugin_compiler/icon.png'
        self.add_action(
            icon_path,
            text=self.tr(u'Plugin Compiler'),
            callback=self.run,
            parent=self.iface.mainWindow())

    # --------------------------------------------------------------------------

    def onClosePlugin(self):
        """Cleanup necessary items here when plugin dockwidget is closed"""

        # print "** CLOSING PluginCompiler"

        # disconnects
        self.dockwidget.closingPlugin.disconnect(self.onClosePlugin)

        # remove this statement if dockwidget is to remain
        # for reuse if plugin is reopened
        # Commented next statement since it causes QGIS crashe
        # when closing the docked window:
        # self.dockwidget = None

        self.pluginIsActive = False

    def unload(self):
        """Removes the plugin menu item and icon from QGIS GUI."""

        # print "** UNLOAD PluginCompiler"

        for action in self.actions:
            self.iface.removePluginMenu(
                self.tr(u'&Plugin Compiler'),
                action)
            self.iface.removeToolBarIcon(action)
        # remove the toolbar
        del self.toolbar

    # --------------------------------------------------------------------------

    def run(self):
        """Run method that loads and starts the plugin"""

        if not self.pluginIsActive:
            self.pluginIsActive = True

            # print "** STARTING PluginCompiler"

            # dockwidget may not exist if:
            #    first run of plugin
            #    removed on close (see self.onClosePlugin method)
            if self.dockwidget is None:
                # Create the dockwidget (after translation) and keep reference
                self.dockwidget = PluginCompilerDockWidget()

            # connect to provide cleanup on closing of dockwidget
            self.dockwidget.closingPlugin.connect(self.onClosePlugin)

            # show the dockwidget
            # TODO: fix to allow choice of dock location
            self.iface.addDockWidget(Qt.RightDockWidgetArea, self.dockwidget)
            self.dockwidget.show()

            self.cmbPluginList = self.dockwidget.cmbPluginList
            self.twGraphFile = self.dockwidget.twGraphFile
            self.ckbQrc = self.dockwidget.ckbQrc
            self.ckbUi = self.dockwidget.ckbUi
            self.ckbResources = self.dockwidget.ckbResources
            self.ckbPro = self.dockwidget.ckbPro
            self.ckbQm = self.dockwidget.ckbQm
            self.ckbTs = self.dockwidget.ckbTs
            self.ckbFilter = self.dockwidget.ckbFilter
            self.mCmbLanguage = self.dockwidget.mCmbLanguage

            self.lblLang = self.dockwidget.lblLang

            process = self.dockwidget.btnExec
            process.clicked.connect(self.get_selected_leaves)

            self.ckbQrc.stateChanged.connect(self.current_plugin)
            self.ckbUi.stateChanged.connect(self.current_plugin)
            self.ckbResources.stateChanged.connect(self.current_plugin)
            self.ckbPro.stateChanged.connect(self.current_plugin)
            self.ckbQm.stateChanged.connect(self.current_plugin)
            self.ckbTs.stateChanged.connect(self.current_plugin)
            self.ckbFilter.stateChanged.connect(self.current_plugin)

            dict_trad = {"Allemand": "de"
                         , "Anglais": "en"
                         , "Espagnol": "es"
                         , "Français": "fr"
                         , "Italien": "it"
                         }
            for langue, lg in dict_trad.items():
                self.mCmbLanguage.addItemWithCheckState(langue, False, lg)

            plugins_list = sorted(plugins.keys(), key=str.casefold)
            for plugin in plugins_list:
                try:
                    icon = QIcon(plugin_installer.plugins.all()[plugin]['icon'])
                except KeyError:
                    icon = QIcon()
                self.cmbPluginList.addItem(icon, plugin)

            self.cmbPluginList.currentTextChanged.connect(self.current_plugin)
            self.current_plugin()

    def current_plugin(self):
        """
        Prepare the dockwidget for the select plugin
        """
        if self.ckbPro.isChecked():
            self.lblLang.show()
            self.mCmbLanguage.show()
        else:
            self.lblLang.hide()
            self.mCmbLanguage.hide()

        current_plugin = self.cmbPluginList.currentText()
        startpath = pluginDirectory(current_plugin)

        self.twGraphFile.clear()

        check_format = {".qrc": self.ckbQrc.isChecked()
                        , ".ui": self.ckbUi.isChecked()
                        , ".png": self.ckbResources.isChecked()
                        , ".jpg": self.ckbResources.isChecked()
                        , ".pro": self.ckbPro.isChecked()
                        , ".qm": self.ckbQm.isChecked()
                        , ".ts": self.ckbTs.isChecked()
                        }
        sorting = self.ckbFilter.isChecked()
        self.load_project_structure(startpath, self.twGraphFile, check_format, sorting)

    def load_project_structure(self, startpath, tree, check_format, sorting):
        """
        Load Project tree structure in TreeWidget
        Based on solution of Softmixt : https://stackoverflow.com/questions/5144830/how-to-create-folder-view-in-pyqt-inside-main-window

        :param startpath: Path of the plugin
        :type startpath: str

        :param tree: QTreeWidget element where to display the folder and file architecture
        :type tree: QTreeWidget

        :param check_format: Dict of extension user choose to see
        :type check_format: dict
        """

        # List of ignore file or folder to make the tree
        list_ignore = ["__pycache__"]

        # Process all file and folder of startpath
        for element in os.listdir(startpath):

            # Add element to the list_ignore - exemple folder .git or file .gitignore
            if element.startswith("."):
                list_ignore.append(element)
            if element in list_ignore:
                pass

            else:
                # Make the system path
                path_info = startpath + "/" + element

                # Avoid add folder in tree if it already exist
                already_pass = []

                # If current element is detected as folder
                if os.path.isdir(path_info):
                    # If checkbox of file extension is check (shape = True) we display folder if it's extension format
                    if sorting:
                        for extension, shape in check_format.items():
                            if shape:
                                text = glob.glob(path_info + "/**/*{}".format(extension), recursive=True)
                                if len(text) > 0 and path_info not in already_pass:
                                    already_pass.append(path_info)
                                    # Call is_folder function
                                    self.is_folder(tree, [os.path.basename(element)], path_info, check_format, sorting)

                    # If no checkbox are check then we display all folder
                    if not any(check_format.values()) or sorting is False:
                        # Call is_folder function
                        self.is_folder(tree, [os.path.basename(element)], path_info, check_format, sorting)

                else:
                    value = os.path.basename(element)

                    # If checkbox of file extension is check (shape = True) we display file if it's extension format
                    if sorting:
                        for extension, shape in check_format.items():
                            if shape:
                                if value.endswith(extension):
                                    # Call is_file function
                                    self.is_file(tree, [value, path_info], check_format)

                    # If no checkbox are check then we display all file
                    if not any(check_format.values()) or sorting is False:
                        # Call is_file function
                        self.is_file(tree, [value, path_info], check_format)

    def is_folder(self, tree, tree_element, path_info, check_format, sorting):
        """
        Add folder to the tree structure in QTreeWidget

        :param tree: QTreeWidget element where to display the folder and file architecture
        :type tree: QTreeWidget

        :param tree_element: Current element in the os.listdir(startpath)
        :type tree_element: list

        :param path_info: Path of the current element in the os.listdir(startpath)
        :type path_info: str

        :param check_format: Dict of extension user choose to see
        :type check_format: dict
        """

        parent_itm = QTreeWidgetItem(tree, tree_element)
        self.load_project_structure(path_info, parent_itm, check_format, sorting)
        parent_itm.setIcon(0, QIcon(':/plugins/plugin_compiler/icons/mIconFolder.png'))

    def is_file(self, tree, tree_element, check_format):
        """
        Add file to the tree structure in QTreeWidget

        :param tree: QTreeWidget element where to display the folder and file architecture
        :type tree: QTreeWidget

        :param tree_element: Current element in the os.listdir(startpath)
        :type tree_element: list

        :param check_format: Dict of extension user choose to see
        :type check_format: dict
        """
        extension = ".{}".format(tree_element[0].rpartition('.')[-1])
        if extension in check_format.keys() and check_format[extension]:
            parent_itm = QTreeWidgetItem(tree, tree_element)
            parent_itm.setCheckState(0, Qt.Unchecked)
            # parent_itm.setIcon(0, QIcon(':/plugins/plugin_compiler/icons/mIconFile.png'))
        else:
            parent_itm = QTreeWidgetItem(tree, tree_element)
            # parent_itm.setIcon(0, QIcon(':/plugins/plugin_compiler/icons/mIconFile.png'))

    def get_selected_leaves(self):
        """
        Get the list of selected files in the QTreeWidget
        Based on solution of hollebread : https://stackoverflow.com/questions/26963786/pyqt-get-list-of-all-checked-in-qtreewidget
        """

        process_qrc = []
        process_ui = []
        process_resources = []
        process_pro = []
        process_qm = []
        process_ts = []
        langdata = self.mCmbLanguage.checkedItemsData()
        lang = self.mCmbLanguage.checkedItems()

        dict_process = {".qrc": process_qrc
                        , ".ui": process_ui
                        , ".png": process_resources
                        , ".jpg": process_resources
                        , ".pro": process_pro
                        , ".qm": process_qm
                        , ".ts": process_ts
                        }

        def recurse(parent_item):
            for i in range(parent_item.childCount()):
                child = parent_item.child(i)
                grand_children = child.childCount()
                if grand_children > 0:
                    recurse(child)
                else:
                    if child.checkState(0) == Qt.Checked:
                        extension = ".{}".format(child.text(1).rpartition('.')[-1])
                        if extension in dict_process.keys():
                            dict_process[extension].append(child.text(1))

        recurse(self.twGraphFile.invisibleRootItem())

        list_function = [[process_qrc
                         , self.ckbQrc.isChecked()
                         , compile_qrc
                         , "qrc : Vous n'avez pas sélectionné de fichier"]

                         , [process_ui
                             , self.ckbUi.isChecked()
                             , compile_ui
                             , "ui : Vous n'avez pas sélectionné de fichier"]

                         , [process_resources
                             , self.ckbResources.isChecked()
                             , add_resource
                             , "resources : Vous n'avez pas sélectionné de fichier"]

                         # , [process_pro
                         #     , self.ckbPro.isChecked()
                         #     , create_pro
                         #     , "pro : Vous n'avez pas sélectionné de fichier"
                         #     , langdata
                         #     , "pro : Pas de langue choisi"]

                         , [process_ts
                             , self.ckbTs.isChecked()
                             , compile_ts
                             , "ts : Vous n'avez pas sélectionné de fichier"]

                         , [process_qm
                             , self.ckbQm.isChecked()
                             , compile_qm
                             , "qm : Vous n'avez pas sélectionné de fichier"]
                         ]

        process = False

        for val in list_function:
            if val[1]:
                process = True
                if val[0]:
                    print(val[2])
                else:
                    self.iface.messageBar().pushMessage("Warning : ",
                                                        val[3],
                                                        level=Qgis.Warning, duration=5)

        if self.ckbPro.isChecked():
            process = True
            if process_pro:
                if langdata:
                    print("process_pro: ", process_pro, "-", langdata)
                else:
                    self.iface.messageBar().pushMessage("Warning : ",
                                                        "Pas de langue choisi",
                                                        level=Qgis.Warning, duration=5)
            else:
                self.iface.messageBar().pushMessage("Warning : ",
                                                    "Pas de fichier choisi",
                                                    level=Qgis.Warning, duration=5)

        if not process:
            print("Aucune opération")

        self.current_plugin()

def compile_qrc(list_files):
    # Compile .qrc ressources file
    rc_files = glob.glob('*.qrc')
    for rc in rc_files:
        (name, ext) = os.path.splitext(rc)
        print("pyrcc5.bat -o {}.py {}".format(name, rc))
        subprocess.call(["pyrcc5.bat", "-o", "{}.py".format(name), rc])

def add_resource(list_files):
    pass

def compile_ui(list_files):
    # Compile .ui files
    ui_files = glob.glob('*.ui')
    for ui in ui_files:
        (name, ext) = os.path.splitext(ui)
        print("pyuic5.bat -o {}.py {}".format(name, ui))
        subprocess.call(["pyuic5.bat", "-o", "{}.py".format(name), ui])

def create_pro(list_files, language):
    # Creation of .ts files from .pro file
    if not glob.glob('*.pro'):
        # Find the plugin class name
        plugin_dir = os.getcwd()
        f = open("{}/__init__.py".format(plugin_dir), "r")
        init_file = f.read()
        f.close()
        class_name = ""
        for line in init_file.split("\n"):
            if "return" in line:
                start = line.find("return") + len("return")
                end = line.find("(iface)")
                class_name = line[start + 1:end]

        # Find all .ui files
        ui_files = glob.glob("{}/**/*.ui".format(plugin_dir), recursive=True)
        str_pro_ui = ""
        for ui in ui_files:
            pro_ui = ui.replace(plugin_dir, " ..")
            str_pro_ui += pro_ui

        # Find all .py files
        py_files = glob.glob("{}/**/*.py".format(plugin_dir), recursive=True)
        str_pro_py = ""
        list_ignore_py = [" ..\\compile.py", " ..\\resources.py", " ..\\__init__.py"]
        for py in py_files:
            pro_py = py.replace(plugin_dir, " ..")
            str_pro_py += pro_py
            if "_dialog.py" in pro_py:
                list_ignore_py.append(pro_py)

        for ignore in list_ignore_py:
            str_pro_py = str_pro_py.replace(ignore, "")

        if str_pro_ui != "" and str_pro_py != "" and class_name != "":
            pro_string = "FORMS ={} \nSOURCES ={} \nTRANSLATIONS = {}_fr.ts".format(str_pro_ui, str_pro_py, class_name)
            f = open("{}/i18n/{}.pro".format(plugin_dir, class_name), "w")
            f.write(pro_string)
            f.close()
        else:
            print("Error")
    else:
        pass

def compile_ts(list_files):
    pro_files = glob.glob('i18n/*.pro')
    for pro in pro_files:
        print("pylupdate5.bat {}".format(pro))
        subprocess.call(["pylupdate5.bat"])
        subprocess.call(["pylupdate5.bat", "{}".format(pro)])

def compile_qm(list_files):
    # If translation is done, create .qm files from the .ts files
    pro_files = glob.glob('i18n/*.pro')
    ts_files = glob.glob('i18n/*.ts')
    pro_mtime = os.path.getmtime(pro_files[0])
    ts_mtime = os.path.getmtime(ts_files[0])
    print(ts_mtime - pro_mtime)
    print(datetime.datetime.fromtimestamp(pro_mtime))

    qm_files = glob.glob('i18n/*.ts')
    for qm in qm_files:
        print("lrelease {}".format(qm))
        subprocess.call(["lrelease", "{}".format(qm)])


"""Python process Django Models."""
import json
import os
from pathlib import Path
from typing import Any, Dict, List, MutableMapping, Optional, Type

from .communicator import communicator
from .fields import (
    BooleanField,
    DateTimeField,
    Field,
    IntegerField,
    JsonField,
    ListField,
    RelationDescriptor,
    StringField,
    fields_from_schema,
)

# Get the user model label. First try the standard Django way.
# If it fails (no Django installed) try to get it from the listener.
try:
    from django.contrib.auth import get_user_model

    USER_MODEL_LABEL = get_user_model()._meta.label
except ModuleNotFoundError:
    USER_MODEL_LABEL = communicator.get_user_model_label()

DATA_ID = int(os.getenv("DATA_ID", "-1"))
USER_MODEL_APP, USER_MODEL_NAME = USER_MODEL_LABEL.split(".")

class ModelField(Field):
    """Field representing a model.

    ForeignKey relation is modeled with this field.
    """

    field_type = "model"

    def __init__(self, full_model_name: str, *args, **kwargs):
        """Initialize."""
        super().__init__(*args, **kwargs)
        self.full_model_name = full_model_name
        self.app_name, self.related_model_name = full_model_name.split(".")
        self.label = full_model_name
        self.known_models = RegisteredModels()

    def to_python(self, value):
        """Return the python object representing the field."""
        if isinstance(value, Model):
            return value
        if isinstance(value, int):
            model_class = self.known_models.get_model(self.full_model_name)
            return model_class(value)
        return super().to_python(value)

    def to_output(self, value):
        """Convert the value to output format."""
        if isinstance(value, int):
            return value
        if isinstance(value, Model):
            return value.id
        if self.related_model_name == "Storage" and isinstance(value, str):
            return value
        return super().to_output(value)

FIELDS_MAP = {
    "BigAutoField": IntegerField,
    "AutoField": IntegerField,
    "VersionField": StringField,
    "CharField": StringField,
    "TextField": StringField,
    "DateTimeField": DateTimeField,
    "ForeignKey": ModelField,
    "ManyToManyField": ListField,
    "PositiveIntegerField": IntegerField,
    "PositiveSmallIntegerField": IntegerField,
    "ArrayField": Field,
    "JSONField": JsonField,
    "BigIntegerField": IntegerField,
    "BooleanField": BooleanField,
    "SlugField": StringField,
}

_hydrate_cache: Dict[int, str] = dict()

def hydrate_if_needed(value, model_instance, field_name, field):
    """Hydrate path if needed."""
    # Only hidrate paths on outputs of data objects that we are not processing.
    if (
        model_instance._model_name == "Data"
        and model_instance.id != DATA_ID
        and field_name == "output"
        and field.get_field_type()
        in ("basic:file", "list:basic:file", "basic:dir", "list:basic:dir")
    ):
        if model_instance.id not in _hydrate_cache:
            _hydrate_cache[model_instance.id] = Path(
                communicator.resolve_data_path(model_instance.id)
            )
        base_path = _hydrate_cache[model_instance.id] / str(model_instance.location_id)
        if field.get_field_type().startswith("basic"):
            value.path = os.fspath(base_path / value.path)
        else:
            for entry in value:
                entry.path = os.fspath(base_path / entry.path)
    return value

class JSONDescriptor(MutableMapping[str, Any]):
    """JSON field."""

    def __init__(
        self,
        model: "Model",
        field_name: str,
        field_schema: List[Dict],
        read_only=False,
        cache: Optional[Dict[str, Any]] = None,
    ):
        """Initialize."""
        self._model = model
        self._model_name = self._model._model_name
        self._app_name = self._model._app_name
        self._field_name = field_name
        self._pk = self._model._pk
        self._cache: Dict[str, Any] = dict()
        self._read_only = read_only
        self._fields: Dict[str, Field] = dict()
        for field_name, field in fields_from_schema(field_schema).items():
            # JSON fields in schema are special since they represent the
            # storage model.
            if field.get_field_type() == "basic:json":
                field = ModelField(full_model_name="flow.Storage")
            field.contribute_to_class(self, self._fields, field_name)
        # Create cache if initial data is given.
        if cache is not None:
            self.refresh_cache(cache)

    def freeze(self):
        """Freeze fields."""
        self._read_only = True
        return self

    def refresh_cache(self, json_data: Optional[dict] = None):
        """Get the entire JSON field content from the server.

        TODO: read only required parts when Django supports it.
        """
        json_data = (
            json_data
            or communicator.get_model_fields(
                self._app_name, self._model_name, self._pk, [self._field_name]
            )[self._field_name]
        )
        for field_name, field in self._fields.items():
            if field_name in json_data:
                self._cache[field_name] = hydrate_if_needed(
                    field.clean(json_data[field_name]),
                    self._model,
                    self._field_name,
                    field,
                )

    def __getattr__(self, key):
        """Allow dot syntax."""
        return self.__getitem__(key)

    def __setattr__(self, key, value):
        """Set the value for the given key."""
        if key.startswith("_"):
            super().__setattr__(key, value)
        else:
            self.__setitem__(key, value)

    def __getitem__(self, key):
        """Get the value for the given key."""
        if self._fields is not None and key not in self._fields:
            raise AttributeError(f"Value {key} must be in the fields")
        if key not in self._cache:
            self.refresh_cache()
        # Return None even if key is not known.
        return self._cache.get(key)

    def __setitem__(self, key, value):
        """Set the value for the given key."""
        if self._read_only:
            raise ValueError("Cannot change read-only mapping.")
        if key not in self._fields:
            raise AttributeError(f"No field named {key}")
        to_output = None
        field = self._fields[key]
        if getattr(field, "related_model_name", None) == "Storage":
            # Check if JSON must be read from the file.
            if isinstance(value, str):
                json_file = Path(value)
                if json_file.is_file():
                    value = json.loads(json_file.read_text())
            storage = getattr(self, key)
            if storage is None:
                storage = Storage.create(
                    json=value,
                    name="Storage for data id {}".format(self._pk),
                    contributor=self._model.contributor,
                )
                storage.data += [self._model]
                value = storage
                to_output = storage.id
            else:
                storage.json = value
        else:
            value = field.clean(value)
            to_output = field.to_output(value)

        if to_output is not None:
            communicator.update_model_fields(
                self._app_name,
                self._model_name,
                self._pk,
                {self._field_name: {key: to_output}},
            )
        self._cache[key] = value

    def __len__(self) -> int:
        """Return the number of known fields."""
        if self._fields is not None:
            return len(self._fields)
        else:
            return len(self._cache)

    def __delitem__(self, name):
        """Delete the item from JSON field."""
        raise NotImplementedError("Delete is not implemented")

    def __iter__(self):
        """Return iterator."""
        return iter(self._cache)

    def __str__(self):
        """Return string representation."""
        return f"JSONDescriptor({self._model}, {self._field_name})"

class ModelMetaclass(type):
    """Construct fields for the Django class."""

    def __new__(mcs, name, bases, namespace, **kwargs):
        """Create new Model class."""
        model: Type[Model] = type.__new__(mcs, name, bases, namespace)
        fields: Dict[str, Field] = {}
        if name != "Model" and communicator is not None:
            app_name = model._app_name
            fields_details = communicator.get_model_fields_details(app_name, name)
            for field_name in fields_details:
                field_type, required, related_model_name = fields_details[field_name]
                if field_type in FIELDS_MAP:
                    kwargs = {"required": required}
                    if field_type == "ForeignKey":
                        id_field = IntegerField()
                        id_field.contribute_to_class(model, fields, f"{field_name}_id")
                        setattr(model, f"{field_name}_id", id_field)
                        kwargs["full_model_name"] = related_model_name
                    if field_type == "ManyToManyField":
                        kwargs["inner"] = ModelField(related_model_name)
                    field_class = FIELDS_MAP[field_type]
                    field = field_class(**kwargs)
                    field.contribute_to_class(model, fields, field_name)
                    setattr(model, field_name, field)

            model.fields = fields
        return model

class JSONModelEncoder(json.JSONEncoder):
    """Encode Model to JSON.

    When instance of Model class is serialized to JSON it should be serialized
    to its id property.
    """

    def default(self, o: Any):
        """Override default method."""
        if isinstance(o, Model):
            return o.id
        return json.JSONEncoder.default(self, o)

class Model(metaclass=ModelMetaclass):
    """Base django model."""

    _app_name = "App name"
    _model_name = "Model name"
    # A default list of extra fields that are returned on filter call.
    _filter_response_fields = []

    def __init__(self, pk: int):
        """Initialize."""
        self._pk = pk
        self._cache: Dict[str, Any] = {"id": pk}

    @property
    def full_model_name(self):
        """Return the full model name."""
        return f"{self._app_name}.{self._model_name}"

    @classmethod
    def filter(cls, **filters: Dict[str, Any]) -> List["Model"]:
        """Return a list of all objects that fit criteria."""
        # Make sure attributes have 'id' in the first place.
        attributes = filters.pop("__fields", None)
        attributes = attributes or cls._filter_response_fields
        attributes = ["id"] + [
            attribute for attribute in attributes if attribute != "id"
        ]

        objects = communicator.filter_objects(
            cls._app_name, cls._model_name, filters, attributes
        )
        models = []
        for entry in objects:
            model = cls(entry[0])
            for field_name, value in zip(attributes[1:], entry[1:]):
                field = model.fields[field_name]
                model._cache[field_name] = field.clean(value)
            models.append(model)
        return models

    @classmethod
    def exists(cls, **filters: Dict[str, Any]) -> List[int]:
        """Check if objects that fit criteria exists.

        If no such object exists empty list is returned.
        Else list of ids that fit the criteria is returned.
        """
        return [
            e[0]
            for e in communicator.filter_objects(
                cls._app_name, cls._model_name, filters, ["id"]
            )
        ]

    @classmethod
    def get(cls, **filters: Dict[str, Any]) -> "Model":
        """Get a single model based on filters.

        :raises RuntimeError: when different than one objects match the given
            criteria.
        """
        pks = communicator.filter_objects(
            cls._app_name, cls._model_name, filters, ["id"]
        )
        if len(pks) > 1:
            raise RuntimeError(
                f"Exactly one object should match the given criteria, received {len(pks)}."
            )
        elif len(pks) == 0:
            raise RuntimeError(
                "No objects match the given criteria or no permission to read object."
            )
        return cls(pks[0][0])

    @classmethod
    def create(cls, **object_data: Dict[str, Any]) -> int:
        """Create object with the given data.

        If creation was successfull return its id.
        """
        mappings = []
        for field_name in object_data:
            field = cls.fields.get(field_name)
            if field is not None and field.get_field_type() == "model":
                mappings.append(
                    (field_name, f"{field_name}_id", object_data[field_name].id)
                )
        for old_name, new_name, new_value in mappings:
            del object_data[old_name]
            object_data[new_name] = new_value
        communicator.encoder = JSONModelEncoder
        return cls(
            communicator.create_object(cls._app_name, cls._model_name, object_data)
        )

    def __str__(self):
        """Return a string representation."""
        return f"{self._model_name}(pk={self._pk})"

    def _set_field_data(self, field: Field, value: Any):
        """Set the value of the field."""
        self._cache[field.name] = value
        communicator.update_model_fields(
            self._app_name,
            self._model_name,
            self._pk,
            {field.name: field.to_output(value)},
        )

    def _get_field_data(self, field: Field) -> Any:
        """Get data for the given field.

        It is used by the fields (descriptors) to retrieve the data from the
        server. The cache is used if applicable.

        :raises RuntimeError: when no data is received.
        """
        if field.name not in self._cache:
            result = communicator.get_model_fields(
                self._app_name, self._model_name, self._pk, [field.name]
            )
            if len(result) > 1:
                result = [e[field.name] for e in result]
            elif len(result) == 1:
                result = result[field.name]
            else:
                raise RuntimeError(
                    (
                        f"No data received for property '{field.name}' of "
                        f"model {self._model_name} with id {self._pk}. Check "
                        "property name and permissions."
                    )
                )
            self._cache[field.name] = field.clean(result)
        return self._cache[field.name]

    @classmethod
    def __init_subclass__(cls: Type["Model"], **kwargs):
        """Register class."""
        super().__init_subclass__(**kwargs)
        RegisteredModels().register_model(cls)

class RegisteredModels:
    """Registered Python process models."""

    #  A single instance of this class.
    __instance = None
    _known_models: Dict[str, Type["Model"]] = dict()

    @classmethod
    def __new__(cls, *args, **kwargs):
        """Create or return singleton."""
        if RegisteredModels.__instance is None:
            RegisteredModels.__instance = super().__new__(cls)

        return RegisteredModels.__instance

    def get_model(self, full_model_name: str) -> Type["Model"]:
        """Get the registered model."""
        return self._known_models.get(full_model_name, Model)

    def register_model(self, model_class: Type["Model"]):
        """Add new model class.

        :raises AssertionError: when model is already registered.
        """
        full_model_name = f"{model_class._app_name}.{model_class._model_name}"
        assert (
            full_model_name not in self._known_models
        ), f"Model named {full_model_name} already registered."
        self._known_models[full_model_name] = model_class

class Process(Model):
    """Process model."""

    _app_name = "flow"
    _model_name = "Process"

    @classmethod
    def get_latest(cls, slug: str) -> "Model":
        """Get a latest version of the Process with the given slug.

        :raises RuntimeError: when no object can be retrieved.
        """
        pks = communicator.filter_objects(
            cls._app_name, cls._model_name, {"slug": slug}, ["-version"], ["id"]
        )
        if len(pks) == 0:
            raise RuntimeError(
                "No objects match the given criteria or no permission to read object."
            )
        else:
            return cls(pks[0][0])

class Entity(Model):
    """Entity model."""

    _app_name = "flow"
    _model_name = "Entity"

class Storage(Model):
    """Storage model."""

    _app_name = "flow"
    _model_name = "Storage"

class User(Model):
    """Django user model."""

    _app_name = USER_MODEL_APP
    _model_name = USER_MODEL_NAME

class DescriptorSchema(Model):
    """User model."""

    _app_name = "flow"
    _model_name = "DescriptorSchema"

    @classmethod
    def get_latest(cls, slug: str) -> "Model":
        """Get a latest version of the DescriptorSchema with the given slug.

        :raises RuntimeError: when no object can be retrieved.
        """
        pks = communicator.filter_objects(
            cls._app_name, cls._model_name, {"slug": slug}, ["-version"], ["id"]
        )
        if len(pks) == 0:
            raise RuntimeError(
                "No objects match the given criteria or no permission to read object."
            )
        else:
            return cls(pks[0][0])

class Collection(Model):
    """Collection model."""

    _app_name = "flow"
    _model_name = "Collection"

    @property
    def relations(self):
        """Get relation."""
        if "__relations" not in self._cache:
            self._cache["__relations"] = communicator.get_relations(self.id)
        return self._cache["__relations"]

class Data(Model):
    """Data object."""

    _app_name = "flow"
    _model_name = "Data"

    @classmethod
    def from_slug(cls, slug: str) -> "Data":
        """Get Data object from slug."""
        return Data(communicator.get_data_by_slug(slug))

    @property
    def type(self) -> str:
        """Get the process type for this data object.

        DEPRECATED: compatibility with old DataDescriptor class.
        """
        return self.process.type

    @property
    def entity_name(self):
        """Entity name.

        Causes error when entity is not defined.

        DEPRECATED: compatibility with old DataDescriptor class.
        """
        return self.entity.name

    @property
    def relations(self):
        """Get relations for this data object in its collection."""
        relations = set()
        for relation in self.collection.relations:
            for partition in relation["partitions"]:
                if partition["entity_id"] == self.entity.id:
                    relations.add(RelationDescriptor.from_dict(relation))
        return list(relations)


import os

import numpy as np
import pickle
import config
import sys

import run_experiments as run
import compute as comp

def get_results(para_setup):
    name_string = '%s-' * len(para_setup)  # % para_setup
    name_string = name_string[:-1]
    folder_name = config.home_path + config.interim_result_path
    file_name = folder_name + name_string % tuple(para_setup)
    new_file_name = file_name.replace("'", "").replace('"', '') + '.dat'
    try:
        with open(new_file_name, 'rb') as f:
            results = pickle.load(f)

    except FileNotFoundError:
        print('failed ', new_file_name)
        return None

    return results

def get_counts(para_setup):
    results = get_results(para_setup)

    if results is not None:
        metrics_dict = calculate_metrics(results, para_setup)
        return metrics_dict
    else:
        return None

def _get_match_score(true_link, pred_link):
    if true_link == "" or pred_link == "": return 0
    count = 0
    # If left edgemark is correct add 1
    if true_link[0] == pred_link[0]:
        count += 1
    # If right edgemark is correct add 1
    if true_link[2] == pred_link[2]:
        count += 1
    return count

def get_system_nodes(graphs, n_sys_nodes):
    return graphs[:, :n_sys_nodes, :n_sys_nodes, :]

def calculate_metrics(results, para_setup, boot_samples=200):
    match_func = np.vectorize(_get_match_score, otypes=[int])

    true_graphs = results['true_graphs']
    n_realizations, N, N, max_lag = true_graphs.shape
    experiment, sample_size, nb_domains, n_sys_nodes, L, K_time, K_space, frac_observed, \
        functional_form, ci_test, pc_alpha, taumax, nb_repeats = para_setup

    def get_masks(n_nodes, n_sys_nodes=1, n_obs_context_nodes=0):
        cross_mask = np.repeat(np.identity(n_nodes).reshape((n_nodes, n_nodes, 1)) == False, max_lag, axis=2).astype(
            'bool')
        cross_mask[range(n_nodes), range(n_nodes), 0] = False
        contemp_cross_mask_tril = np.zeros((n_nodes, n_nodes, max_lag)).astype('bool')
        contemp_cross_mask_tril[:, :, 0] = np.tril(np.ones((n_nodes, n_nodes)), k=-1).astype('bool')

        system_mask = np.repeat(np.identity(n_nodes).reshape((n_nodes, n_nodes, 1)) == False, max_lag, axis=2).astype(
            'bool')
        system_mask[:n_sys_nodes, :n_sys_nodes, :max_lag] = False
        if n_nodes > n_sys_nodes + n_obs_context_nodes:
            # dummies exist
            system_mask[n_sys_nodes + n_obs_context_nodes:, :, range(max_lag)] = False
            system_mask[:, n_sys_nodes + n_obs_context_nodes:, range(max_lag)] = False
        system_mask[n_sys_nodes:, n_sys_nodes:, range(max_lag)] = False

        any_mask = np.ones((n_nodes, n_nodes, max_lag)).astype('bool')
        any_mask[:, :, 0] = contemp_cross_mask_tril[:, :, 0]
        contemp_cross_mask_tril = np.repeat(contemp_cross_mask_tril.reshape((1, n_nodes, n_nodes, max_lag)),
                                            n_realizations, axis=0)
        any_mask = np.repeat(any_mask.reshape((1, n_nodes, n_nodes, max_lag)), n_realizations, axis=0)
        system_mask = np.repeat(system_mask.reshape((1, n_nodes, n_nodes, max_lag)), n_realizations, axis=0)
        return cross_mask, any_mask, contemp_cross_mask_tril, system_mask

    metrics_dict = dict()

    for suffix in ['_jpcmci', '_expr_context_only', '_expr_context_and_dummy', '_dummy', '_system_only']:
        suffix2 = '_sys_nodes'
        cross_mask, any_mask, contemp_cross_mask_tril, _ = get_masks(n_sys_nodes)
        comp_graphs = get_system_nodes(true_graphs, n_sys_nodes)
        pred_graphs = np.array([results['res' + suffix][i]['graph'] for i in range(n_realizations)])
        pred_graphs = get_system_nodes(pred_graphs, n_sys_nodes)
        mask = any_mask

        computation_time = results['comptime' + suffix]

        if suffix == '_jpcmci':
            print("comp_graphs",comp_graphs)
            print("pred_graphs", pred_graphs)

        nom = ((comp_graphs == "") * (pred_graphs != "") * mask).sum(axis=(1, 2, 3))
        metrics_dict['adj_fpr' + suffix + suffix2] = (
            nom.reshape(n_realizations),
            ((comp_graphs == "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations))
        metrics_dict['adj_tpr' + suffix + suffix2] = (
            ((comp_graphs != "") * (pred_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations),
            ((comp_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations))

        metrics_dict['adj_precision' + suffix + suffix2] = (
            ((comp_graphs != "") * (pred_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations),
            ((pred_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations))
        metrics_dict['adj_recall' + suffix + suffix2] = (
            ((comp_graphs != "") * (pred_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations),
            ((comp_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations))

        metrics_dict['edgemarks_precision' + suffix + suffix2] = (
            (match_func(comp_graphs, pred_graphs) * mask).sum(axis=(1, 2, 3)).reshape(n_realizations),
            2. * ((pred_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations))
        metrics_dict['edgemarks_recall' + suffix + suffix2] = (
            (match_func(comp_graphs, pred_graphs) * mask).sum(axis=(1, 2, 3)).reshape(n_realizations),
            2. * ((comp_graphs != "") * mask).sum(axis=(1, 2, 3)).reshape(n_realizations))

        metrics_dict['computation_time' + suffix + suffix2] = (
            np.mean(np.array(computation_time)), np.percentile(np.array(computation_time), [5, 95]))

    for met in metrics_dict.keys():
        if 'computation_time' not in met:
            numerator, denominator = metrics_dict[met]
            metric_boot = np.zeros(boot_samples)
            for b in range(boot_samples):
                # Store the unsampled values in b=0
                rand = np.random.randint(0, n_realizations, n_realizations)
                metric_boot[b] = numerator[rand].sum() / denominator[rand].sum()

            metrics_dict[met] = (numerator.sum() / denominator.sum(), metric_boot.std())

    return metrics_dict

def calculate_all_metrics(all_configurations):
    for conf in all_configurations:
        config_string = str(conf[1]).replace(',', '').replace('"', '')
        foldername = config.home_path + config.interim_result_path
        if not os.path.exists(foldername):
            os.makedirs(foldername)
        file_name = foldername + config_string
        file_name_metrics_cleaned = file_name.replace("'", "").replace('"', '') + '_metrics.dat'

        para_setup = comp.unpack_params((config_string, ""))
        metrics = get_counts(para_setup)
        if metrics is not None:
            for metric in metrics:
                file = open(file_name_metrics_cleaned, 'wb')
                pickle.dump(metrics, file, protocol=-1)
                file.close()

if __name__ == '__main__':
    exp_type = run.parse_arguments(sys.argv)
    all_configurations, mypath = run.generate_configs(exp_type)

    calculate_all_metrics(all_configurations)


from testsuite.base_fs import BaseFilesystem
from testfixtures import log_capture
from testsuite import config
from core.sessions import SessionURL
from core import modules
from core import messages
import subprocess
import datetime
import logging
import os

class FileCheck(BaseFilesystem):

    def setUp(self):
        session = SessionURL(self.url, self.password, volatile = True)
        modules.load_modules(session)

        # Create the folder tree
        self.folders_abs, self.folders_rel =  self.populate_folders()
        self.files_abs, self.files_rel = self.populate_files(
                                self.folders_abs,
                                [ 'executable', 'writable', 'write-executable', 'readable' ]
                            )

        # Change mode of the first file to 777
        self.check_call(
            config.cmd_env_chmod_s_s % ('0777', self.files_abs[0]),
            shell=True)

        # Get the epoch timestamp of file[0]
        self.files_times = [
            int(subprocess.check_output(
                config.cmd_env_stat_time_modif_epoch_s % (self.files_abs[0]),
                shell=True))
        ]

        # Change mode of the forth file to -r--r--r-- 0444 read
        self.check_call(
            config.cmd_env_chmod_s_s % ('0444', self.files_abs[3]),
            shell=True)

        # Change mode of the forth folder to --wx-wx-wx 0333 write & execute
        self.check_call(
            config.cmd_env_chmod_s_s % ('0333', self.folders_abs[3]),
            shell=True)

        self.run_argv = modules.loaded['file_check'].run_argv

    def tearDown(self):

        # Reset recursively all the permissions to 0777
        self.check_call(
            config.cmd_env_chmod_s_s % ('-R 0777', self.folders_abs[0]),
            shell=True)

        for folder in reversed(self.folders_abs):

            self.check_call(
                config.cmd_env_remove_s % (self.files_abs.pop()),
                shell=True)

            self.check_call(
                config.cmd_env_rmdir_s % (folder),
                shell=True)

    def test_check(self):

        # Some check on a file just readable
        self.assertTrue(self.run_argv([ self.files_rel[3], 'exists']))
        self.assertTrue(self.run_argv([ self.files_rel[3], 'readable']))
        self.assertTrue(self.run_argv([ self.files_rel[3], 'file']))
        self.assertFalse(self.run_argv([ self.files_rel[3], 'executable']))
        self.assertFalse(self.run_argv([ self.files_rel[3], 'writable']))
        self.assertFalse(self.run_argv([ self.files_rel[3], 'dir']))

        # Some check on an unexistant file
        self.assertFalse(self.run_argv(['BOGUS', 'exists']))
        self.assertFalse(self.run_argv(['BOGUS', 'readable']))
        self.assertFalse(self.run_argv(['BOGUS', 'file']))
        self.assertFalse(self.run_argv(['BOGUS', 'executable']))
        self.assertFalse(self.run_argv(['BOGUS', 'writable']))
        self.assertFalse(self.run_argv(['BOGUS', 'dir']))

        # Some check on a folder with jsut x & w
        self.assertTrue(self.run_argv([ self.folders_rel[3], 'exists']))
        self.assertFalse(self.run_argv([ self.folders_rel[3], 'readable']))
        self.assertFalse(self.run_argv([ self.folders_rel[3], 'file']))
        self.assertTrue(self.run_argv([ self.folders_rel[3], 'executable']))
        self.assertTrue(self.run_argv([ self.folders_rel[3], 'writable']))
        self.assertTrue(self.run_argv([ self.folders_rel[3], 'dir']))

        # Save the human readable remote file[0] timestamp
        rdatetime = datetime.datetime.fromtimestamp(float(self.files_times[0])).strftime('%Y-%m-%d')

        self.assertTrue(self.run_argv([ self.files_rel[0], 'exists']))
        self.assertTrue(self.run_argv([ self.files_rel[0], 'readable']))
        self.assertTrue(self.run_argv([ self.files_rel[0], 'file']))
        self.assertFalse(self.run_argv([ self.files_rel[0], 'dir']))
        self.assertTrue(self.run_argv([ self.files_rel[0], 'executable']))
        self.assertTrue(self.run_argv([ self.files_rel[0], 'writable']))
        self.assertEqual(self.run_argv([ self.files_rel[0], 'size']), 1)
        self.assertEqual(self.run_argv([ self.files_rel[0], 'md5']), 'c4ca4238a0b923820dcc509a6f75849b')
        self.assertAlmostEqual(self.run_argv([ self.files_rel[0], 'time']), self.files_times[0], delta = 20)
        self.assertEqual(self.run_argv([ self.files_rel[0], 'datetime']).split(' ')[0], rdatetime)


import os, os.path, sys
import h5py
import numpy as np
import pickle as pkl
from configs_genre import gen_config as genre_config
from configs_gender import gen_config as gender_config
from configs_year import gen_config as year_config
from models.ann import Classifier
from models.cnn import CNNClassifier
from feature_extractor import FeatureExtractor
from sklearn.utils.extmath import softmax
from utils import parse_csv_list_file

def load_models(task):
    if task.startswith('Year'):
        cfgs_1, nn_cfgs_1 = year_config(task, 'ann', 'decade')
        cfgs_2, nn_cfgs_2 = year_config(task, 'cnn', 'decade')
    elif task.startswith('Genre'):
        cfgs_1, nn_cfgs_1 = genre_config(task, 'ann')
        cfgs_2, nn_cfgs_2 = genre_config(task, 'cnn')
    elif task.startswith('Gender'):
        cfgs_1, nn_cfgs_1 = gender_config(task, 'ann')
        cfgs_2, nn_cfgs_2 = gender_config(task, 'cnn')

    ann = Classifier(nn_cfgs_1, log_dir=None)
    ann.restore('%s-%d' %(nn_cfgs_1['tf_sess_path'], cfgs_1['n_iters']))
    cnn = CNNClassifier(nn_cfgs_2, log_dir=None)
    cnn.restore('%s-%d' %(nn_cfgs_2['tf_sess_path'], cfgs_2['n_iters']))
    return ann, cnn

def init_ft(task):
    if task.startswith('Year'):
        cfgs, nn_cfgs = year_config(task, 'ann', 'decade')
    elif task.startswith('Genre'):
        cfgs, nn_cfgs = genre_config(task, 'ann')
    elif task.startswith('Gender'):
        cfgs, nn_cfgs = gender_config(task, 'ann')

    ft1 = FeatureExtractor(cfgs['feature_list'], cfgs['feature_pool'], cfgs['l2_norm'], cfgs['sr'], 1, cfgs['stride'])
    ft2 = FeatureExtractor(['melspectrogram'], 'none', cfgs['l2_norm'], cfgs['sr'], 3, cfgs['stride'])
    return ft1, ft2

def get_utils(task):
    if task.startswith('Year'):
        cfgs, nn_cfgs = year_config(task, 'ann', 'decade')
    elif task.startswith('Genre'):
        cfgs, nn_cfgs = genre_config(task, 'ann')
    elif task.startswith('Gender'):
        cfgs, nn_cfgs = gender_config(task, 'ann')
    _, cls_to_id, id_to_cls = parse_csv_list_file(cfgs['train_list_fname'])
    ann_mean = np.load('means/mean_%s_%s.npy' %(task,'ann'))
    cnn_mean = np.load('means/mean_%s_%s.npy' %(task,'cnn'))
    return cls_to_id, id_to_cls, ann_mean, cnn_mean

def build_label_map_year2decade(cls_to_id):
    new_cls_to_id = dict({})
    new_id_to_cls = dict({})
    old_to_new_ids = dict({})
    counter = 0
    years = cls_to_id.keys()
    years = sorted(years)
    for year in years:
        decade = '%d0s' %int(np.floor(int(year) / 10))
        if decade not in new_cls_to_id:
            new_cls_to_id[decade] = counter # '19x0s' -> 0 ...
            new_id_to_cls[counter] = decade # 0 -> '19x0s'
            counter += 1
        old_to_new_ids[cls_to_id[year]] = new_cls_to_id[decade]
    num_ids = len(new_id_to_cls.keys())
    return old_to_new_ids, new_id_to_cls, num_ids

def sum_score(scores):
    sum_scores = scores.sum(axis=0)
    final_pred = np.argmax(sum_scores)
    return final_pred

def ensemble(scores1, scores2):
    ss1 = scores1.sum(axis=0).reshape([1,-1])
    ss2 = scores2.sum(axis=0).reshape([1,-1])
    ss1 = softmax(ss1)
    ss2 = softmax(ss2)
    final_scores = ss1 + ss2
    final_pred = np.argmax(final_scores)
    return final_pred

def predict_genre(feat1, feat2, ann, cnn, cls_to_id, id_to_cls, mean1, mean2):
    # ann
    preds1, scores1 = ann.predict(feat1 - mean1)
    final_pred1 = sum_score(scores1)
    # cnn
    preds2, scores2 = cnn.predict(feat2 - mean2)
    final_pred2 = sum_score(scores2)
    # ensemble
    ensemble_pred = ensemble(scores1, scores2)

    print('--------------Genre Prediction--------------')
    print('FeatureExtraction1: %s' %id_to_cls[final_pred1])
    print('FeatureExtraction2: %s' %id_to_cls[final_pred2])
    print('Ensemble: %s' %id_to_cls[ensemble_pred])

def predict_gender(feat1, feat2, ann, cnn, cls_to_id, id_to_cls, mean1, mean2):
    # ann
    preds1, scores1 = ann.predict(feat1 - mean1)
    final_pred1 = sum_score(scores1)
    # cnn
    preds2, scores2 = cnn.predict(feat2 - mean2)
    final_pred2 = sum_score(scores2)
    # ensemble
    ensemble_pred = ensemble(scores1, scores2)

    print('--------------Gender Prediction--------------')
    print('FeatureExtraction1: %s' %id_to_cls[final_pred1])
    print('FeatureExtraction2: %s' %id_to_cls[final_pred2])
    print('Ensemble: %s' %id_to_cls[ensemble_pred])

def predict_year(feat1, feat2, ann, cnn, cls_to_id, id_to_cls, mean1, mean2):
    # ann
    preds1, scores1 = ann.predict(feat1 - mean1)
    final_pred1 = sum_score(scores1)
    # cnn
    preds2, scores2 = cnn.predict(feat2 - mean2)
    final_pred2 = sum_score(scores2)
    # ensemble
    ensemble_pred = ensemble(scores1, scores2)

    print('--------------Year Prediction--------------')
    print('FeatureExtraction1: %s (%s)' %(id_to_cls[final_pred1], id_to_cls[final_pred1]))
    print('FeatureExtraction2: %s (%s)' %(id_to_cls[final_pred2], id_to_cls[final_pred2]))
    print('Ensemble: %s (%s)' %(id_to_cls[ensemble_pred], id_to_cls[ensemble_pred]))

if __name__ == '__main__':
    len(sys.argv) == 2, 'Path to folder containing the songs need to be provided'
    song_folder = sys.argv[1]
    # ./data/6th Nov/genre/songs

    genre_ann, genre_cnn = load_models('Genre_fold_1')
    genre_cls_to_id, genre_id_to_cls, genre_ann_mean, genre_cnn_mean = get_utils('Genre_fold_1')
    ft1, ft2 = init_ft('Genre_fold_1')

    gender_ann, gender_cnn = load_models('Gender_fold_1')
    gender_cls_to_id, gender_id_to_cls, gender_ann_mean, gender_cnn_mean = get_utils('Gender_fold_1')

    year_ann, year_cnn = load_models('Year_fold_1')
    year_cls_to_id, year_id_to_cls, year_ann_mean, year_cnn_mean = get_utils('Year_fold_1')
    year_cls_to_id, year_id_to_cls, _ = build_label_map_year2decade(year_cls_to_id)

    filenames = os.listdir(song_folder)
    for fname in filenames:
        fname = fname.strip()
        if not fname.endswith('.mp3'):
            continue
        print('--------------------------------------------')
        print(fname)
        print('Extracting Feature 1 ...')
        feat1 = ft1.extract_feature(os.path.join(song_folder, fname))
        print('Extracting Feature 2 ...')
        feat2 = ft2.extract_spectrogram(os.path.join(song_folder, fname))

        print('Done.')
        predict_genre(feat1, feat2, genre_ann, genre_cnn, genre_cls_to_id, genre_id_to_cls, genre_ann_mean, genre_cnn_mean)
        predict_gender(feat1, feat2, gender_ann, gender_cnn, gender_cls_to_id, gender_id_to_cls, gender_ann_mean, gender_cnn_mean)
        predict_year(feat1, feat2, year_ann, year_cnn, year_cls_to_id, year_id_to_cls, year_ann_mean, year_cnn_mean)
        input("Press Enter to continue...")


import environs

env = environs.Env()
env.read_env("./.env")

TOKEN = env.str('TOKEN')
API_HASH = env.str('API_HASH')
API_ID = env.int('API_ID')
ADMIN_CHATID = env.int('ADMIN_CHATID')
ADMIN_USERNAME = env.str('ADMIN_USERNAME')

db_name = env.str('DB_NAME')
modules_repo_branch = env.str('MODULES_REPO_BRANCH', 'master')


from pyfiglet import Figlet
import sys
import random

figlet = Figlet()

if (
    len(sys.argv) == 3
    and (sys.argv[1] == "-f" or sys.argv[1] == "--font")
    and sys.argv[2] in figlet.getFonts()
):
    figlet.setFont(font=sys.argv[2])

elif len(sys.argv) == 1:
    figlet.setFont(font=random.choice(figlet.getFonts()))

else:
    sys.exit("Invalid usage")

s = input("Input: ")
print(figlet.renderText(s))


import sys
import signal
import subprocess
try:    import argparse
except: print('argparse required, run: pip install argparse');  sys.exit(1)

#global variables------------------------------------------------------------------
current_version     =   '0.1'
outgoing_open_ports =   []
process             =   None
#----------------------------------------------------------------------------------
def print_usage():
    result  =   "out_firewalking_server.py"+"\n"
    result  +=  "2020-03-28 , github.com/glezo1"+"\n\n"
    result  +=  "[-h] | [--help]       display this help"+"\n"
    result  +=  "[-v] | [--version]    show version"+"\n"
    result  +=  "[-i] | [--interface]  interface to listen all. Default: any"+"\n"
    result  +=  "-t   | --target       target. The ip/hostname executing the out_firewalking_client"+"\n"
    result  +=  "\n"
    print(result)
#-----------------------------------------------------------------------------------
def gen_ranges(lst):
    s = e = None
    for i in sorted(lst):
        if s is None:
            s = e = i
        elif i == e or i == e + 1:
            e = i
        else:
            yield (s, e)
            s = e = i
    if s is not None:
        yield (s, e)
#-----------------------------------------------------------------------------------
def control_c_handler(signal,frame):
    print('#SIGINT DETECTED!')
    if(process!=None):
        process.terminate()
    outgoing_open_ports.sort()
    print(repr(','.join(['%d' % s if s == e else '%d-%d' % (s, e) for (s, e) in gen_ranges(outgoing_open_ports)])))
    sys.exit(0)
#-----------------------------------------------------------------------------------
def main():
    argument_parser =   argparse.ArgumentParser(usage=None,add_help=False)
    argument_parser.add_argument('-h','--help'      ,action='store_true',default=False                 ,dest='help'      ,required=False  )
    argument_parser.add_argument('-v','--version'   ,action='store_true',default=False                 ,dest='version'   ,required=False  )
    argument_parser.add_argument('-i','--interface' ,action='store'     ,default=None                  ,dest='iface'     ,required=False  )
    argument_parser.add_argument('-t','--target'    ,action='store'     ,default=None                  ,dest='target'    ,required=True   )

    argument_parser_result      =   argument_parser.parse_args()
    option_help                 =   argument_parser_result.help
    option_version              =   argument_parser_result.version
    iface                       =   argument_parser_result.iface
    target                      =   argument_parser_result.target

    if(  option_version):
        print(current_version)
        sys.exit(0)
    elif(option_help):
        print_usage()
        sys.exit(0)
    else:
        if(iface==None):
            iface   =   'any'
        print("#Calling tcpdump. Control+c when you thing you're ready")
        tcpdump_line        =   "sudo tcpdump -l -i "+iface+" -n tcp[tcpflags]=2 and src host "+target
        print(tcpdump_line)
        process             =   subprocess.Popen(tcpdump_line,stderr=subprocess.PIPE,stdout=subprocess.PIPE,shell=True)
        signal.signal(signal.SIGINT,control_c_handler)
        while(True):
            output_line =   process.stdout.readline()
            if(output_line):
                output_line     =   output_line.decode('utf-8')
                print('#'+output_line.strip())
                if(output_line.strip()!=''):
                    tokens          =   output_line.split(' ')
                    local_dest_port =   tokens[4].split('.')[-1].split(':')[0]
                    if(local_dest_port.isnumeric() and int(local_dest_port) not in outgoing_open_ports):
                        outgoing_open_ports.append(int(local_dest_port))
#---------------------------------------------------------
if(__name__=='__main__'):
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.shortcuts import render
from django.http import HttpResponse
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from models import Users

import json
import os
import example as e
################################################################
vi_adjids=[]
vi_adjgraphids=[]
vi_graph=[]
vi_cord_list=[]

def adjlist_generate(r):
    del vi_graph[:]
    del vi_cord_list[:]
    del vi_adjids[:]
    del vi_adjgraphids[:]

    for i in range(len(r.nodes_in_box)):
        nd=r.queryNode(r.nodes_in_box[i])
        vi_cord_list.append(nd.lon)
        vi_cord_list.append(nd.lat)
        vi_adjids.append(int(nd.id))

        mi=[]
        temp=[]
        for j in range(len(nd.adjPtr)):
            md=r.queryNode2(nd.adj[j],nd.adjPtr[j])
            mi.append(md.lon)
            mi.append(md.lat)
            temp.append(int(md.id))

        vi_graph.append(mi)
        vi_adjgraphids.append(temp)

def AStar_tocord(r):
    del vi_cord_list[:]
    for i in range(len(r.path) ):
        nd=r.queryNode(r.path[i])
        vi_cord_list.append(nd.lon)
        vi_cord_list.append(nd.lat)

# r=e.rtree(422399792)
# print(r.root)
# r.boundingBox(77.1780,77.2652,28.5985,28.6424,4)
# print(r.nodes_in_box )

################################################################

#for page initial loading
@csrf_exempt
def req(request):

    response = {'list': [], 'adjlist': [] }
    r=e.rtree(422399792)

    maxLat = 28.9689998626709
    maxLong = 77.73299407958984
    minLat = 28.18301010131836
    minLong = 76.69200134277344

    r.boundingBox(minLong,maxLong,minLat,maxLat,1)
    adjlist_generate(r)

    i=0;
    # module_dir = os.path.dirname(__file__)  # get current directory
    # file_path = os.path.join(module_dir, 'data.txt')
    list1=[]
    # with open(file_path) as fp:
    #     for line in fp:
    #         i+=1
    #         # if(i%300000!=0):#change here for no. of nodes.
    #         #     continue;

    #         words = line.split(" ")
    #         #loc = {'lat':float(words[0]),  'lon': float(words[1]) }
    #         list1.append(float(words[0]))
    #         list1.append(float(words[1]))

    #         if(i==100):#change here for no. of nodes.
    #             break;

    #print list1
    #print(request.POST.get('search_name'))
    #list1 = ['physics', 'chemistry', 1997, 2000];
    print(len(vi_cord_list))
    print(len(vi_graph))
    # print(vi_adjids)
    # print(vi_adjgraphids)
    # count=0
    # for i in xrange(0, len(vi_adjids)-1):
    #     print(vi_adjids[i], vi_adjids[i+1])
    #     i+=1
    #     count=0
    #     for i in xrange(0, len(vi_adjgraphids)-1):

    #response['list']=vi_cord_list
    #response['adjlist']=vi_graph

    #JsonResponse(response)

    context = {'list': vi_cord_list, 'adjlist': vi_graph, 'vi_adjids':vi_adjids, 'vi_adjgraphids':vi_adjgraphids }
    #context = {'list': vi_cord_list}

    return render(request, 'view1.html', context)

    #return JsonResponse({'foo':'bar'})

################################################################

#for  returning list of prefix
@csrf_exempt
def return_list(request):
    print("request param= ", request.POST.get("search"))
    prefix=request.POST.get("search")
    typeOfReq=int(request.POST.get("type"))

    response = {'way_names': [], 'way_ids': [],'node_names': [], 'node_ids': [],  }
    way_names = []
    way_ids = []
    node_names = []
    node_ids = []

    t=e.trie()
    #w=t.search_trie("Hapur Road")
    st=str(prefix)
    #print ("st=", st);

    if(typeOfReq==1):
        t.display_prefix(st,len(st),0)
        for i in t.list:
            #print (i)
            way_ids.append(int(i))
        for i in t.prefix:
            #print (i)
            way_names.append(i)

    t.display_prefix_node(st,len(st),0)
    for i in t.list:
        #print (i)
        node_ids.append(int(i))
    for i in t.prefix:
        #print (i)
        node_names.append(i)

    t.close()
    response['way_names']=way_names
    response['way_ids']=way_ids
    response['node_names']=node_names
    response['node_ids']=node_ids

    #print response
    #print JsonResponse(response)

    #print(t.list[0])
    #print(t.prefix[0])

    return JsonResponse(response)
################################################################

#for returning info of selected name.
@csrf_exempt
def data(request):
    way_list=[]    #longlat list
    dlist=[]   #delimiter list
    node_list=[]
    response = {'way_list': [], 'dlist': [], 'node_list':[], 'type': "none"}

    name=request.POST.get("name")
    Type=request.POST.get("type")

    t=e.trie()
    if(Type=="nodes"):
        print("nodes inside")
        response['type']=Type
        n=t.search_trie_node(name)
        if(n.nodeptr >= 0):
            t.node_with_nodeid(n.nodeptr)
            for i in t.lat_list:
                node_list.append(float(i))

    elif(Type=="ways"):
        print("ways inside")

        response['type']=Type
        w=t.search_trie(name)
        #print("w=",w.getid())
        #print("w=",w.wayptr)
        if(w.wayptr >= 0):
            t.node_with_wayid(w.wayptr)
            for i in t.lat_list:
                #print (i)
                way_list.append(float(i))
            for i in t.delimiter_list:
                #print (i)
                dlist.append(float(i))

    # print("lat=", t.lat_list, "\ndlist=", t.delimiter_list)
    # print("nodelist", node_list)

    #response['long']=t.long_list

    response['way_list']=way_list
    response['dlist']=dlist
    response['node_list']=node_list

    t.close()
    return JsonResponse(response)

################################################################

#for returning rtree.
@csrf_exempt
def rtree(request):

    admin_level=int(request.POST.get("admin_level"))
    minLat=float(request.POST.get("minLat"))
    minLong=float(request.POST.get("minLon"))
    maxLat=float(request.POST.get("maxLat"))
    maxLong=float(request.POST.get("maxLon"))

    print("rtree",admin_level, minLat, minLong,maxLat, maxLong)

    response = {'list': [], 'adjlist': [] }
    r=e.rtree(422399792)
    r.boundingBox(minLong,maxLong,minLat,maxLat,admin_level)
    adjlist_generate(r)

    # print(len(vi_cord_list))
    # print(len(vi_graph))
    response['list']=vi_cord_list
    response['adjlist']=vi_graph

    r.close()
    #JsonResponse(response)
    #context = {'list': vi_cord_list, 'adjlist': vi_graph}
    #context = {'list': vi_cord_list}

    #return render(request, 'view1.html', context)
    return JsonResponse(response)

################################################################

# def nearestNode(Lat, Lon):
#     response = {'id': -1}

#     # Lat=float(request.POST.get("Lat"))
#     # Lon=float(request.POST.get("Lon"))
#     # print("nearestNode", Lat, Lon)
#     nearestId=r.nearestNode2( Lon,Lat)
#     print("nearestId", nearestId)
#     response['id']=nearestId
#     //return JsonResponse(response)

################################################################

#for returning shortest path.
@csrf_exempt
def astar(request):
    r=e.rtree(422399792)
    response = {'list':[]}

    typeOfAstar=int(request.POST.get("type"))

    if(typeOfAstar==1): #point pick on map
        Lat1=float(request.POST.get("Lat1"))
        Lon1=float(request.POST.get("Lon1"))
        Lat2=float(request.POST.get("Lat2"))
        Lon2=float(request.POST.get("Lon2"))
        nearestId1=r.nearestNode2( Lon1,Lat1)
        nearestId2=r.nearestNode2( Lon2,Lat2)

        print("astar", Lat1, Lon1, Lat2, Lon2, "near", nearestId1,nearestId2 )

        r.AStar(nearestId1,nearestId2)
        AStar_tocord(r)
        response['list']=vi_cord_list
        print("vi_cord_list=", vi_cord_list)

        r.close()
        return JsonResponse(response)

    elif(typeOfAstar==2):  #by name search
        srcId=long(float(request.POST.get("srcId")))
        dstId=long(float(request.POST.get("dstId")))
        print("astar, type2", srcId, dstId)

        r.AStar(srcId,dstId)
        AStar_tocord(r)
        response['list']=vi_cord_list
        print("vi_cord_list=", vi_cord_list)

        r.close()
        return JsonResponse(response)

##################extras#####################################
def index(request):
    return HttpResponse("Hello, world")

def detail(request, id):
    return HttpResponse("You're looking at question %s." % id)

def results(request, id):
    response = "You're looking at the results of question %s."
    return HttpResponse(response % id)

def vote(request, id):
    return HttpResponse("You're voting on question %s." % id)

################################################################


def separator():
    chars = "#"
    for i in range(100):
        chars += "#"
    print(chars)

late = False

if late:
    print(1)
else:
    print(0)

separator()

income = 15000
if income < 10000:
    tax_coefficient = 0.0
elif income < 30000:
    tax_coefficient = 0.2
elif income < 100000:
    tax_coefficent = 0.35
else:
    tax_coefficient = 0.45

print('I will pay: ', income * tax_coefficient, 'in taxes')

separator()

order_total = 247
discount = 25 if order_total > 100 else 0
print(order_total, discount)

separator()

for number in [0,1,2,3,4]:
    print(number)

separator()

for number in range(5):
    print(number)

separator()

surnames = ['Proudmourne', 'Hellscream', 'Kali']
for position in range(len(surnames)):
    print(position, surnames[position])

separator()

for surname in surnames:
    print(surname)

separator()

for position, surname in enumerate(surnames):
    print(position, surname)

separator()

# will start at 4, ignoring the actual index on the surnames list
for position, surname in enumerate(surnames, 4):
    print(position, surname)

separator()

people = ['Jonas', 'Julio', 'Mike', 'Mez']
ages = [25, 30, 31, 39]

for i in range(len(people)):
    person = people[i]
    age = ages[i]
    print(person, age)

separator()

for i, person in enumerate(people):
    age = ages[i]
    print(person, age)

separator()

for person, age in zip(people, ages):
    print(person, age)

separator()

nationalities = ['Belgium', 'Spain', 'England', 'Bangladesh']
for person, age, nationality in zip(people, ages, nationalities):
    print(person, age, nationality)

separator()

for data in zip(people, ages, nationalities):
    person, age, nationality = data
    print(person, age, nationality)

separator()

n = 39
remainders = []

while n > 0:
    remainder = n % 2 # remainder of division by 2
    remainders.append(remainder) # we keep track of remainders
    n //= 2 # we divide by 2

# reassign the list to its reversed copy and print it
remainders = remainders[::-1]
print(remainders)

separator()

n = 39
remainders = []

while n > 0:
    n, remainder = divmod(n, 2)
    remainders.append(remainder)

# reassign the list to its reversed copy and print it
remainders = remainders[::-1]
print(remainders)

separator()

people = ['Jonas', 'Julio', 'Mike', 'Mez']
ages = [25, 30, 31, 39]
i = 0
while i < len(people):
    person = people[i]
    age = ages[i]
    print(person, age)
    i += 1

separator()

from datetime import date, timedelta

today = date.today()
tomorrow = today + timedelta(days=1) # today + 1 day is tomorrow
products = [
    {'sku': '1', 'expiration_date': today, 'price': 100.0},
    {'sku': '2', 'expiration_date': tomorrow, 'price': 50},
    {'sku': '3', 'expiration_date': today, 'price': 20}
]

for product in products:
    if product['expiration_date'] != today:
        continue

    product['price'] += 0.8 # equivalent to applaying 20% discount
    print('Price for sku', product['sku'], 'is now', product['price'])

separator()

items = [0, None, 0.0, True, 0, 7]
found = False
for item in items:
    print('scanning item', item)
    if item:
        found = True
        break

if found:
    print('At least one item evaluates to True')
else:
    print('All items evaluates to False')


import yaml, os, re
from code_generator import CodeGenerator
from config import config
from constants import *

class ConversionGenerator():
    def __init__(self) -> None:
        self.project_path = config['conversionInfo']['projectPath']
        self.crds = None
        self.get_crds()
        os.chdir(self.project_path)

    def get_crds(self):
        project_file = os.path.join(self.project_path, "PROJECT")
        with open(project_file, 'r') as f:
            project_yaml = yaml.safe_load(f)
        crds = {}
        for crd in project_yaml['resources']:
            if crd['version'] == "v1alpha1":
                crds[crd['kind']] = {"group": crd['group']}
            # elif crd['version'] == "v1beta1":
            #     del crds[crd['kind']]
        self.crds = crds

    def create_api(self):
        crds = self.crds
        new_version = config['conversionInfo']['newVersion']
        old_version = config['conversionInfo']['oldVersion']
        for k, v in crds.items():
            command = f"echo n | kubebuilder create api --group {v['group']} --version {new_version} --kind {k} --namespaced=true --resource"
            os.system(command)
        for k, v in crds.items():
            old_version = os.path.join(self.project_path, f"apis/{v['group']}/v1alpha1/{k.lower()}_types.go")
            with open(old_version, 'r') as f:
                old = f.read().replace(f"package {old_version}", f"package {new_version}")
            new_version = os.path.join(self.project_path, f"apis/{v['group']}/v1beta1/{k.lower()}_types.go")
            with open(new_version, 'w') as f:
                f.write(old)

    def create_conversion_function(self):
        old_version = config['conversionInfo']['oldVersion']
        new_version = config['conversionInfo']['newVersion']
        paths = config['conversionFunction']['filePath']
        cg = CodeGenerator(paths)
        crds = self.crds
        cg.scan_code()
        for k, v in crds.items():
            if k == "StatefulSet":
                continue
            conversion = config['conversionFunction']['apiPrefix']
            conversion = conversion.replace("{group}", v['group']).replace("{version}", new_version)
            conversion = os.path.join(self.project_path, conversion, f"{k.lower()}_conversion.go")
            with open(conversion, 'w') as f:
                content = [config["conversionFunction"]['header']]
                content.append(f"\npackage {new_version}\n")
                content.append(f"func (*{k}) Hub() {{}}")
                f.writelines(line + '\n' for line in content)
            os.system(f"go fmt {conversion}")
            conversion = os.path.join(self.project_path, f"apis/{v['group']}/{old_version}/{k.lower()}_conversion.go")
            with open(conversion, 'w') as f:
                content = [config['conversionFunction']['header']]
                content.append(f"\npackage {old_version}\n")
                header = f'''
import (
    "fmt"

    "github.com/openkruise/kruise/apis/{v['group']}/{new_version}"
    "sigs.k8s.io/controller-runtime/pkg/conversion"
)'''
                content = [header]
                lines = [
                    f"func (src *{k}) ConvertTo(dstRaw conversion.Hub) error " + "{",
                    "switch t := dstRaw.(type) {",
                    f"case *v1beta1.{k}:",
                    f"dst := dstRaw.(*v1beta1.{k})"
                ]
                lines.extend(cg.deep_gen(cg.objects, k, "dst", "", "src", "", "v1beta1"))
                content.extend(lines)
                lines = [
                    "default:\nreturn fmt.Errorf(\"unsupported type %v\", t)\n}\nreturn nil\n}\n",
                    f"func (dst *{k}) ConvertFrom(srcRaw conversion.Hub) error " + "{",
                    "switch t := srcRaw.(type) {",
                    f"case *v1beta1.{k}:",
                    f"src := srcRaw.(*v1beta1.{k})"
                ]
                lines.extend(cg.deep_gen(cg.objects, k, "dst", "", "src", "", ""))
                content.extend(lines)
                content.append("default:\nreturn fmt.Errorf(\"unsupported type %v\", t)\n}\nreturn nil\n}\n")
                f.writelines(line + '\n' for line in content)
            os.system(f"go fmt {conversion}")

    def update_dependencies(self):
        file_list = []
        for path in config['dependencyUpdate']['filePath']:
            file_list.extend(self.get_all_files(path))
        is_kruise = re.compile(config['dependencyUpdate']['updatePackage'])
        for file in file_list:
            with open(file, 'r') as f:
                lines = f.readlines()
            new_lines = []
            contains = False
            in_import = False
            # check import
            patterns = []
            prefixes = []
            for line in lines:
                if "import (" in line or in_import:
                    if not in_import:
                        in_import = True
                    elif ")" in line:
                        in_import = False
                        if not contains:
                            new_lines = lines
                            break
                    elif is_kruise.findall(line):
                        tmp = line.strip()
                        prefix = tmp.split()
                        if len(prefix) > 1:
                            prefix = prefix[0]
                        else:
                            prefix = "v1alpha1"
                        line = line.replace("v1alpha1", "v1beta1")
                        new_prefix = prefix.replace("v1alpha1", "v1beta1")
                        prefixes.append(new_prefix)
                        patterns.append(f"(?<![a-zA-Z]){prefix}(?![a-zA-Z])")
                        contains = True
                elif contains and "apps.kruise.io/v1beta1" not in line:
                    line = line.replace("AppsV1alpha1()", "AppsV1beta1()")
                    line = line.replace("PolicyV1alpha1()", "PolicyV1beta1()")
                    for pattern in patterns:
                        line = re.sub(pattern, prefixes[patterns.index(pattern)], line)
                new_lines.append(line)
            with open(file, 'w') as f:
                f.writelines(new_lines)
            os.system(f"go fmt {file}")

    def change_kubebuilder_anno(self):
        files = []
        paths = config['webhookUpdate']['filePath']
        for path in paths:
            files.extend(self.get_all_files(path))
        for file in files:
            if not RE_WEBHOOK.findall(file):
                continue
            with open(file, "r") as f:
                lines = f.readlines()
            new_lines = []
            for line in lines:
                line = line.replace("versions=v1alpha1", "versions=v1alpha1;v1beta1")
                line = line.replace("-v1alpha1", "")
                new_lines.append(line)
            with open(file, "w") as f:
                f.writelines(new_lines)

    def get_all_files(self, directory):
        file_list = []
        for root, directories, files in os.walk(directory):
            if "client" in root[len(directory) - 1: ]:
                continue
            for file in files:
                if ".go" in file:
                    file_path = os.path.join(root, file)
                    file_list.append(file_path)
        return file_list

    def do_filter(self, conditions, line) -> bool:
        for cond in conditions:
            if cond in line:
                return True
        return False

    def do_replace(self, line, rep):
        for r in rep:
            line.replace(r['from'], r['to'])

if __name__ == "__main__":
    cg = ConversionGenerator()
    # cg.change_kubebuilder_anno()
    cg.update_dependencies()
    # cg.create_api()
    # cg.create_conversion_function()
    # cg.get_crds()
    # crds = cg.crds
    # cg = CodeGenerator("/Users/kuromesi/MyCOde/kind/share/kuromesi.com/kruise/apis/apps/v1alpha1")
    # cg.scan_code(crds)


from django.contrib import admin
from django.urls import path, include
from . import views

urlpatterns = [
    path('home', views.home, name = 'home'),
    path('signup', views.signup, name = 'signup'),
    path('signin', views.signin, name = 'signin'),
    path('signout', views.signout, name = 'signout'),
    path('science', views.science, name = 'science'),
    path('commerce', views.commerce, name = 'commerce'),
    path('arts', views.arts, name = 'arts'),
    path('tenth', views.tenth, name = 'tenth'),
    path('activate/<uidb64>/<token>', views.activate, name = 'activate'),

]


import pandas as pd
from datetime import datetime

df = pd.read_csv('https://query.data.world/s/tt5efa64gl6ruwmxz4wdgmhfwroi4r',
                 usecols=['longitude', 'latitude', 'region', 'admin1', 'location',
                          'event_type', 'fatalities', 'event_date'])

print('Loading in dataset from data.world/makeovermonday/2018w34-visualizing-conflict...')

df['event_type'] = ['Battle' if e[:6] == 'Battle' else e
                    for e in df['event_type']]

df['event_date'] = [datetime.strptime(d, "%d-%b-%y") for d in df['event_date']]
df['year_month'] = [str(d.year) + '_' + str(d.month) for d in df['event_date']]

df['region'] = [r if r == 'Middle East' else r.split()[1] for r in df['region']]

df.to_csv('data/world_conflict.csv', index=False)